diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..aa81b0c2 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,15 @@ +{ + "permissions": { + "allow": [ + "PowerShell(\"----- last commit -----\")", + "PowerShell(\"----- branch vs remote -----\")", + "PowerShell(\"HEAD: \" + \\(git rev-parse HEAD\\); \"origin/main: \" + \\(git rev-parse origin/main\\))", + "PowerShell(git add config.json config.py requirements.txt)", + "Bash(python -m py_compile agent.py config.py)", + "Bash(python -c \"import json; print\\('sherpa_num_threads =', json.load\\(open\\('config.json'\\)\\)['sherpa_num_threads']\\)\")", + "Bash(git checkout *)", + "Bash(git add *)", + "Bash(git commit *)" + ] + } +} diff --git a/.gitignore b/.gitignore index 72df27ad..7a666a09 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,7 @@ test_audio.wav .DS_Store Thumbs.db ehthumbs.db +whisper.cpp/ + +# --- Sleep Flow Cache --- +cache/ diff --git a/README.md b/README.md index 51646068..ba7b30b0 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ python agent.py You can modify the hardware behavior and personality in `config.json`. The `agent.py` script creates this on the first run if it doesn't exist, but you can create it manually: +For the sleep-focused Chinese companion flow in this fork, see the dedicated prompt guide: [睡前情绪梳理 Prompt 说明文档](docs/PROMPT_GUIDE.md). + ```json { "text_model": "gemma3:1b", diff --git a/agent.py b/agent.py index 1553842b..f220ddc5 100644 --- a/agent.py +++ b/agent.py @@ -1,1081 +1,967 @@ -# ========================================================================= -# Be More Agent 🤖 -# A Local, Offline-First AI Agent for Raspberry Pi -# -# Copyright (c) 2026 brenpoly -# Licensed under the MIT License -# Source: https://github.com/brenpoly/be-more-agent -# -# DISCLAIMER: -# This software is provided "as is", without warranty of any kind. -# This project is a generic framework and includes no copyrighted assets. -# ========================================================================= - -import tkinter as tk -from tkinter import ttk -from PIL import Image, ImageTk -import threading -import time -import json -import os -import subprocess -import random -import re -import sys -import select -import traceback -import atexit -import datetime -import warnings -import wave -import struct - -# Suppress harmless library warnings -warnings.filterwarnings("ignore", category=RuntimeWarning, module="duckduckgo_search") - -# Core dependencies -import sounddevice as sd -import numpy as np -import scipy.signal - -# --- AI ENGINES --- -import openwakeword -from openwakeword.model import Model -import ollama - -# --- WEB SEARCH (Using your working import) --- -from duckduckgo_search import DDGS - -# ========================================================================= -# 1. CONFIGURATION & CONSTANTS -# ========================================================================= - -CONFIG_FILE = "config.json" -MEMORY_FILE = "memory.json" -BMO_IMAGE_FILE = "current_image.jpg" -WAKE_WORD_MODEL = "./wakeword.onnx" -WAKE_WORD_THRESHOLD = 0.5 - -# HARDWARE SETTINGS -INPUT_DEVICE_NAME = None - -DEFAULT_CONFIG = { - "text_model": "gemma3:1b", - "vision_model": "moondream", - "voice_model": "piper/en_GB-semaine-medium.onnx", - "chat_memory": True, - "camera_rotation": 0, - "system_prompt_extras": "", - "input_device": None, - "input_sample_rate": None -} - -# LLM SETTINGS -OLLAMA_OPTIONS = { - 'keep_alive': '-1', - 'num_thread': 4, - 'temperature': 0.7, - 'top_k': 40, - 'top_p': 0.9 -} - -def load_config(): - config = DEFAULT_CONFIG.copy() - if os.path.exists(CONFIG_FILE): - try: - with open(CONFIG_FILE, "r") as f: - user_config = json.load(f) - config.update(user_config) - except Exception as e: - print(f"Config Error: {e}. Using defaults.") - return config - -CURRENT_CONFIG = load_config() -TEXT_MODEL = CURRENT_CONFIG["text_model"] -VISION_MODEL = CURRENT_CONFIG["vision_model"] - -def resolve_input_device(config): - requested = config.get("input_device") - if requested in (None, "", "default"): - return None - - try: - devices = sd.query_devices() - except Exception as e: - print(f"[AUDIO] Device query failed: {e}", flush=True) - return None - - if isinstance(requested, int) or (isinstance(requested, str) and requested.isdigit()): - index = int(requested) - if 0 <= index < len(devices): - return index - print(f"[AUDIO] Input device index not found: {index}", flush=True) - return None - - requested_lower = str(requested).lower() - for idx, dev in enumerate(devices): - print(f"[AUDIO DEBUG] Index {idx}: {dev.get('name')} (In: {dev.get('max_input_channels')})", flush=True) # DEBUG LINE - if dev.get("max_input_channels", 0) > 0 and requested_lower in dev.get("name", "").lower(): - return idx - - print(f"[AUDIO] Input device name not found: {requested}", flush=True) - return None - -INPUT_DEVICE_NAME = resolve_input_device(CURRENT_CONFIG) -if INPUT_DEVICE_NAME is not None: - try: - device_info = sd.query_devices(INPUT_DEVICE_NAME) - print(f"[AUDIO] Using input device: {device_info.get('name', INPUT_DEVICE_NAME)}", flush=True) - except Exception: - print(f"[AUDIO] Using input device index: {INPUT_DEVICE_NAME}", flush=True) - -def choose_input_samplerate(device, preferred=None): - candidates = [] - if preferred: - candidates.append(preferred) - try: - device_info = sd.query_devices(device) - print(f"[AUDIO DEBUG] Device Info: {device_info}", flush=True) # DEBUG - if "default_samplerate" in device_info: - candidates.append(int(device_info["default_samplerate"])) - except Exception as e: - print(f"[AUDIO DEBUG] Query failed: {e}", flush=True) - pass - - candidates.extend([48000, 44100, 32000, 16000]) - seen = set() - for rate in candidates: - if not rate or rate in seen: - continue - seen.add(rate) - try: - sd.check_input_settings(device=device, samplerate=rate, channels=1, dtype="int16") - return rate - except Exception: - continue - - return int(candidates[0]) if candidates else 44100 - -class BotStates: - IDLE = "idle" - LISTENING = "listening" - THINKING = "thinking" - SPEAKING = "speaking" - ERROR = "error" - CAPTURING = "capturing" - WARMUP = "warmup" - -# --- SYSTEM PROMPT --- -BASE_SYSTEM_PROMPT = """You are a helpful robot assistant running on a Raspberry Pi. -Personality: Cute, helpful, robot. -Style: Short sentences. Enthusiastic. - -INSTRUCTIONS: -- If the user asks for a physical action (time, search, photo), output JSON. -- If the user just wants to chat, reply with NORMAL TEXT. - -### EXAMPLES ### - -User: What time is it? -You: {"action": "get_time", "value": "now"} - -User: Hello! -You: Hi! I am ready to help! - -User: Search for news about robots. -You: {"action": "search_web", "value": "robots news"} - -User: What do you see right now? -You: {"action": "capture_image", "value": "environment"} - -### END EXAMPLES ### -""" - -SYSTEM_PROMPT = BASE_SYSTEM_PROMPT + "\n\n" + CURRENT_CONFIG.get("system_prompt_extras", "") - -# Sound Directories -greeting_sounds_dir = "sounds/greeting_sounds" -ack_sounds_dir = "sounds/ack_sounds" -thinking_sounds_dir = "sounds/thinking_sounds" -error_sounds_dir = "sounds/error_sounds" - -# ========================================================================= -# 2. GUI CLASS -# ========================================================================= - -class BotGUI: - BG_WIDTH, BG_HEIGHT = 800, 480 - OVERLAY_WIDTH, OVERLAY_HEIGHT = 400, 300 - - def __init__(self, master): - self.master = master - master.title("Pi Assistant") - master.attributes('-fullscreen', True) - master.bind('', self.exit_fullscreen) - - # Inputs - master.bind('', self.handle_ptt_toggle) - master.bind('', self.handle_speaking_interrupt) - atexit.register(self.safe_exit) - - # State - self.current_state = BotStates.WARMUP - self.current_volume = 0 - self.animations = {} - self.current_frame_index = 0 - self.current_overlay_image = None - - self.permanent_memory = self.load_chat_history() - self.session_memory = [] - self.thinking_sound_active = threading.Event() - - self.last_ptt_time = 0 - self.ptt_event = threading.Event() - self.recording_active = threading.Event() - self.interrupted = threading.Event() - - self.tts_queue = [] - self.tts_queue_lock = threading.Lock() - self.tts_thread = None - self.tts_active = threading.Event() - self.current_audio_process = None - self.exiting = False - - # --- WAKE WORD INITIALIZATION --- - print("[INIT] Loading Wake Word...", flush=True) - self.oww_model = None - if os.path.exists(WAKE_WORD_MODEL): - try: - self.oww_model = Model(wakeword_model_paths=[WAKE_WORD_MODEL]) - print("[INIT] Wake Word Loaded.", flush=True) - except TypeError: - try: - self.oww_model = Model(wakeword_models=[WAKE_WORD_MODEL]) - print("[INIT] Wake Word Loaded (New API).", flush=True) - except Exception as e: - print(f"[CRITICAL] Failed to load model: {e}") - except Exception as e: - print(f"[CRITICAL] Failed to load model: {e}") - else: - print(f"[CRITICAL] Model not found: {WAKE_WORD_MODEL}") - - # GUI Setup - self.background_label = tk.Label(master) - self.background_label.place(x=0, y=0, width=self.BG_WIDTH, height=self.BG_HEIGHT) - self.background_label.bind('', self.toggle_hud_visibility) - - self.overlay_label = tk.Label(master, bg='black') - self.overlay_label.bind('', self.toggle_hud_visibility) - - self.response_text = tk.Text(master, height=6, width=60, wrap=tk.WORD, - state=tk.DISABLED, bg="#ffffff", fg="#000000", font=('Arial', 12)) - - self.status_var = tk.StringVar(value="Initializing...") - self.status_label = ttk.Label(master, textvariable=self.status_var, background="#2e2e2e", foreground="white") - - self.exit_button = ttk.Button(master, text="Exit & Save", command=self.safe_exit) - - self.load_animations() - self.update_animation() - - threading.Thread(target=self.safe_main_execution, daemon=True).start() - - # --- HELPERS --- - - def extract_json_from_text(self, text): - try: - match = re.search(r'\{.*\}', text, re.DOTALL) - if match: - return json.loads(match.group(0)) - return None - except: return None - - def safe_exit(self): - if self.exiting: - return - self.exiting = True - print("\n--- SHUTDOWN SEQUENCE ---", flush=True) - if self.current_audio_process: - try: - self.current_audio_process.terminate() - self.current_audio_process.wait(timeout=1) - except: pass - - self.recording_active.clear() - self.thinking_sound_active.clear() - self.tts_active.clear() - - self.save_chat_history() - - try: - ollama.generate(model=TEXT_MODEL, prompt="", keep_alive=0) - except: pass - try: - sd.stop() - except: pass - - try: - self.master.quit() - except Exception: - pass - - def exit_fullscreen(self, event=None): - self.master.attributes('-fullscreen', False) - self.safe_exit() - - def toggle_hud_visibility(self, event=None): - try: - if self.response_text.winfo_ismapped(): - self.response_text.place_forget() - self.status_label.place_forget() - self.exit_button.place_forget() - else: - self.response_text.place(relx=0.5, rely=0.82, anchor=tk.S) - self.status_label.place(relx=0.5, rely=1.0, anchor=tk.S, relwidth=1) - self.exit_button.place(x=10, y=10) - except tk.TclError: pass - - def handle_ptt_toggle(self, event=None): - current_time = time.time() - if current_time - self.last_ptt_time < 0.5: - return - self.last_ptt_time = current_time - - if self.recording_active.is_set(): - print("[PTT] Toggle OFF", flush=True) - self.recording_active.clear() - else: - if self.current_state == BotStates.IDLE or "Wait" in self.status_var.get(): - print("[PTT] Toggle ON", flush=True) - self.recording_active.set() - self.ptt_event.set() - - def handle_speaking_interrupt(self, event=None): - if self.current_state == BotStates.SPEAKING or self.current_state == BotStates.THINKING: - self.interrupted.set() - self.thinking_sound_active.clear() - with self.tts_queue_lock: - self.tts_queue.clear() - if self.current_audio_process: - try: self.current_audio_process.terminate() - except: pass - self.set_state(BotStates.IDLE, "Interrupted.") - - def load_animations(self): - base_path = "faces" - states = ["idle", "listening", "thinking", "speaking", "error", "capturing", "warmup"] - for state in states: - folder = os.path.join(base_path, state) - self.animations[state] = [] - if os.path.exists(folder): - files = sorted([f for f in os.listdir(folder) if f.lower().endswith('.png')]) - for f in files: - img = Image.open(os.path.join(folder, f)).resize((self.BG_WIDTH, self.BG_HEIGHT)) - self.animations[state].append(ImageTk.PhotoImage(img)) - if not self.animations[state]: - if state in self.animations.get("idle", []): - self.animations[state] = self.animations["idle"] - else: - # Blue screen fallback - blank = Image.new('RGB', (self.BG_WIDTH, self.BG_HEIGHT), color='#0000FF') - self.animations[state].append(ImageTk.PhotoImage(blank)) - - def update_animation(self): - frames = self.animations.get(self.current_state, []) or self.animations.get(BotStates.IDLE, []) - if not frames: - self.master.after(500, self.update_animation) - return - - if self.current_state == BotStates.SPEAKING: - if len(frames) > 1: - self.current_frame_index = random.randint(1, len(frames) - 1) - else: - self.current_frame_index = 0 - else: - self.current_frame_index = (self.current_frame_index + 1) % len(frames) - - self.background_label.config(image=frames[self.current_frame_index]) - - speed = 50 if self.current_state == BotStates.SPEAKING else 500 - self.master.after(speed, self.update_animation) - - def set_state(self, state, msg="", cam_path=None): - def _update(): - if msg: print(f"[STATE] {state.upper()}: {msg}", flush=True) - if self.current_state != state: - self.current_state = state - self.current_frame_index = 0 - if msg: self.status_var.set(msg) - if cam_path and os.path.exists(cam_path) and state in [BotStates.THINKING, BotStates.SPEAKING]: - try: - img = Image.open(cam_path).resize((self.OVERLAY_WIDTH, self.OVERLAY_HEIGHT)) - self.current_overlay_image = ImageTk.PhotoImage(img) - self.overlay_label.config(image=self.current_overlay_image) - self.overlay_label.place(x=200, y=90) - except: pass - else: - self.overlay_label.place_forget() - self.master.after(0, _update) - - def append_to_text(self, text, newline=True): - def _update(): - self.response_text.config(state=tk.NORMAL) - if newline: - self.response_text.insert(tk.END, text + "\n") - else: - self.response_text.insert(tk.END, text) - - self.response_text.see(tk.END) - self.response_text.config(state=tk.DISABLED) - - self.master.after(0, _update) - - def _stream_to_text(self, chunk): - def update_text_stream(): - self.response_text.config(state=tk.NORMAL) - self.response_text.insert(tk.END, chunk) - self.response_text.see(tk.END) - self.response_text.config(state=tk.DISABLED) - self.master.after(0, update_text_stream) - - # ========================================================================= - # 3. ACTION ROUTER - # ========================================================================= - - def execute_action_and_get_result(self, action_data): - raw_action = action_data.get("action", "").lower().strip() - value = action_data.get("value") or action_data.get("query") - - VALID_TOOLS = { - "get_time", "search_web", "capture_image" - } - - ALIASES = { - "google": "search_web", "browser": "search_web", "news": "search_web", - "search_news": "search_web", "look": "capture_image", "see": "capture_image", - "check_time": "get_time" - } - - action = ALIASES.get(raw_action, raw_action) - print(f"ACTION: {raw_action} -> {action}", flush=True) - - if action not in VALID_TOOLS: - if value and isinstance(value, str) and len(value.split()) > 1: - return f"CHAT_FALLBACK::{value}" - return "INVALID_ACTION" - - if action == "get_time": - now = datetime.datetime.now().strftime("%I:%M %p") - return f"The current time is {now}." - - elif action == "search_web": - print(f"Searching web for: {value}...", flush=True) - try: - # 'us-en' region is often more stable for CLI queries - with DDGS() as ddgs: - results = [] - # 1. News search - try: - results = list(ddgs.news(value, region='us-en', max_results=1)) - if results: - print(f"[DEBUG] Found News: {results[0].get('title')}", flush=True) - except Exception as e: - print(f"[DEBUG] News Search Error: {e}", flush=True) - - # 2. Text fallback - if not results: - print("[DEBUG] No news found, trying text search...", flush=True) - try: - results = list(ddgs.text(value, region='us-en', max_results=1)) - if results: - print(f"[DEBUG] Found Text: {results[0].get('title')}", flush=True) - except Exception as e: - print(f"[DEBUG] Text Search Error: {e}", flush=True) - - if results: - r = results[0] - # Safe get - title = r.get('title', 'No Title') - body = r.get('body', r.get('snippet', 'No Body')) - return f"SEARCH RESULTS for '{value}':\nTitle: {title}\nSnippet: {body[:300]}" - else: - print(f"[DEBUG] Search returned 0 results.", flush=True) - return "SEARCH_EMPTY" - except Exception as e: - print(f"[DEBUG] Connection/Library Error: {e}", flush=True) - return "SEARCH_ERROR" - - elif action == "capture_image": - return "IMAGE_CAPTURE_TRIGGERED" - - return None - - # ========================================================================= - # 4. CORE LOGIC - # ========================================================================= - - def safe_main_execution(self): - try: - self.warm_up_logic() - self.tts_active.set() - self.tts_thread = threading.Thread(target=self._tts_worker, daemon=True) - self.tts_thread.start() - - while True: - trigger_source = self.detect_wake_word_or_ptt() - if self.interrupted.is_set(): - self.interrupted.clear() - self.set_state(BotStates.IDLE, "Resetting...") - continue - - self.set_state(BotStates.LISTENING, "I'm listening!") - - audio_file = None - if trigger_source == "PTT": - audio_file = self.record_voice_ptt() - else: - audio_file = self.record_voice_adaptive() - - if not audio_file: - self.set_state(BotStates.IDLE, "Heard nothing.") - continue - - user_text = self.transcribe_audio(audio_file) - if not user_text: - self.set_state(BotStates.IDLE, "Transcription empty.") - continue - - self.append_to_text(f"YOU: {user_text}") - self.interrupted.clear() - self.chat_and_respond(user_text, img_path=None) - - except Exception as e: - traceback.print_exc() - self.set_state(BotStates.ERROR, f"Fatal Error: {str(e)[:40]}") - - def warm_up_logic(self): - self.set_state(BotStates.WARMUP, "Warming up brains...") - try: - ollama.generate(model=TEXT_MODEL, prompt="", keep_alive=-1) - except Exception as e: - print(f"Failed to load {TEXT_MODEL}: {e}", flush=True) - self.play_sound(self.get_random_sound(greeting_sounds_dir)) - print("Models loaded.", flush=True) - - def detect_wake_word_or_ptt(self): - self.set_state(BotStates.IDLE, "Waiting...") - self.ptt_event.clear() - - if self.oww_model: self.oww_model.reset() - - if self.oww_model is None: - self.ptt_event.wait() - self.ptt_event.clear() - return "PTT" - - CHUNK_SIZE = 1280 - OWW_SAMPLE_RATE = 16000 - - input_rate = choose_input_samplerate(INPUT_DEVICE_NAME, CURRENT_CONFIG.get("input_sample_rate")) - use_resampling = (input_rate != OWW_SAMPLE_RATE) - input_chunk_size = int(CHUNK_SIZE * (input_rate / OWW_SAMPLE_RATE)) if use_resampling else CHUNK_SIZE - - stream_args = { - "samplerate": input_rate, - "channels": 1, - "dtype": 'int16', - "blocksize": input_chunk_size, - "device": INPUT_DEVICE_NAME - } - - # Try to find a compatible block size and sample rate - try: - # First attempt: standard settings - self._listen_loop(stream_args, input_chunk_size, CHUNK_SIZE, use_resampling) - except StopIteration as si: - return str(si) - except Exception as e: - print(f"[AUDIO] Stream failed with defaults: {e}. Retrying with loose settings...", flush=True) - try: - # Second attempt: Let PortAudio decide blocksize (0) and latency - stream_args["blocksize"] = 0 - stream_args["latency"] = "high" - # If blocksize is variable, we must read specific amounts manually or handle buffering. - # Simplest fallback: Just attempt small fixed block - stream_args["blocksize"] = 1024 - use_resampling = True - - self._listen_loop(stream_args, 1024, CHUNK_SIZE, use_resampling) - except StopIteration as si: - return str(si) - except Exception as e2: - print(f"[CRITICAL] Wake Word Stream Error: {e2}") - self.ptt_event.wait() - return "PTT" - - return "WAKE" - - def _listen_loop(self, stream_args, input_chunk_size, target_chunk_size, use_resampling): - # Force software backend (no mmap) via environment variable if possible, - # but here we can try to hint loop settings. - # However, the most effective fix for ALSA mmap issues is often just asking for 'blocksize=0' - # and letting portaudio manage the buffering, OR very small chunks. - - # Let's try to be less aggressive with reads. - - with sd.InputStream(**stream_args) as stream: - print(f"[AUDIO] Listening with rate {stream_args['samplerate']} and block {stream_args['blocksize']}", flush=True) - - # Pre-allocate buffer for speed - # If blocksize is 0, we read what is available. - - while True: - if self.ptt_event.is_set(): - self.ptt_event.clear() - raise StopIteration("PTT") - - rlist, _, _ = select.select([sys.stdin], [], [], 0.001) - if rlist: - sys.stdin.readline() - raise StopIteration("CLI") - - # If fallback mode (blocksize 0), read fixed amount - read_size = input_chunk_size - if stream_args.get('blocksize') == 0: - read_size = 1024 # Safe small read - - try: - data, overflow = stream.read(read_size) - if overflow: - print("!", end="", flush=True) - # If we overflow excessively, raise error to trigger fallback to SAFE MODE (PulseAudio/Software) - # We can use a simple counter attached to the function or object, but here raising immediately - # after a few in a row is safest. - raise RuntimeError("Audio Buffer Overflow - Triggering Safe Mode") - except Exception as e: - # Convert uncatchable PaErrorCode wrapper to standard Exception if needed - # But honestly, `raise e` should work... unless it's a SystemExit? - # Let's wrap it in a new exception to be sure it bubbles up - raise RuntimeError(f"Audio read failed: {e}") - - audio_data = np.frombuffer(data, dtype=np.int16) - - # Ensure flattening for openwakeword compatibility - if audio_data.ndim > 1: - audio_data = audio_data.flatten() - - if use_resampling: - # FAST RESAMPLING: Nearest-neighbor slicing instead of scipy.signal.resample - # This avoids the CPU bottleneck that causes overflow (!!!!!!!) on Raspberry Pi - step = len(audio_data) / target_chunk_size - indices = np.arange(0, len(audio_data), step)[:target_chunk_size].astype(int) - audio_data = audio_data[indices] - - # Convert to float for model prediction without needing heavy resampling logic - # The wake word model needs 16000, which we just faked above. - - # Debug volume occasionally - current_max = np.max(np.abs(audio_data)) - - # Only predict if volume is significant to save CPU - if current_max > 200: - prediction = self.oww_model.predict(audio_data) - for mdl in self.oww_model.prediction_buffer.keys(): - score = list(self.oww_model.prediction_buffer[mdl])[-1] - if score > 0.1: # Show potential triggers - print(f"\r[Oww] Score: {score:.3f} | Vol: {current_max} ", end="", flush=True) - - if score > WAKE_WORD_THRESHOLD: - print(f"\n[WAKE] Triggered on '{mdl}' with score: {score:.2f}", flush=True) - self.oww_model.reset() - return # Success - - - def record_voice_adaptive(self, filename="input.wav"): - print("Recording (Adaptive)...", flush=True) - time.sleep(0.5) - samplerate = choose_input_samplerate(INPUT_DEVICE_NAME, CURRENT_CONFIG.get("input_sample_rate")) - - silence_threshold = 0.006 - silence_duration = 1.5 - max_record_time = 30.0 - buffer = [] - silent_chunks = 0 - chunk_duration = 0.05 - chunk_size = int(samplerate * chunk_duration) - - num_silent_chunks = int(silence_duration / chunk_duration) - max_chunks = int(max_record_time / chunk_duration) - recorded_chunks = 0 - silence_started = False - - def callback(indata, frames, time_info, status): - nonlocal silent_chunks, recorded_chunks, silence_started - volume_norm = np.linalg.norm(indata) / np.sqrt(len(indata)) - buffer.append(indata.copy()) - recorded_chunks += 1 - if recorded_chunks < 5: return - if volume_norm < silence_threshold: - silent_chunks += 1 - if silent_chunks >= num_silent_chunks: silence_started = True - else: silent_chunks = 0 - - try: - # Explicitly close stream if it exists to free hardware - sd.stop() - time.sleep(0.2) - - with sd.InputStream(samplerate=samplerate, channels=1, callback=callback, - device=INPUT_DEVICE_NAME, blocksize=chunk_size): - while not silence_started and recorded_chunks < max_chunks: - sd.sleep(int(chunk_duration * 1000)) - except Exception as e: - print(f"[AUDIO ERROR] Adaptive Recording Failed: {e}", flush=True) - return None - - return self.save_audio_buffer(buffer, filename, samplerate) - - def record_voice_ptt(self, filename="input.wav"): - print("Recording (PTT)...", flush=True) - time.sleep(0.5) - samplerate = choose_input_samplerate(INPUT_DEVICE_NAME, CURRENT_CONFIG.get("input_sample_rate")) - - buffer = [] - def callback(indata, frames, time_info, status): buffer.append(indata.copy()) - - try: - # Explicitly close stream if it exists to free hardware - # This is critical on Pi 5 where hardware contention causes freezes - sd.stop() - time.sleep(0.2) - - with sd.InputStream(samplerate=samplerate, channels=1, callback=callback, device=INPUT_DEVICE_NAME): - while self.recording_active.is_set(): - sd.sleep(50) - except Exception as e: - print(f"[AUDIO ERROR] PTT Recording Failed: {e}", flush=True) - return None - - return self.save_audio_buffer(buffer, filename, samplerate) - - def save_audio_buffer(self, buffer, filename, samplerate=16000): - if not buffer: return None - audio_data = np.concatenate(buffer, axis=0).flatten() - audio_data = np.nan_to_num(audio_data, nan=0.0, posinf=0.0, neginf=0.0) - audio_data = (audio_data * 32767).astype(np.int16) - with wave.open(filename, "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(samplerate) - wf.writeframes(audio_data.tobytes()) - self.play_sound(self.get_random_sound(ack_sounds_dir)) - return filename - - def transcribe_audio(self, filename): - print("Transcribing...", flush=True) - try: - result = subprocess.run( - ["./whisper.cpp/build/bin/whisper-cli", "-m", "./whisper.cpp/models/ggml-base.en.bin", "-l", "en", "-t", "4", "-f", filename], - capture_output=True, text=True - ) - transcription_lines = result.stdout.strip().split('\n') - if transcription_lines and transcription_lines[-1].strip(): - last_line = transcription_lines[-1].strip() - if ']' in last_line: transcription = last_line.split("]")[1].strip() - else: transcription = last_line - else: transcription = "" - print(f"Heard: '{transcription}'", flush=True) - return transcription.strip() - except Exception as e: - print(f"Transcription Error: {e}") - return "" - - def capture_image(self): - self.set_state(BotStates.CAPTURING, "Watching...") - try: - subprocess.run(["rpicam-still", "-t", "500", "-n", "--width", "640", "--height", "480", "-o", BMO_IMAGE_FILE], check=True) - rotation = CURRENT_CONFIG.get("camera_rotation", 0) - if rotation != 0: - img = Image.open(BMO_IMAGE_FILE) - img = img.rotate(rotation, expand=True) - img.save(BMO_IMAGE_FILE) - return BMO_IMAGE_FILE - except Exception as e: - print(f"Camera Error: {e}") - return None - - # ========================================================================= - # 5. CHAT & RESPOND - # ========================================================================= - - def chat_and_respond(self, text, img_path=None): - if "forget everything" in text.lower() or "reset memory" in text.lower(): - self.session_memory = [] - self.permanent_memory = [{"role": "system", "content": SYSTEM_PROMPT}] - self.save_chat_history() - with self.tts_queue_lock: - self.tts_queue.append("Okay. Memory wiped.") - self.set_state(BotStates.IDLE, "Memory Wiped") - return - - model_to_use = VISION_MODEL if img_path else TEXT_MODEL - self.set_state(BotStates.THINKING, "Thinking...", cam_path=img_path) - - messages = [] - if img_path: - messages = [{"role": "user", "content": text, "images": [img_path]}] - else: - user_msg = {"role": "user", "content": text} - messages = self.permanent_memory + self.session_memory + [user_msg] - - self.thinking_sound_active.set() - threading.Thread(target=self._run_thinking_sound_loop, daemon=True).start() - - full_response_buffer = "" - sentence_buffer = "" - - try: - stream = ollama.chat(model=model_to_use, messages=messages, stream=True, options=OLLAMA_OPTIONS) - - is_action_mode = False - - for chunk in stream: - if self.interrupted.is_set(): break - content = chunk['message']['content'] - full_response_buffer += content - - if '{"' in content or "action:" in content.lower(): - is_action_mode = True - self.thinking_sound_active.clear() - continue - - if is_action_mode: continue - - self.thinking_sound_active.clear() - if self.current_state != BotStates.SPEAKING: - self.set_state(BotStates.SPEAKING, "Speaking...", cam_path=img_path) - self.append_to_text("BOT: ", newline=False) - - self._stream_to_text(content) - - sentence_buffer += content - if any(punct in content for punct in ".!?\n"): - clean_sentence = sentence_buffer.strip() - if clean_sentence and re.search(r'[a-zA-Z0-9]', clean_sentence): - with self.tts_queue_lock: self.tts_queue.append(clean_sentence) - sentence_buffer = "" - - if is_action_mode: - action_data = self.extract_json_from_text(full_response_buffer) - if action_data: - tool_result = self.execute_action_and_get_result(action_data) - - if tool_result and tool_result.startswith("CHAT_FALLBACK::"): - chat_text = tool_result.split("::", 1)[1] - self.thinking_sound_active.clear() - self.set_state(BotStates.SPEAKING, "Speaking...", cam_path=img_path) - self.append_to_text("BOT: ", newline=False) - self.append_to_text(chat_text, newline=True) - with self.tts_queue_lock: self.tts_queue.append(chat_text) - self.session_memory.append({"role": "assistant", "content": chat_text}) - self.wait_for_tts() - self.set_state(BotStates.IDLE, "Ready") - return - - if tool_result == "IMAGE_CAPTURE_TRIGGERED": - new_img_path = self.capture_image() - if new_img_path: - self.chat_and_respond(text, img_path=new_img_path) - return - - elif tool_result == "INVALID_ACTION": - fallback_text = "I am not sure how to do that." - self.thinking_sound_active.clear() - self.set_state(BotStates.SPEAKING, "Speaking...", cam_path=img_path) - self.append_to_text("BOT: ", newline=False) - self.append_to_text(fallback_text, newline=True) - with self.tts_queue_lock: self.tts_queue.append(fallback_text) - - elif tool_result == "SEARCH_EMPTY": - fallback_text = "I searched, but I couldn't find any news about that." - self.thinking_sound_active.clear() - self.set_state(BotStates.SPEAKING, "Speaking...", cam_path=img_path) - self.append_to_text("BOT: ", newline=False) - self.append_to_text(fallback_text, newline=True) - with self.tts_queue_lock: self.tts_queue.append(fallback_text) - - elif tool_result == "SEARCH_ERROR": - fallback_text = "I cannot reach the internet right now." - self.thinking_sound_active.clear() - self.set_state(BotStates.SPEAKING, "Speaking...", cam_path=img_path) - self.append_to_text("BOT: ", newline=False) - self.append_to_text(fallback_text, newline=True) - with self.tts_queue_lock: self.tts_queue.append(fallback_text) - - elif tool_result: - summary_prompt = [ - {"role": "system", "content": "Summarize this result in one short sentence."}, - {"role": "user", "content": f"RESULT: {tool_result}\nUser Question: {text}"} - ] - - self.set_state(BotStates.THINKING, "Reading...") - self.thinking_sound_active.set() - - final_resp = ollama.chat(model=model_to_use, messages=summary_prompt, stream=False, options=OLLAMA_OPTIONS) - final_text = final_resp['message']['content'] - - self.thinking_sound_active.clear() - self.set_state(BotStates.SPEAKING, "Speaking...", cam_path=img_path) - - self.append_to_text("BOT: ", newline=False) - self.append_to_text(final_text, newline=True) - with self.tts_queue_lock: self.tts_queue.append(final_text) - self.session_memory.append({"role": "assistant", "content": final_text}) - else: - self.append_to_text("") - self.session_memory.append({"role": "assistant", "content": full_response_buffer}) - - self.wait_for_tts() - self.set_state(BotStates.IDLE, "Ready") - - except Exception as e: - print(f"LLM Error: {e}") - self.set_state(BotStates.ERROR, "Brain Freeze!") - - def wait_for_tts(self): - while self.tts_queue or self.tts_active.is_set(): - if self.interrupted.is_set(): break - time.sleep(0.1) - - def _tts_worker(self): - while True: - text = None - with self.tts_queue_lock: - if self.tts_queue: - text = self.tts_queue.pop(0) - self.tts_active.set() - if text: - self.speak(text) - self.tts_active.clear() - else: time.sleep(0.05) - - def speak(self, text): - clean = re.sub(r"[^\w\s,.!?:-]", "", text) - if not clean.strip(): return - - print(f"[PIPER SPEAKING] '{clean}'", flush=True) - voice_model = CURRENT_CONFIG.get("voice_model", "piper/en_GB-semaine-medium.onnx") - - try: - self.current_audio_process = subprocess.Popen( - ["./piper/piper", "--model", voice_model, "--output-raw"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL - ) - - self.current_audio_process.stdin.write(clean.encode() + b'\n') - self.current_audio_process.stdin.close() - - try: - device_info = sd.query_devices(kind='output') - native_rate = int(device_info['default_samplerate']) - except: - native_rate = 48000 - - PIPER_RATE = 22050 - use_native_rate = False - - try: - sd.check_output_settings(device=None, samplerate=PIPER_RATE) - except: - use_native_rate = True - - with sd.RawOutputStream(samplerate=native_rate if use_native_rate else PIPER_RATE, - channels=1, dtype='int16', - device=None, latency='low', blocksize=2048) as stream: - while True: - if self.interrupted.is_set(): break - data = self.current_audio_process.stdout.read(4096) - if not data: break - - audio_chunk = np.frombuffer(data, dtype=np.int16) - if len(audio_chunk) > 0: - self.current_volume = np.max(np.abs(audio_chunk)) - if use_native_rate: - num_samples = int(len(audio_chunk) * (native_rate / PIPER_RATE)) - audio_chunk = scipy.signal.resample(audio_chunk, num_samples).astype(np.int16) - stream.write(audio_chunk.tobytes()) - else: - self.current_volume = 0 - time.sleep(0.5) - - except Exception as e: - print(f"Audio Error: {e}") - finally: - self.current_volume = 0 - if self.current_audio_process: - if self.current_audio_process.stdout: self.current_audio_process.stdout.close() - if self.current_audio_process.poll() is None: self.current_audio_process.terminate() - self.current_audio_process = None - - def _run_thinking_sound_loop(self): - time.sleep(0.5) - while self.thinking_sound_active.is_set(): - sound = self.get_random_sound(thinking_sounds_dir) - if sound: self.play_sound(sound) - for _ in range(50): - if not self.thinking_sound_active.is_set(): return - time.sleep(0.1) - - def get_random_sound(self, directory): - if os.path.exists(directory): - files = [f for f in os.listdir(directory) if f.endswith(".wav")] - return os.path.join(directory, random.choice(files)) if files else None - return None - - def play_sound(self, file_path): - if not file_path or not os.path.exists(file_path): return - try: - with wave.open(file_path, 'rb') as wf: - file_sr = wf.getframerate() - data = wf.readframes(wf.getnframes()) - audio = np.frombuffer(data, dtype=np.int16) - - try: - device_info = sd.query_devices(kind='output') - native_rate = int(device_info['default_samplerate']) - except: - native_rate = 48000 - - playback_rate = file_sr - try: - sd.check_output_settings(device=None, samplerate=file_sr) - except: - playback_rate = native_rate - num_samples = int(len(audio) * (native_rate / file_sr)) - audio = scipy.signal.resample(audio, num_samples).astype(np.int16) - - sd.play(audio, playback_rate) - sd.wait() - except: pass - - def load_chat_history(self): - if os.path.exists(MEMORY_FILE): - try: - with open(MEMORY_FILE, "r") as f: return json.load(f) - except: pass - return [{"role": "system", "content": SYSTEM_PROMPT}] - - def save_chat_history(self): - full = self.permanent_memory + self.session_memory - conv = full[1:] - if len(conv) > 10: conv = conv[-10:] - with open(MEMORY_FILE, "w") as f: - json.dump([full[0]] + conv, f, indent=4) - -if __name__ == "__main__": - print("--- SYSTEM STARTING ---", flush=True) - root = tk.Tk() - app = BotGUI(root) - root.mainloop() +# ========================================================================= +# Be More Agent 🤖 +# A Local, Offline-First AI Agent for Raspberry Pi +# +# Copyright (c) 2026 brenpoly +# Licensed under the MIT License +# Source: https://github.com/brenpoly/be-more-agent +# +# DISCLAIMER: +# This software is provided "as is", without warranty of any kind. +# This project is a generic framework and includes no copyrighted assets. +# ========================================================================= + +import tkinter as tk +from tkinter import ttk +from PIL import Image, ImageTk +import threading +import time +import json +import os +import subprocess +import random +import re +import sys +import select +import traceback +import atexit +import wave +import collections + +# Core dependencies +import sounddevice as sd +import numpy as np +import scipy.signal +import webrtcvad + +# --- AI ENGINES --- +import openwakeword +from openwakeword.model import Model +import ollama + +# ========================================================================= +# 1. 配置 / 提示词从内部模块导入（拆出 config.py / prompts.py） +# ========================================================================= +from config import ( + MEMORY_FILE, WAKE_WORD_MODEL, WAKE_WORD_THRESHOLD, + INPUT_DEVICE_NAME, OLLAMA_OPTIONS, CURRENT_CONFIG, TEXT_MODEL, + BotStates, timed_block, choose_input_samplerate, +) +from prompts import SYSTEM_PROMPT + +# ========================================================================= +# 2. GUI CLASS +# ========================================================================= + +class BotGUI: + BG_WIDTH, BG_HEIGHT = 800, 480 + + def __init__(self, master): + self.master = master + master.title("Pi Assistant") + master.attributes('-fullscreen', True) + self.is_fullscreen = True + master.bind('', self.toggle_fullscreen) # 只切换全屏，不退程序 + master.bind('', lambda e: self.safe_exit()) # 退出程序 + + # Inputs + master.bind('', self.handle_ptt_toggle) + master.bind('', self.handle_speaking_interrupt) + atexit.register(self.safe_exit) + master.focus_force() # 抢焦点，确保 Escape 等按键能被窗口收到 + + # State + self.current_state = BotStates.WARMUP + self.current_volume = 0 + self.animations = {} + self.current_frame_index = 0 + + self.permanent_memory = self.load_chat_history() + self.session_memory = [] + + self.last_ptt_time = 0 + self.ptt_event = threading.Event() + self.recording_active = threading.Event() + self.interrupted = threading.Event() + + self.tts_queue = [] + self.tts_queue_lock = threading.Lock() + self.tts_active = threading.Event() + self.current_audio_process = None + + # --- TTS 两级流水线：合成线程提前渲染，播放线程只管播，消除句间空挡 --- + self.audio_queue = [] # 已渲染音频：(samples float32, rate) + self.audio_queue_lock = threading.Lock() + self.audio_queue_max = 2 # 提前渲染深度（背压上限） + self.synth_active = threading.Event() # 合成线程正在合成某句 + self.play_active = threading.Event() # 播放线程正在播某句 + self.synth_thread = None + self.play_thread = None + self.exiting = False + + # --- WAKE WORD INITIALIZATION --- + print("[INIT] Loading Wake Word...", flush=True) + self.oww_model = None + if os.path.exists(WAKE_WORD_MODEL): + try: + self.oww_model = Model(wakeword_model_paths=[WAKE_WORD_MODEL]) + print("[INIT] Wake Word Loaded.", flush=True) + except TypeError: + try: + self.oww_model = Model(wakeword_models=[WAKE_WORD_MODEL]) + print("[INIT] Wake Word Loaded (New API).", flush=True) + except Exception as e: + print(f"[CRITICAL] Failed to load model: {e}") + except Exception as e: + print(f"[CRITICAL] Failed to load model: {e}") + else: + print(f"[CRITICAL] Model not found: {WAKE_WORD_MODEL}") + + # --- SHERPA TTS INITIALIZATION --- + self.sherpa_tts = None + if CURRENT_CONFIG.get("tts_engine") == "sherpa": + self._init_sherpa_tts() + + # GUI Setup + self.background_label = tk.Label(master) + self.background_label.place(x=0, y=15, width=self.BG_WIDTH, height=450) + self.background_label.bind('', self.toggle_hud_visibility) + + self.response_text = tk.Text(master, height=6, width=60, wrap=tk.WORD, + state=tk.DISABLED, bg="#ffffff", fg="#000000", font=('Arial', 12)) + + self.status_var = tk.StringVar(value="Initializing...") + self.status_label = ttk.Label(master, textvariable=self.status_var, background="#2e2e2e", foreground="white") + + self.exit_button = ttk.Button(master, text="Exit & Save", command=self.safe_exit) + + self.load_animations() + self.update_animation() + + threading.Thread(target=self.safe_main_execution, daemon=True).start() + + # --- HELPERS --- + + def safe_exit(self): + if self.exiting: + return + self.exiting = True + print("\n--- SHUTDOWN SEQUENCE ---", flush=True) + if self.current_audio_process: + try: + self.current_audio_process.terminate() + self.current_audio_process.wait(timeout=1) + except: pass + + self.recording_active.clear() + self.tts_active.clear() + + self.save_chat_history() + + try: + ollama.generate(model=TEXT_MODEL, prompt="", keep_alive=0) + except: pass + try: + sd.stop() + except: pass + + try: + self.master.quit() + except Exception: + pass + + def toggle_fullscreen(self, event=None): + # Escape：在全屏 / 窗口化之间切换，程序继续运行（退出请用 Ctrl+Q 或 Exit 按钮）。 + self.is_fullscreen = not self.is_fullscreen + self.master.attributes('-fullscreen', self.is_fullscreen) + + def toggle_hud_visibility(self, event=None): + try: + if self.response_text.winfo_ismapped(): + self.response_text.place_forget() + self.status_label.place_forget() + self.exit_button.place_forget() + else: + self.response_text.place(relx=0.5, rely=0.82, anchor=tk.S) + self.status_label.place(relx=0.5, rely=1.0, anchor=tk.S, relwidth=1) + self.exit_button.place(x=10, y=10) + except tk.TclError: pass + + def handle_ptt_toggle(self, event=None): + current_time = time.time() + if current_time - self.last_ptt_time < 0.5: + return + self.last_ptt_time = current_time + + if self.recording_active.is_set(): + print("[PTT] Toggle OFF", flush=True) + self.recording_active.clear() + else: + if self.current_state == BotStates.IDLE or "Wait" in self.status_var.get(): + print("[PTT] Toggle ON", flush=True) + self.recording_active.set() + self.ptt_event.set() + + def handle_speaking_interrupt(self, event=None): + if self.current_state == BotStates.SPEAKING or self.current_state == BotStates.THINKING: + self.interrupted.set() + with self.tts_queue_lock: + self.tts_queue.clear() + with self.audio_queue_lock: + self.audio_queue.clear() + if self.current_audio_process: + try: self.current_audio_process.terminate() + except: pass + try: sd.stop() + except: pass + self.set_state(BotStates.IDLE, "Interrupted.") + + def load_animations(self): + base_path = "faces" + states = ["idle", "listening", "thinking", "speaking", "greeting", "sleep", "warmup"] + for state in states: + folder = os.path.join(base_path, state) + self.animations[state] = [] + if os.path.exists(folder): + files = sorted([f for f in os.listdir(folder) if f.lower().endswith('.png')]) + for f in files: + img = Image.open(os.path.join(folder, f)).resize((800, 450), Image.NEAREST) + self.animations[state].append(ImageTk.PhotoImage(img)) + if not self.animations[state]: + if "idle" in self.animations and self.animations["idle"]: + self.animations[state] = self.animations["idle"] + else: + blank = Image.new('RGB', (800, 450), color='#1a1a2e') + self.animations[state].append(ImageTk.PhotoImage(blank)) + + def update_animation(self): + frames = self.animations.get(self.current_state, []) or self.animations.get(BotStates.IDLE, []) + if not frames: + self.master.after(250, self.update_animation) + return + + self.current_frame_index = (self.current_frame_index + 1) % len(frames) + self.background_label.config(image=frames[self.current_frame_index]) + + self.master.after(250, self.update_animation) + + def set_state(self, state, msg=""): + def _update(): + if msg: print(f"[STATE] {state.upper()}: {msg}", flush=True) + if self.current_state != state: + self.current_state = state + self.current_frame_index = 0 + if msg: self.status_var.set(msg) + self.master.after(0, _update) + + def append_to_text(self, text, newline=True): + def _update(): + self.response_text.config(state=tk.NORMAL) + if newline: + self.response_text.insert(tk.END, text + "\n") + else: + self.response_text.insert(tk.END, text) + + self.response_text.see(tk.END) + self.response_text.config(state=tk.DISABLED) + + self.master.after(0, _update) + + def _stream_to_text(self, chunk): + def update_text_stream(): + self.response_text.config(state=tk.NORMAL) + self.response_text.insert(tk.END, chunk) + self.response_text.see(tk.END) + self.response_text.config(state=tk.DISABLED) + self.master.after(0, update_text_stream) + + # ========================================================================= + # 4. CORE LOGIC + # ========================================================================= + + def safe_main_execution(self): + try: + self.warm_up_logic() + self.synth_thread = threading.Thread(target=self._synth_worker, daemon=True) + self.synth_thread.start() + self.play_thread = threading.Thread(target=self._play_worker, daemon=True) + self.play_thread.start() + + while True: + if self.exiting: + break + # 全程免手：持续监听，VAD 自动检测说话起止（取代唤醒词/PTT 触发闸门）。 + # detect_wake_word_or_ptt() / record_voice_adaptive() / record_voice_ptt() + # 及唤醒词加载代码均保留但已停用，便于回滚对照。 + self.set_state(BotStates.LISTENING, "我在听…") + audio_file = self.record_voice_vad() + + if self.interrupted.is_set(): + self.interrupted.clear() + self.set_state(BotStates.IDLE, "Resetting...") + continue + + if not audio_file: + # 没听到，安静地继续监听（不报错停顿，符合助眠场景） + continue + + user_text = self.transcribe_audio(audio_file) + if not user_text: + self.set_state(BotStates.IDLE, "Transcription empty.") + continue + + self.append_to_text(f"YOU: {user_text}") + self.interrupted.clear() + with timed_block("完整一轮对话"): + self.chat_and_respond(user_text) + + except Exception as e: + traceback.print_exc() + self.set_state(BotStates.ERROR, f"Fatal Error: {str(e)[:40]}") + + def warm_up_logic(self): + self.set_state(BotStates.WARMUP, "Warming up brains...") + # 不只是载入权重，还要把第1轮真实会话要用的 KV 前缀（system prompt + 历史） + # 提前评估一遍，否则首轮 prompt-eval 会拖慢 LLM 首 Token（实测 ~16s）。 + # 跑一次真实 ollama.chat，丢弃输出、不写入 memory，让真实第1轮退化成"第2轮"速度。 + try: + with timed_block("LLM warmup (prefix)"): + warmup_messages = self.permanent_memory + [ + {"role": "user", "content": "你好"} + ] + ollama.chat( + model=TEXT_MODEL, + messages=warmup_messages, + stream=False, + options=OLLAMA_OPTIONS, + keep_alive=-1, + ) + except Exception as e: + print(f"Failed to load {TEXT_MODEL}: {e}", flush=True) + # 档1: 原来播放英文游戏音效 greeting_sounds，改成中文开场问候（顺带预热首次 TTS 合成）。 + # 档2 会把开场/过渡/收尾固定话术预合成为 wav 缓存，届时这里替换为直接播缓存。 + self.speak("你好，我在。今天过得怎么样？") + print("Models loaded.", flush=True) + + def detect_wake_word_or_ptt(self): + self.set_state(BotStates.IDLE, "Waiting...") + self.ptt_event.clear() + + if self.oww_model: self.oww_model.reset() + + if self.oww_model is None: + self.ptt_event.wait() + self.ptt_event.clear() + return "PTT" + + CHUNK_SIZE = 1280 + OWW_SAMPLE_RATE = 16000 + + input_rate = choose_input_samplerate(INPUT_DEVICE_NAME, CURRENT_CONFIG.get("input_sample_rate")) + use_resampling = (input_rate != OWW_SAMPLE_RATE) + input_chunk_size = int(CHUNK_SIZE * (input_rate / OWW_SAMPLE_RATE)) if use_resampling else CHUNK_SIZE + + stream_args = { + "samplerate": input_rate, + "channels": 1, + "dtype": 'int16', + "blocksize": input_chunk_size, + "device": INPUT_DEVICE_NAME + } + + # Try to find a compatible block size and sample rate + try: + # First attempt: standard settings + self._listen_loop(stream_args, input_chunk_size, CHUNK_SIZE, use_resampling) + except StopIteration as si: + return str(si) + except Exception as e: + print(f"[AUDIO] Stream failed with defaults: {e}. Retrying with loose settings...", flush=True) + try: + # Second attempt: Let PortAudio decide blocksize (0) and latency + stream_args["blocksize"] = 0 + stream_args["latency"] = "high" + # If blocksize is variable, we must read specific amounts manually or handle buffering. + # Simplest fallback: Just attempt small fixed block + stream_args["blocksize"] = 1024 + use_resampling = True + + self._listen_loop(stream_args, 1024, CHUNK_SIZE, use_resampling) + except StopIteration as si: + return str(si) + except Exception as e2: + print(f"[CRITICAL] Wake Word Stream Error: {e2}") + self.ptt_event.wait() + return "PTT" + + return "WAKE" + + def _listen_loop(self, stream_args, input_chunk_size, target_chunk_size, use_resampling): + # Force software backend (no mmap) via environment variable if possible, + # but here we can try to hint loop settings. + # However, the most effective fix for ALSA mmap issues is often just asking for 'blocksize=0' + # and letting portaudio manage the buffering, OR very small chunks. + + # Let's try to be less aggressive with reads. + + with sd.InputStream(**stream_args) as stream: + print(f"[AUDIO] Listening with rate {stream_args['samplerate']} and block {stream_args['blocksize']}", flush=True) + + # Pre-allocate buffer for speed + # If blocksize is 0, we read what is available. + + while True: + if self.ptt_event.is_set(): + self.ptt_event.clear() + raise StopIteration("PTT") + + rlist, _, _ = select.select([sys.stdin], [], [], 0.001) + if rlist: + sys.stdin.readline() + raise StopIteration("CLI") + + # If fallback mode (blocksize 0), read fixed amount + read_size = input_chunk_size + if stream_args.get('blocksize') == 0: + read_size = 1024 # Safe small read + + try: + data, overflow = stream.read(read_size) + if overflow: + print("!", end="", flush=True) + # If we overflow excessively, raise error to trigger fallback to SAFE MODE (PulseAudio/Software) + # We can use a simple counter attached to the function or object, but here raising immediately + # after a few in a row is safest. + raise RuntimeError("Audio Buffer Overflow - Triggering Safe Mode") + except Exception as e: + # Convert uncatchable PaErrorCode wrapper to standard Exception if needed + # But honestly, `raise e` should work... unless it's a SystemExit? + # Let's wrap it in a new exception to be sure it bubbles up + raise RuntimeError(f"Audio read failed: {e}") + + audio_data = np.frombuffer(data, dtype=np.int16) + + # Ensure flattening for openwakeword compatibility + if audio_data.ndim > 1: + audio_data = audio_data.flatten() + + if use_resampling: + # FAST RESAMPLING: Nearest-neighbor slicing instead of scipy.signal.resample + # This avoids the CPU bottleneck that causes overflow (!!!!!!!) on Raspberry Pi + step = len(audio_data) / target_chunk_size + indices = np.arange(0, len(audio_data), step)[:target_chunk_size].astype(int) + audio_data = audio_data[indices] + + # Convert to float for model prediction without needing heavy resampling logic + # The wake word model needs 16000, which we just faked above. + + # Debug volume occasionally + current_max = np.max(np.abs(audio_data)) + + # Only predict if volume is significant to save CPU + if current_max > 200: + prediction = self.oww_model.predict(audio_data) + for mdl in self.oww_model.prediction_buffer.keys(): + score = list(self.oww_model.prediction_buffer[mdl])[-1] + if score > 0.1: # Show potential triggers + print(f"\r[Oww] Score: {score:.3f} | Vol: {current_max} ", end="", flush=True) + + if score > WAKE_WORD_THRESHOLD: + print(f"\n[WAKE] Triggered on '{mdl}' with score: {score:.2f}", flush=True) + self.oww_model.reset() + return # Success + + + def record_voice_adaptive(self, filename="input.wav"): + print("Recording (Adaptive)...", flush=True) + time.sleep(0.5) + samplerate = choose_input_samplerate(INPUT_DEVICE_NAME, CURRENT_CONFIG.get("input_sample_rate")) + + silence_threshold = 0.006 + silence_duration = 1.5 + max_record_time = 30.0 + buffer = [] + silent_chunks = 0 + chunk_duration = 0.05 + chunk_size = int(samplerate * chunk_duration) + + num_silent_chunks = int(silence_duration / chunk_duration) + max_chunks = int(max_record_time / chunk_duration) + recorded_chunks = 0 + silence_started = False + + def callback(indata, frames, time_info, status): + nonlocal silent_chunks, recorded_chunks, silence_started + volume_norm = np.linalg.norm(indata) / np.sqrt(len(indata)) + buffer.append(indata.copy()) + recorded_chunks += 1 + if recorded_chunks < 5: return + if volume_norm < silence_threshold: + silent_chunks += 1 + if silent_chunks >= num_silent_chunks: silence_started = True + else: silent_chunks = 0 + + try: + # Explicitly close stream if it exists to free hardware + sd.stop() + time.sleep(0.2) + + with sd.InputStream(samplerate=samplerate, channels=1, callback=callback, + device=INPUT_DEVICE_NAME, blocksize=chunk_size): + while not silence_started and recorded_chunks < max_chunks: + sd.sleep(int(chunk_duration * 1000)) + except Exception as e: + print(f"[AUDIO ERROR] Adaptive Recording Failed: {e}", flush=True) + return None + + return self.save_audio_buffer(buffer, filename, samplerate) + + def record_voice_vad(self, filename="input.wav"): + """全程免手：webrtcvad 持续监听，检测到人声起始自动开始录音， + 尾部静音自动停止。阻塞直到捕获完整一句话，返回 wav 路径；没听到则返回 None。 + 助眠场景的主输入路径（取代唤醒词/PTT 触发闸门）。""" + VAD_RATE = 16000 + FRAME_MS = 30 + frame_samples = int(VAD_RATE * FRAME_MS / 1000) # 16000Hz×30ms = 480 + + aggressiveness = int(CURRENT_CONFIG.get("vad_aggressiveness", 2)) + start_frames = max(1, int(CURRENT_CONFIG.get("vad_start_ms", 150) / FRAME_MS)) + silence_frames = max(1, int(CURRENT_CONFIG.get("vad_silence_ms", 900) / FRAME_MS)) + max_frames = max(1, int(CURRENT_CONFIG.get("vad_max_record_ms", 30000) / FRAME_MS)) + preroll_frames = max(0, int(CURRENT_CONFIG.get("vad_preroll_ms", 300) / FRAME_MS)) + skip_frames = int(200 / FRAME_MS) # 丢弃头部 ~200ms，避开上一句 TTS 的房间回声尾巴 + + vad = webrtcvad.Vad(aggressiveness) + + # webrtcvad 只吃 8/16/32/48kHz。优先 16000Hz 直采；设备只能跑 44100/48000 时 + # 按原生率采集，再用最近邻重采样把每帧降到 480 个样本（复用唤醒词循环里的技巧）。 + input_rate = choose_input_samplerate(INPUT_DEVICE_NAME, VAD_RATE) + use_resampling = (input_rate != VAD_RATE) + read_size = int(input_rate * FRAME_MS / 1000) if use_resampling else frame_samples + + buffer = [] # 已确认录音的帧（int16, 16000Hz） + preroll = collections.deque(maxlen=preroll_frames) # 起始前回看缓冲 + recording = False + voiced_run = 0 + silence_run = 0 + total_frames = 0 + + try: + # 释放硬件，避免 Pi 上音频争用死锁（沿用 PTT 路径做法） + sd.stop() + time.sleep(0.2) + with sd.InputStream(samplerate=input_rate, channels=1, dtype='int16', + blocksize=read_size, device=INPUT_DEVICE_NAME) as stream: + print("[VAD] Listening...", flush=True) + while True: + if self.exiting: + return None + + data, _overflow = stream.read(read_size) + frame = np.frombuffer(data, dtype=np.int16) + if frame.ndim > 1: + frame = frame.flatten() + + if use_resampling: + step = len(frame) / frame_samples + idx = np.arange(0, len(frame), step)[:frame_samples].astype(int) + frame = frame[idx] + if len(frame) != frame_samples: # webrtcvad 要求帧长精确，长度不对就跳过 + continue + + if skip_frames > 0: + skip_frames -= 1 + continue + + is_speech = vad.is_speech(frame.tobytes(), VAD_RATE) + + if not recording: + preroll.append(frame.copy()) + if is_speech: + voiced_run += 1 + if voiced_run >= start_frames: + recording = True + buffer.extend(preroll) # 预缓冲并入开头，避免吞掉第一个字 + preroll.clear() + total_frames = len(buffer) + silence_run = 0 + print("[VAD] Speech detected, recording...", flush=True) + else: + voiced_run = 0 + else: + buffer.append(frame.copy()) + total_frames += 1 + if is_speech: + silence_run = 0 + else: + silence_run += 1 + if silence_run >= silence_frames: + print("[VAD] Trailing silence, stop.", flush=True) + break + if total_frames >= max_frames: + print("[VAD] Max record time reached, stop.", flush=True) + break + except Exception as e: + print(f"[AUDIO ERROR] VAD Recording Failed: {e}", flush=True) + return None + + if not buffer: + return None + return self.save_audio_buffer(buffer, filename, samplerate=VAD_RATE, already_int16=True) + + def record_voice_ptt(self, filename="input.wav"): + print("Recording (PTT)...", flush=True) + time.sleep(0.5) + samplerate = choose_input_samplerate(INPUT_DEVICE_NAME, CURRENT_CONFIG.get("input_sample_rate")) + + buffer = [] + def callback(indata, frames, time_info, status): buffer.append(indata.copy()) + + try: + # Explicitly close stream if it exists to free hardware + # This is critical on Pi 5 where hardware contention causes freezes + sd.stop() + time.sleep(0.2) + + with sd.InputStream(samplerate=samplerate, channels=1, callback=callback, device=INPUT_DEVICE_NAME): + while self.recording_active.is_set(): + sd.sleep(50) + except Exception as e: + print(f"[AUDIO ERROR] PTT Recording Failed: {e}", flush=True) + return None + + return self.save_audio_buffer(buffer, filename, samplerate) + + def save_audio_buffer(self, buffer, filename, samplerate=16000, already_int16=False): + if not buffer: return None + audio_data = np.concatenate(buffer, axis=0).flatten() + if already_int16: + # VAD 路径的 buffer 已是 int16 PCM，直接落盘，跳过 float×32767 换算。 + audio_data = audio_data.astype(np.int16) + else: + audio_data = np.nan_to_num(audio_data, nan=0.0, posinf=0.0, neginf=0.0) + audio_data = (audio_data * 32767).astype(np.int16) + with wave.open(filename, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(samplerate) + wf.writeframes(audio_data.tobytes()) + # 档1: 去掉录音结束后的英文确认音效（got_it.wav 等）；录音→思考的切换由 GUI 状态体现。 + return filename + + def transcribe_audio(self, filename): + print("Transcribing...", flush=True) + whisper_model = CURRENT_CONFIG.get("whisper_model", "ggml-base.en.bin") + whisper_lang = CURRENT_CONFIG.get("whisper_lang", "en") + try: + with timed_block("STT whisper-cli"): + result = subprocess.run( + ["./whisper.cpp/build/bin/whisper-cli", + "-m", f"./whisper.cpp/models/{whisper_model}", + "-l", whisper_lang, "-t", "4", "-f", filename], + capture_output=True, text=True + ) + transcription_lines = result.stdout.strip().split('\n') + if transcription_lines and transcription_lines[-1].strip(): + last_line = transcription_lines[-1].strip() + if ']' in last_line: transcription = last_line.split("]")[1].strip() + else: transcription = last_line + else: transcription = "" + print(f"Heard: '{transcription}'", flush=True) + return transcription.strip() + except Exception as e: + print(f"Transcription Error: {e}") + return "" + + # ========================================================================= + # 5. CHAT & RESPOND + # ========================================================================= + + def chat_and_respond(self, text): + # 档1: 纯聊天路径。睡前梳理场景不需要工具调用（拍照/联网搜索已删除）， + # 模型只负责"接住用户这一句"，直接流式输出 → TTS。 + if "forget everything" in text.lower() or "reset memory" in text.lower() \ + or "清空记忆" in text or "忘记一切" in text: + self.session_memory = [] + self.permanent_memory = [{"role": "system", "content": SYSTEM_PROMPT}] + self.save_chat_history() + with self.tts_queue_lock: + self.tts_queue.append("好的，我把记忆清空了。") + self.set_state(BotStates.IDLE, "Memory Wiped") + return + + self.set_state(BotStates.THINKING, "Thinking...") + + lang = CURRENT_CONFIG.get("whisper_lang", "en") + lang_hint = "请用中文回答。" if lang == "zh" else "" + user_msg = {"role": "user", "content": text + ("\n" + lang_hint if lang_hint else "")} + messages = self.permanent_memory + self.session_memory + [user_msg] + + full_response_buffer = "" + sentence_buffer = "" + + try: + stream = ollama.chat(model=TEXT_MODEL, messages=messages, stream=True, options=OLLAMA_OPTIONS) + + _t_llm = time.perf_counter() + _ttft_logged = False + + for chunk in stream: + if self.interrupted.is_set(): break + content = chunk['message']['content'] + if not _ttft_logged: + print(f"[TIMER] LLM 首Token延迟 {time.perf_counter()-_t_llm:.2f}s", flush=True) + _ttft_logged = True + full_response_buffer += content + + if self.current_state != BotStates.SPEAKING: + self.set_state(BotStates.SPEAKING, "Speaking...") + self.append_to_text("BOT: ", newline=False) + + self._stream_to_text(content) + + sentence_buffer += content + if any(punct in content for punct in ".!?\n。！？"): + clean_sentence = sentence_buffer.strip() + if clean_sentence and re.search(r'[\w一-鿿]', clean_sentence): + with self.tts_queue_lock: self.tts_queue.append(clean_sentence) + sentence_buffer = "" + + if sentence_buffer.strip() and re.search(r'[\w一-鿿]', sentence_buffer): + with self.tts_queue_lock: self.tts_queue.append(sentence_buffer.strip()) + self.append_to_text("") + self.session_memory.append({"role": "assistant", "content": full_response_buffer}) + + self.wait_for_tts() + self.set_state(BotStates.IDLE, "Ready") + + except Exception as e: + print(f"LLM Error: {e}") + self.set_state(BotStates.ERROR, "Brain Freeze!") + + def wait_for_tts(self): + # 两级都空闲才算"说完"：两个队列空，且合成/播放线程都不忙。 + while (self.tts_queue or self.audio_queue + or self.synth_active.is_set() or self.play_active.is_set()): + if self.interrupted.is_set(): break + time.sleep(0.1) + + def _synth_worker(self): + # 阶段一：从 tts_queue 取文本，提前合成成音频缓冲，推入 audio_queue。 + # 这样第 N 句播放期间第 N+1 句已在合成，句间空挡被消除。 + while True: + text = None + with self.tts_queue_lock: + if self.tts_queue: + self.synth_active.set() # 先置忙再出队，避免 wait_for_tts 抢到"空队列+未置忙" + text = self.tts_queue.pop(0) + if text is None: + time.sleep(0.05) + continue + try: + if self.interrupted.is_set(): + continue + rendered = self._render(text) # (samples, rate) 或 None + if rendered is None or self.interrupted.is_set(): + continue + # 背压：audio_queue 满则等播放线程消化，避免提前渲染堆积过多。 + while not self.interrupted.is_set(): + with self.audio_queue_lock: + if len(self.audio_queue) < self.audio_queue_max: + self.audio_queue.append(rendered) + break + time.sleep(0.02) + finally: + self.synth_active.clear() + + def _play_worker(self): + # 阶段二：从 audio_queue 取已渲染音频并播放。 + while True: + item = None + with self.audio_queue_lock: + if self.audio_queue: + self.play_active.set() # 先置忙再出队，理由同上 + item = self.audio_queue.pop(0) + if item is None: + time.sleep(0.05) + continue + try: + if not self.interrupted.is_set(): + self._play_samples(*item) + finally: + self.play_active.clear() + + def _init_sherpa_tts(self): + try: + import sherpa_onnx + model_dir = CURRENT_CONFIG.get("sherpa_model_dir", "sherpa-models/vits-zh-aishell3") + num_threads = CURRENT_CONFIG.get("sherpa_num_threads", 4) + print(f"[INIT] Sherpa num_threads (from config) = {num_threads}", flush=True) + cfg = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model=f"{model_dir}/vits-aishell3.int8.onnx", + lexicon=f"{model_dir}/lexicon.txt", + tokens=f"{model_dir}/tokens.txt", + ), + # 默认单线程合成在 Pi 上慢到 ~0.5s/字；吃满多核可砍掉一半以上耗时。 + num_threads=num_threads, + provider="cpu", + ), + rule_fsts=( + f"{model_dir}/date.fst," + f"{model_dir}/number.fst," + f"{model_dir}/phone.fst," + f"{model_dir}/new_heteronym.fst" + ), + rule_fars=f"{model_dir}/rule.far", + max_num_sentences=1, + ) + self.sherpa_tts = sherpa_onnx.OfflineTts(cfg) + print("[INIT] Sherpa TTS loaded.", flush=True) + except Exception as e: + print(f"[INIT] Sherpa TTS load failed: {e}. Falling back to piper.", flush=True) + self.sherpa_tts = None + + def speak(self, text): + # 同步合成并播放一句（阻塞）。用于开场问候等流水线 worker 启动前的场景。 + rendered = self._render(text) + if rendered is not None: + self._play_samples(*rendered) + + # --- 合成阶段：文本 → (samples float32 [-1,1], rate)，不播放 --- + + def _render(self, text): + clean = re.sub(r"[^\w\s,.!?:-，。！？、；：]", "", text) + if not clean.strip(): return None + if self.sherpa_tts is not None: + return self._render_sherpa(clean) + return self._render_piper(clean) + + def _fit_samplerate(self, samples, rate): + # 设备支持模型原生采样率就直接用；否则用多相重采样（比 FFT 法 resample 快很多）。 + try: + sd.check_output_settings(samplerate=rate) + return samples, rate + except Exception: + try: + native_rate = int(sd.query_devices(kind='output')['default_samplerate']) + except Exception: + native_rate = 48000 + resampled = scipy.signal.resample_poly(samples, native_rate, rate).astype(np.float32) + return resampled, native_rate + + def _render_sherpa(self, text): + with timed_block(f"TTS sherpa synth [{text[:15]}...]"): + print(f"[SHERPA TTS] '{text}'", flush=True) + try: + audio = self.sherpa_tts.generate( + text, + sid=CURRENT_CONFIG.get("sherpa_speaker_id", 0), + speed=CURRENT_CONFIG.get("sherpa_speed", 1.0), + ) + samples = np.array(audio.samples, dtype=np.float32) + # 归一到 [-1,1]，让偏小的模型输出以满音量播放。 + max_val = np.max(np.abs(samples)) + if max_val > 0: + samples /= max_val + return self._fit_samplerate(samples, audio.sample_rate) + except Exception as e: + print(f"[SHERPA TTS ERROR] {e}, falling back to piper") + return self._render_piper(text) + + def _render_piper(self, text): + with timed_block(f"TTS piper synth [{text[:15]}...]"): + print(f"[PIPER SPEAKING] '{text}'", flush=True) + voice_model = CURRENT_CONFIG.get("voice_model", "piper/en_GB-semaine-medium.onnx") + try: + proc = subprocess.Popen( + ["./piper/piper", "--model", voice_model, "--output-raw"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL + ) + raw, _ = proc.communicate(text.encode() + b'\n') + if not raw: return None + samples = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0 + return self._fit_samplerate(samples, 22050) + except Exception as e: + print(f"Audio Error: {e}") + return None + + # --- 播放阶段：消费已渲染的音频缓冲 --- + + def _play_samples(self, samples, rate): + with timed_block(f"TTS play [{rate}Hz {len(samples)}smp]"): + try: + sd.play(samples, rate) + while True: + if self.interrupted.is_set(): + sd.stop() + break + try: + if not sd.get_stream().active: + sd.stop() + break + except Exception: + break + time.sleep(0.05) + time.sleep(0.1) + except Exception as e: + print(f"Audio playback error: {e}") + finally: + self.current_volume = 0 + + def play_sound(self, file_path): + # 通用 wav 播放器（档2 放松音频会复用）。 + + if not file_path or not os.path.exists(file_path): return + try: + with wave.open(file_path, 'rb') as wf: + file_sr = wf.getframerate() + data = wf.readframes(wf.getnframes()) + audio = np.frombuffer(data, dtype=np.int16) + + try: + device_info = sd.query_devices(kind='output') + native_rate = int(device_info['default_samplerate']) + except: + native_rate = 48000 + + playback_rate = file_sr + try: + sd.check_output_settings(device=None, samplerate=file_sr) + except: + playback_rate = native_rate + num_samples = int(len(audio) * (native_rate / file_sr)) + audio = scipy.signal.resample(audio, num_samples).astype(np.int16) + + sd.play(audio, playback_rate) + sd.wait() + except: pass + + def load_chat_history(self): + system_msg = {"role": "system", "content": SYSTEM_PROMPT} + if os.path.exists(MEMORY_FILE): + try: + with open(MEMORY_FILE, "r") as f: + turns = json.load(f) + # memory.json 只存对话轮次，不存 system message + turns = [t for t in turns if t.get("role") != "system"] + return [system_msg] + turns + except: pass + return [system_msg] + + def save_chat_history(self): + full = self.permanent_memory + self.session_memory + # 只保存 user/assistant 轮次，system prompt 是配置不是历史 + turns = [t for t in full if t.get("role") != "system"] + if len(turns) > 10: turns = turns[-10:] + with open(MEMORY_FILE, "w") as f: + json.dump(turns, f, indent=4) + +if __name__ == "__main__": + print("--- SYSTEM STARTING ---", flush=True) + root = tk.Tk() + app = BotGUI(root) + root.mainloop() diff --git a/config.json b/config.json index 400f7b64..ebd97f7d 100644 --- a/config.json +++ b/config.json @@ -1,11 +1,37 @@ -{ - "text_model": "gemma3:1b", - "vision_model": "moondream", - "voice_model": "piper/en_GB-semaine-medium.onnx", - "chat_memory": true, - "camera_rotation": 180, - "system_prompt": "You are a helpful robot assistant running on a Raspberry Pi. You have access to the following tools. To use one, reply ONLY with the JSON format shown:\n\n1. Check Time: {\"action\": \"get_time\"}\n2. Take Photo: {\"action\": \"capture_image\"}\n3. Search Web: {\"action\": \"search_web\", \"query\": \"your search term\"}\n\nIf no tool is needed, just reply normally. Keep responses short and friendly.", - "system_prompt_extras": "", - "input_device": null, - "input_sample_rate": 44100 +{ + "text_model": "gemma3:1b", + "voice_model": "piper/zh_CN-huayan-medium.onnx", + "tts_engine": "sherpa", + "sherpa_model_dir": "sherpa-models/vits-zh-aishell3", + "sherpa_speaker_id": 0, + "sherpa_speed": 1.0, + "sherpa_num_threads": 4, + "chat_memory": true, + "system_prompt": "你是一个睡前陪伴机器人，帮用户在睡前梳理情绪。说话温和、简短，每次回应不超过两句话。只负责接住用户当下这一句，不出主意、不深挖、不在睡前帮用户解决烦心事；用户提到烦心事就先“寄存”——记下了，明天再想。", + "system_prompt_extras": "", + "input_device": null, + "input_sample_rate": 44100, + "whisper_model": "ggml-base.bin", + "whisper_lang": "zh", + "vad_aggressiveness": 3, + "vad_start_ms": 150, + "vad_silence_ms": 900, + "vad_max_record_ms": 30000, + "vad_preroll_ms": 300, + + "sleep_flow": { + "max_chat_rounds": 5, + "silence_timeout_chat": 75, + "audio_type": "white_noise", + "audio_types": ["white_noise", "light_music", "meditation"], + "shutdown_timeout": 2700, + "shutdown_enabled": false, + "cache_dir": "cache", + "pre_synthesize_texts": { + "greeting": "晚上好，今天过得怎么样？有什么想说的吗？", + "transition": "好的，已经记下了。现在让我们慢慢放松，准备休息吧。", + "soft_close": "如果你没什么想说的了，我们就开始放松吧。", + "round5_close": "我们先到这里，准备休息吧。" + } + } } \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 00000000..f68b2dda --- /dev/null +++ b/config.py @@ -0,0 +1,148 @@ +# ========================================================================= +# Be More Agent · 配置与常量 +# 从 agent.py 抽出：纯配置 / 设备解析 / 计时工具 / 状态枚举。 +# 本模块无运行时状态、无 GUI 依赖，可被 agent.py / prompts.py / flow.py 复用。 +# ========================================================================= + +import os +import json +import time +import contextlib + +import sounddevice as sd + +# ========================================================================= +# 1. CONFIGURATION & CONSTANTS +# ========================================================================= + +CONFIG_FILE = "config.json" +MEMORY_FILE = "memory.json" +WAKE_WORD_MODEL = "./wakeword.onnx" +WAKE_WORD_THRESHOLD = 0.5 + +# HARDWARE SETTINGS +INPUT_DEVICE_NAME = None + +DEFAULT_CONFIG = { + "text_model": "gemma3:1b", + "voice_model": "piper/en_GB-semaine-medium.onnx", + "chat_memory": True, + "system_prompt_extras": "", + "input_device": None, + "input_sample_rate": None, + "whisper_model": "ggml-base.en.bin", + "whisper_lang": "en", + # --- VAD（免手持续监听）--- + "vad_aggressiveness": 3, # webrtcvad 灵敏度 0~3，越大越严格（越不易把噪声当人声） + "vad_start_ms": 150, # 连续多少毫秒判定为人声才算"开始说话"（防瞬时噪声误触发） + "vad_silence_ms": 900, # 尾部静音多久判定"说完" + "vad_max_record_ms": 30000,# 单次最长录音 + "vad_preroll_ms": 300, # 起始前回看缓冲，避免吞掉第一个字 +} + +# LLM SETTINGS +OLLAMA_OPTIONS = { + 'keep_alive': '-1', + 'num_thread': 4, + 'temperature': 0.7, + 'top_k': 40, + 'top_p': 0.9 +} + + +@contextlib.contextmanager +def timed_block(label): + t0 = time.perf_counter() + print(f"[TIMER] >>> {label}", flush=True) + try: + yield + finally: + print(f"[TIMER] <<< {label} {time.perf_counter()-t0:.2f}s", flush=True) + + +def load_config(): + config = DEFAULT_CONFIG.copy() + if os.path.exists(CONFIG_FILE): + try: + with open(CONFIG_FILE, "r") as f: + user_config = json.load(f) + config.update(user_config) + except Exception as e: + print(f"Config Error: {e}. Using defaults.") + return config + +CURRENT_CONFIG = load_config() +TEXT_MODEL = CURRENT_CONFIG["text_model"] + + +def resolve_input_device(config): + requested = config.get("input_device") + if requested in (None, "", "default"): + return None + + try: + devices = sd.query_devices() + except Exception as e: + print(f"[AUDIO] Device query failed: {e}", flush=True) + return None + + if isinstance(requested, int) or (isinstance(requested, str) and requested.isdigit()): + index = int(requested) + if 0 <= index < len(devices): + return index + print(f"[AUDIO] Input device index not found: {index}", flush=True) + return None + + requested_lower = str(requested).lower() + for idx, dev in enumerate(devices): + print(f"[AUDIO DEBUG] Index {idx}: {dev.get('name')} (In: {dev.get('max_input_channels')})", flush=True) # DEBUG LINE + if dev.get("max_input_channels", 0) > 0 and requested_lower in dev.get("name", "").lower(): + return idx + + print(f"[AUDIO] Input device name not found: {requested}", flush=True) + return None + +INPUT_DEVICE_NAME = resolve_input_device(CURRENT_CONFIG) +if INPUT_DEVICE_NAME is not None: + try: + device_info = sd.query_devices(INPUT_DEVICE_NAME) + print(f"[AUDIO] Using input device: {device_info.get('name', INPUT_DEVICE_NAME)}", flush=True) + except Exception: + print(f"[AUDIO] Using input device index: {INPUT_DEVICE_NAME}", flush=True) + +def choose_input_samplerate(device, preferred=None): + candidates = [] + if preferred: + candidates.append(preferred) + try: + device_info = sd.query_devices(device) + print(f"[AUDIO DEBUG] Device Info: {device_info}", flush=True) # DEBUG + if "default_samplerate" in device_info: + candidates.append(int(device_info["default_samplerate"])) + except Exception as e: + print(f"[AUDIO DEBUG] Query failed: {e}", flush=True) + pass + + candidates.extend([48000, 44100, 32000, 16000]) + seen = set() + for rate in candidates: + if not rate or rate in seen: + continue + seen.add(rate) + try: + sd.check_input_settings(device=device, samplerate=rate, channels=1, dtype="int16") + return rate + except Exception: + continue + + return int(candidates[0]) if candidates else 44100 + + +class BotStates: + IDLE = "idle" + LISTENING = "listening" + THINKING = "thinking" + SPEAKING = "speaking" + WARMUP = "warmup" + GREETING = "greeting" + SLEEP = "sleep" diff --git a/docs/PROMPT_GUIDE.md b/docs/PROMPT_GUIDE.md new file mode 100644 index 00000000..64870a1b --- /dev/null +++ b/docs/PROMPT_GUIDE.md @@ -0,0 +1,26 @@ +# 二、prompts.py — 系统提示词 + +只放提示词文本，依赖 `config.CURRENT_CONFIG`。档2 的“每状态窄 prompt + few-shot”都加在这里，让负责调 prompt 的人独占此文件、不与改 `agent.py` 的人冲突。 + +| 名称 | 含义 | +|------|------| +| `BASE_SYSTEM_PROMPT` | 内置兜底人设：睡前情绪梳理、温和简短、只接住不出主意，并包含安全边界 | +| `CHAT_STAGE_PROMPT` | CHAT 阶段窄 prompt：只回应用户这一句，先接住情绪，再寄存到明天 | +| `CHAT_FEW_SHOTS` | few-shot 示例：工作压力、人际冲突、后悔自责、索要方案、自然收尾 | +| `NEAR_END_PROMPT` | 倒数第二轮提示：减少追问，开始自然收束 | +| `FINAL_ROUND_PROMPT` | 最后一轮提示：必须收尾，引导放松休息 | +| `SYSTEM_PROMPT` | **实际生效的系统提示** = `config.json` 的 `system_prompt`（无则用兜底）+ `system_prompt_extras` | + +用法： + +```python +from prompts import SYSTEM_PROMPT +messages = [{"role": "system", "content": SYSTEM_PROMPT}, ...] +``` + +睡前状态机中应优先使用带轮次信息的版本： + +```python +from prompts import get_chat_prompt +chat_system = get_chat_prompt(round_num, max_rounds) +``` diff --git a/faces/capturing/capturing 01.png b/faces/capturing/capturing 01.png deleted file mode 100644 index e8f59137..00000000 Binary files a/faces/capturing/capturing 01.png and /dev/null differ diff --git a/faces/error/error 01.png b/faces/error/error 01.png deleted file mode 100644 index 3ac1108c..00000000 Binary files a/faces/error/error 01.png and /dev/null differ diff --git a/faces/greeting/greeting_00.png b/faces/greeting/greeting_00.png new file mode 100644 index 00000000..dd0a5251 Binary files /dev/null and b/faces/greeting/greeting_00.png differ diff --git a/faces/greeting/greeting_01.png b/faces/greeting/greeting_01.png new file mode 100644 index 00000000..65a3da62 Binary files /dev/null and b/faces/greeting/greeting_01.png differ diff --git a/faces/greeting/greeting_02.png b/faces/greeting/greeting_02.png new file mode 100644 index 00000000..c4392b90 Binary files /dev/null and b/faces/greeting/greeting_02.png differ diff --git a/faces/greeting/greeting_03.png b/faces/greeting/greeting_03.png new file mode 100644 index 00000000..94d73f26 Binary files /dev/null and b/faces/greeting/greeting_03.png differ diff --git a/faces/greeting/greeting_04.png b/faces/greeting/greeting_04.png new file mode 100644 index 00000000..9753f5db Binary files /dev/null and b/faces/greeting/greeting_04.png differ diff --git a/faces/greeting/greeting_05.png b/faces/greeting/greeting_05.png new file mode 100644 index 00000000..94d73f26 Binary files /dev/null and b/faces/greeting/greeting_05.png differ diff --git a/faces/greeting/greeting_06.png b/faces/greeting/greeting_06.png new file mode 100644 index 00000000..65a3da62 Binary files /dev/null and b/faces/greeting/greeting_06.png differ diff --git a/faces/idle/idle 01.png b/faces/idle/idle 01.png deleted file mode 100644 index 3a2e42ae..00000000 Binary files a/faces/idle/idle 01.png and /dev/null differ diff --git a/faces/idle/idle_00.png b/faces/idle/idle_00.png new file mode 100644 index 00000000..b4ad4f60 Binary files /dev/null and b/faces/idle/idle_00.png differ diff --git a/faces/idle/idle_01.png b/faces/idle/idle_01.png new file mode 100644 index 00000000..9f055d25 Binary files /dev/null and b/faces/idle/idle_01.png differ diff --git a/faces/idle/idle_02.png b/faces/idle/idle_02.png new file mode 100644 index 00000000..e74be6c1 Binary files /dev/null and b/faces/idle/idle_02.png differ diff --git a/faces/idle/idle_03.png b/faces/idle/idle_03.png new file mode 100644 index 00000000..6e51a98c Binary files /dev/null and b/faces/idle/idle_03.png differ diff --git a/faces/idle/idle_04.png b/faces/idle/idle_04.png new file mode 100644 index 00000000..b4ad4f60 Binary files /dev/null and b/faces/idle/idle_04.png differ diff --git a/faces/listening/listen 01.png b/faces/listening/listen 01.png deleted file mode 100644 index e8609e75..00000000 Binary files a/faces/listening/listen 01.png and /dev/null differ diff --git a/faces/listening/listen 02.png b/faces/listening/listen 02.png deleted file mode 100644 index be73c4e8..00000000 Binary files a/faces/listening/listen 02.png and /dev/null differ diff --git a/faces/listening/listening_00.png b/faces/listening/listening_00.png new file mode 100644 index 00000000..fe040a5e Binary files /dev/null and b/faces/listening/listening_00.png differ diff --git a/faces/listening/listening_01.png b/faces/listening/listening_01.png new file mode 100644 index 00000000..c4af27a9 Binary files /dev/null and b/faces/listening/listening_01.png differ diff --git a/faces/listening/listening_02.png b/faces/listening/listening_02.png new file mode 100644 index 00000000..6cda1856 Binary files /dev/null and b/faces/listening/listening_02.png differ diff --git a/faces/listening/listening_03.png b/faces/listening/listening_03.png new file mode 100644 index 00000000..1a2649f3 Binary files /dev/null and b/faces/listening/listening_03.png differ diff --git a/faces/listening/listening_04.png b/faces/listening/listening_04.png new file mode 100644 index 00000000..26044615 Binary files /dev/null and b/faces/listening/listening_04.png differ diff --git a/faces/pi-faces-Sheet.png b/faces/pi-faces-Sheet.png new file mode 100644 index 00000000..c5e5a5e6 Binary files /dev/null and b/faces/pi-faces-Sheet.png differ diff --git a/faces/sleep/sleep_00.png b/faces/sleep/sleep_00.png new file mode 100644 index 00000000..b914008f Binary files /dev/null and b/faces/sleep/sleep_00.png differ diff --git a/faces/sleep/sleep_01.png b/faces/sleep/sleep_01.png new file mode 100644 index 00000000..8ed46264 Binary files /dev/null and b/faces/sleep/sleep_01.png differ diff --git a/faces/sleep/sleep_02.png b/faces/sleep/sleep_02.png new file mode 100644 index 00000000..b46ff076 Binary files /dev/null and b/faces/sleep/sleep_02.png differ diff --git a/faces/sleep/sleep_03.png b/faces/sleep/sleep_03.png new file mode 100644 index 00000000..6d38b28c Binary files /dev/null and b/faces/sleep/sleep_03.png differ diff --git a/faces/sleep/sleep_04.png b/faces/sleep/sleep_04.png new file mode 100644 index 00000000..7743e0dc Binary files /dev/null and b/faces/sleep/sleep_04.png differ diff --git a/faces/speaking/speaking 01.png b/faces/speaking/speaking 01.png deleted file mode 100644 index 81901aec..00000000 Binary files a/faces/speaking/speaking 01.png and /dev/null differ diff --git a/faces/speaking/speaking 02.png b/faces/speaking/speaking 02.png deleted file mode 100644 index dfd6f985..00000000 Binary files a/faces/speaking/speaking 02.png and /dev/null differ diff --git a/faces/speaking/speaking 03.png b/faces/speaking/speaking 03.png deleted file mode 100644 index bd7fb99a..00000000 Binary files a/faces/speaking/speaking 03.png and /dev/null differ diff --git a/faces/speaking/speaking_00.png b/faces/speaking/speaking_00.png new file mode 100644 index 00000000..dd0a5251 Binary files /dev/null and b/faces/speaking/speaking_00.png differ diff --git a/faces/speaking/speaking_01.png b/faces/speaking/speaking_01.png new file mode 100644 index 00000000..c2a20ed0 Binary files /dev/null and b/faces/speaking/speaking_01.png differ diff --git a/faces/speaking/speaking_02.png b/faces/speaking/speaking_02.png new file mode 100644 index 00000000..60eb3c8e Binary files /dev/null and b/faces/speaking/speaking_02.png differ diff --git a/faces/speaking/speaking_03.png b/faces/speaking/speaking_03.png new file mode 100644 index 00000000..3ce08c32 Binary files /dev/null and b/faces/speaking/speaking_03.png differ diff --git a/faces/speaking/speaking_04.png b/faces/speaking/speaking_04.png new file mode 100644 index 00000000..dd0a5251 Binary files /dev/null and b/faces/speaking/speaking_04.png differ diff --git a/faces/thinking/thinking 01.png b/faces/thinking/thinking 01.png deleted file mode 100644 index 9d9ceb5d..00000000 Binary files a/faces/thinking/thinking 01.png and /dev/null differ diff --git a/faces/thinking/thinking 02.png b/faces/thinking/thinking 02.png deleted file mode 100644 index 9056ebf9..00000000 Binary files a/faces/thinking/thinking 02.png and /dev/null differ diff --git a/faces/thinking/thinking 03.png b/faces/thinking/thinking 03.png deleted file mode 100644 index f6ae9c7f..00000000 Binary files a/faces/thinking/thinking 03.png and /dev/null differ diff --git a/faces/thinking/thinking 04.png b/faces/thinking/thinking 04.png deleted file mode 100644 index 70663901..00000000 Binary files a/faces/thinking/thinking 04.png and /dev/null differ diff --git a/faces/thinking/thinking_00.png b/faces/thinking/thinking_00.png new file mode 100644 index 00000000..255f170d Binary files /dev/null and b/faces/thinking/thinking_00.png differ diff --git a/faces/thinking/thinking_01.png b/faces/thinking/thinking_01.png new file mode 100644 index 00000000..8d1df6a2 Binary files /dev/null and b/faces/thinking/thinking_01.png differ diff --git a/faces/thinking/thinking_02.png b/faces/thinking/thinking_02.png new file mode 100644 index 00000000..c80455dc Binary files /dev/null and b/faces/thinking/thinking_02.png differ diff --git a/faces/thinking/thinking_03.png b/faces/thinking/thinking_03.png new file mode 100644 index 00000000..4d729be2 Binary files /dev/null and b/faces/thinking/thinking_03.png differ diff --git a/faces/thinking/thinking_04.png b/faces/thinking/thinking_04.png new file mode 100644 index 00000000..ff8c0672 Binary files /dev/null and b/faces/thinking/thinking_04.png differ diff --git a/faces/thinking/thinking_05.png b/faces/thinking/thinking_05.png new file mode 100644 index 00000000..d05f075d Binary files /dev/null and b/faces/thinking/thinking_05.png differ diff --git a/faces/thinking/thinking_06.png b/faces/thinking/thinking_06.png new file mode 100644 index 00000000..524edb34 Binary files /dev/null and b/faces/thinking/thinking_06.png differ diff --git a/faces/warmup/warmup 01.png b/faces/warmup/warmup 01.png deleted file mode 100644 index a9be5420..00000000 Binary files a/faces/warmup/warmup 01.png and /dev/null differ diff --git a/faces/warmup/warmup_00.png b/faces/warmup/warmup_00.png new file mode 100644 index 00000000..8daaa019 Binary files /dev/null and b/faces/warmup/warmup_00.png differ diff --git a/faces/warmup/warmup_01.png b/faces/warmup/warmup_01.png new file mode 100644 index 00000000..3abe9f8a Binary files /dev/null and b/faces/warmup/warmup_01.png differ diff --git a/faces/warmup/warmup_02.png b/faces/warmup/warmup_02.png new file mode 100644 index 00000000..9bfe0a25 Binary files /dev/null and b/faces/warmup/warmup_02.png differ diff --git a/faces/warmup/warmup_03.png b/faces/warmup/warmup_03.png new file mode 100644 index 00000000..f4f5db76 Binary files /dev/null and b/faces/warmup/warmup_03.png differ diff --git a/faces/warmup/warmup_04.png b/faces/warmup/warmup_04.png new file mode 100644 index 00000000..f34bc1c4 Binary files /dev/null and b/faces/warmup/warmup_04.png differ diff --git a/faces/warmup/warmup_05.png b/faces/warmup/warmup_05.png new file mode 100644 index 00000000..34848c8a Binary files /dev/null and b/faces/warmup/warmup_05.png differ diff --git a/faces/warmup/warmup_06.png b/faces/warmup/warmup_06.png new file mode 100644 index 00000000..f1140e63 Binary files /dev/null and b/faces/warmup/warmup_06.png differ diff --git a/faces/warmup/warmup_07.png b/faces/warmup/warmup_07.png new file mode 100644 index 00000000..9b1c978d Binary files /dev/null and b/faces/warmup/warmup_07.png differ diff --git a/faces/warmup/warmup_08.png b/faces/warmup/warmup_08.png new file mode 100644 index 00000000..c5a53cae Binary files /dev/null and b/faces/warmup/warmup_08.png differ diff --git a/faces/warmup/warmup_09.png b/faces/warmup/warmup_09.png new file mode 100644 index 00000000..aba1935a Binary files /dev/null and b/faces/warmup/warmup_09.png differ diff --git a/faces/warmup/warmup_10.png b/faces/warmup/warmup_10.png new file mode 100644 index 00000000..d43a109d Binary files /dev/null and b/faces/warmup/warmup_10.png differ diff --git a/faces/warmup/warmup_11.png b/faces/warmup/warmup_11.png new file mode 100644 index 00000000..4bf1123f Binary files /dev/null and b/faces/warmup/warmup_11.png differ diff --git a/faces/warmup/warmup_12.png b/faces/warmup/warmup_12.png new file mode 100644 index 00000000..8daaa019 Binary files /dev/null and b/faces/warmup/warmup_12.png differ diff --git a/flow.md b/flow.md new file mode 100644 index 00000000..0fd31c1e --- /dev/null +++ b/flow.md @@ -0,0 +1,369 @@ +# flow.py · 睡前情绪梳理状态机 + +## 一、设计目标 + +将原来的"开放式问答"主循环收窄为一条**单向状态机**，引导用户完成从开机到入睡的完整流程。全程不需要唤醒词、不需要按键交互，是"插上电源就开始"的线性体验。 + +## 二、状态流转图 + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ │ +│ ┌──────┐ ┌──────┐ ┌──────────┐ ┌──────┐ ┌───────┐ ┌──────────┐ +│ │ BOOT │──▶│ CHAT │──▶│TRANSITION│──▶│AUDIO │──▶│ SLEEP │──▶│ SHUTDOWN │ +│ └──────┘ └──────┘ └──────────┘ └──────┘ └───────┘ └──────────┘ +│ │ │ │ +│ │ 开机 │ 最多5轮，静默超 │ +│ │ 打招呼 │ 时75s或第5轮后 │ +│ │ │ 自动推进 │ +│ └──────────┘ │ +│ │ +│ BOOT ─── 播放预合成开场白 │ +│ CHAT ─── VAD 免手录音 → STT → LLM → TTS（每轮计数） │ +│ TRANSITION ─ 过渡话术 → 准备放松 │ +│ AUDIO ─── 循环播放白噪音/轻音乐/冥想音频，最长45分钟 │ +│ SLEEP ─── 短暂停留5秒，确认入眠 │ +│ SHUTDOWN ─ 开发期打日志 / 生产期执行 sudo halt │ +│ │ +│ ⚠ 单向不可逆：不支持回退，用户说"再聊会"也不返回 CHAT │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +### 特殊路径 + +``` +CHAT 内部细节： + + ┌─ start ─────────────────────────────────────────────┐ + │ │ + │ record_vad_with_timeout(timeout=75s) │ + │ │ │ + │ ├── [超时无人说话] → 播放 soft_close → TRANSITION + │ │ │ + │ └── [检测到语音] │ + │ │ │ + │ ▼ │ + │ STT 转写 │ + │ │ │ + │ ┌────┴────┐ │ + │ │ 失败 │ │ + │ │ 重试1次 │ │ + │ └────┬────┘ │ + │ ┌────┴────┐ │ + │ │ 仍失败 │──→ 提示"我没听清" → 回到监听 │ + │ └─────────┘ （不占轮次） │ + │ │ │ + │ ▼ │ + │ LLM 回复（带轮次 system prompt） │ + │ │ │ + │ ▼ │ + │ TTS 播放回复 │ + │ │ │ + │ chat_round += 1 │ + │ │ │ + │ ┌────┴────┐ │ + │ │ <5轮 │──→ 回到录音继续监听 │ + │ │ =第5轮 │──→ 播放 round5_close → TRANSITION│ + │ └─────────┘ │ + └─────────────────────────────────────────────────────┘ +``` + +## 三、SleepFlow 类 + +### `SleepFlow.__init__(config, gui=None)` + +初始化状态机。接收配置字典和可选的 BotGUI 实例。 + +| 参数 | 类型 | 说明 | +|------|------|------| +| `config` | `dict` | 配置字典（建议传入 `CURRENT_CONFIG`） | +| `gui` | `BotGUI` 或 `None` | BotGUI 实例。提供 `speak()`、`transcribe_audio()`、`set_state()`、`play_sound()`、`_render()` 等方法。为 `None` 时进入纯日志模式（无音频 I/O） | + +初始化时自动完成： +1. 读取 `sleep_flow` 配置段 +2. 创建缓存目录 (`cache/`) +3. 逐条预合成固定话术（开场白、过渡语、收尾句）并落盘 + +### `SleepFlow.start()` + +**阻塞**运行状态机主循环，直到进入 `SHUTDOWN` 状态。主循环结构： + +```python +while self.state != SleepState.SHUTDOWN: + if self.state == BOOT: _run_boot() + elif self.state == CHAT: _run_chat() + elif self.state == TRANSITION: _run_transition() + elif self.state == AUDIO: _run_audio() + elif self.state == SLEEP: _run_sleep() +``` + +任何状态内抛出未捕获异常 → `_force_next()` 强制推进到下一状态，**不会**导致整个进程崩溃。 + +### `SleepFlow.stop()` + +线程安全的停止方法。设置 `_exit_flag`，状态机在下一个检测点退出。 + +### `SleepFlow._transition_to(new_state)` + +状态转移统一入口：打印 `[FLOW] prev -> next` 日志，修改 `self.state`。未来可在此处加入状态变更回调/埋点。 + +### `SleepFlow._force_next()` + +异常紧急推进。按 `SleepState` 枚举顺序跳到下一个状态；已在最后一个状态则跳到 `SHUTDOWN`。 + +## 四、各状态详解 + +### 4.1 BOOT — 开机问候 + +- 调用 `gui.set_state("greeting", "晚上好")` 更新表情动画 +- 播放预合成开场白（key: `greeting`） +- 自动转入 `CHAT`，`chat_round` 置 0 + +### 4.2 CHAT — 倾听与对话 + +核心对话状态，控制逻辑如下： + +| 配置项 | 默认值 | 作用 | +|--------|--------|------| +| `max_chat_rounds` | 5 | 最大对话轮次，第 5 轮强制过渡 | +| `silence_timeout_chat` | 75 (秒) | 用户安静超过此秒数 → 自动软收尾 | + +**每轮流程**： + +1. **录音**：调用 `record_vad_with_timeout(timeout=silence_timeout)`（详见第五节） +2. **超时分支**：返回 `None` → 播放 `soft_close` → 跳 `TRANSITION` +3. **STT**：通过 `gui.transcribe_audio()` 转写 + - 失败：重试 1 次，仍失败则 TTS 提示"我没听清"并继续监听（不占轮次） +4. **LLM**：用 `get_chat_prompt(round_num, max_rounds)` 构造 system prompt，调用 `ollama.chat` + - 超时/失败：使用随机 fallback 回复 +5. **TTS**：通过 `gui.speak(reply)` 播放回复 +6. **计数**：`chat_round += 1`，达 `max_rounds` 时播放 `round5_close` 并跳 `TRANSITION` + +### 4.3 TRANSITION — 过渡 + +- 播放预合成过渡话术（key: `transition`） +- 直接转入 `AUDIO` + +### 4.4 AUDIO — 放松音频播放 + +| 配置项 | 默认值 | 作用 | +|--------|--------|------| +| `audio_type` | `"white_noise"` | 音频类型，可选 `white_noise` / `light_music` / `meditation` | +| `shutdown_timeout` | 2700 (45分钟) | 音频最长播放时长 | + +播放逻辑： +1. 检查 `sounds/relax/.wav` 是否存在 +2. 存在 → 循环播放该文件直到超时 +3. 不存在 → 用 `scipy` 生成白噪音音频并实时播放 +4. 超时后自动转入 `SLEEP` + +### 4.5 SLEEP — 确认入眠 + +- 静态等待 5 秒 +- 转入 `SHUTDOWN` + +### 4.6 SHUTDOWN — 关机 + +| 配置项 | 默认值 | 作用 | +|--------|--------|------| +| `shutdown_enabled` | `false` | `true` = 执行 `sudo halt`；`false` = 仅打印日志 | + +无论开关闭，最后都会调用 `gui.safe_exit()` 安全退出 GUI。 + +## 五、record_vad_with_timeout() — 带超时的 VAD 录音 + +独立于 `BotGUI.record_voice_vad()` 的录音函数，**关键区别是新增 `timeout` 和 `exit_flag` 参数**。 + +```python +record_vad_with_timeout( + timeout=75, # 等待人声起始的最大秒数；None = 一直等 + config=CURRENT_CONFIG, + exit_flag=None, # threading.Event，置位时立即退出 +) -> str | None # wav 路径，或 None（超时/退出/错误） +``` + +**状态机**： + +``` +WAITING ──→ 连续人声达 vad_start_ms ──→ RECORDING + │ │ + │ 超时 / exit_flag_set │ 尾部静音达 vad_silence_ms + │ │ 或达 vad_max_record_ms + ▼ ▼ +None wav 文件路径 +``` + +**配置项**（复用 `config.json` 顶层 VAD 参数）： + +| 键 | 说明 | +|----|------| +| `vad_aggressiveness` | webrtcvad 灵敏度 0~3 | +| `vad_start_ms` | 判定"开始说话"的连续人声时长 | +| `vad_silence_ms` | 判定"说完"的尾部静音时长 | +| `vad_max_record_ms` | 单次最长录音 | +| `vad_preroll_ms` | 起始前回看缓冲，防吞字 | + +## 六、预合成缓存机制 + +### 目的 + +- 开机白（BOOT）、过渡句（TRANSITION）、软收尾句等固定话术提前合成为 WAV 文件 +- 运行时直接播放文件，避免每次开机都重新合成（节省树莓派 CPU） + +### 流程 + +``` +__init__ 时: + for each key in pre_synthesize_texts: + cache/.wav 是否存在? + ├── 是 → 跳过（命中缓存） + └── 否 → 调用 gui._render(text) 合成 → _save_float32_wav() 落盘 + 如果 _render 不可用或失败 → 跳过缓存，运行时实时 TTS + +运行时: + _play_cached(key) 时: + cache/.wav 是否存在? + ├── 是 → gui.play_sound(path) 直接播放 + └── 否 → gui.speak(fallback_text) 实时合成 +``` + +### 缓存目录 + +- 默认 `cache/`（由 `sleep_flow.cache_dir` 控制） +- 已在 `.gitignore` 中忽略 +- 文件名：`{key}.wav`，key 对应 `pre_synthesize_texts` 字典的键 + +### 更新缓存 + +删除 `cache/` 目录下对应 WAV 文件后重启程序即可重新生成。 + +## 七、与现有模块的集成 + +### 依赖关系图 + +``` +flow.py + ├── 依赖: config.py (CURRENT_CONFIG, OLLAMA_OPTIONS, TEXT_MODEL, VAD函数) + ├── 依赖: prompts.py (SYSTEM_PROMPT, get_chat_prompt) + ├── 可选: BotGUI (gui 参数, 提供 TTS/STT/GUI 方法) + ├── 直接: ollama (LLM 调用) + ├── 直接: sounddevice / webrtcvad / numpy (录音与音频播放) + └── 不依赖: agent.py（不修改 agent.py 任何代码） +``` + +### prompts.py 新增内容 + +```python +from prompts import ( + SYSTEM_PROMPT, # 原有的系统提示词 + get_chat_prompt(round, max), # 带轮次信息的 chat prompt + get_transition_prompt(), # 过渡 prompt（预留） + get_boot_prompt(), # 开机 prompt（预留） +) +``` + +`get_chat_prompt()` 内部根据轮次自动追加收尾引导： + +| 轮次 | 追加内容 | +|------|----------| +| 前几轮 | 仅标注 `(当前第 X/Y 轮)` | +| 倒数第 2 轮 + `对话接近尾声，可以开始引导用户放松了` | +| 最后一轮 | `这是本轮最后一次交流，请自然收尾，告诉用户准备放松休息` | + +### config.json 新增配置段 + +```json +"sleep_flow": { + "max_chat_rounds": 5, // CHAT 状态最大对话轮次 + "silence_timeout_chat": 75, // CHAT 安静超时秒数 + "audio_type": "white_noise", // 放松音频类型 + "audio_types": ["white_noise", "light_music", "meditation"], // 可选类型列表（用于切换） + "shutdown_timeout": 2700, // 音频播放时长（秒），2700 = 45分钟 + "shutdown_enabled": false, // 是否真关机（开发期设为 false） + "cache_dir": "cache", // 预合成缓存目录 + "pre_synthesize_texts": { // 固定话术列表 + "greeting": "晚上好，今天过得怎么样？有什么想说的吗？", + "transition": "好的，已经记下了。现在让我们慢慢放松，准备休息吧。", + "soft_close": "如果你没什么想说的了，我们就开始放松吧。", + "round5_close": "我们先到这里，准备休息吧。" + } +} +``` + +### main.py 使用示例 + +将原来的开放式问答循环替换为 SleepFlow，`agent.py` 不动： + +```python +# main.py — 睡前情绪梳理机器人入口 +from config import CURRENT_CONFIG +from agent import BotGUI + +def main(): + root = tk.Tk() + app = BotGUI(root) + + # 创建状态机并运行（接管主逻辑线程） + from flow import SleepFlow + flow = SleepFlow(config=CURRENT_CONFIG, gui=app) + + # 启动状态机（在单独线程中运行，避免阻塞 Tk 主循环） + import threading + threading.Thread(target=flow.start, daemon=True).start() + + root.mainloop() + +if __name__ == "__main__": + main() +``` + +要点： +- `SleepFlow` 接管 `BotGUI` 的 TTS/STT/状态显示，不再需要 `safe_main_execution` +- `BotGUI` 的 `speak()`、`transcribe_audio()`、`set_state()` 等方法被 `SleepFlow` 调用 +- `root.mainloop()` 仍在主线程，GUI 保持响应 + +## 八、错误处理策略 + +| 故障点 | 处理方式 | +|--------|----------| +| **STT 首次失败** | 静默重试 1 次 | +| **STT 二次失败** | TTS 提示"我没听清，可以再说一遍吗"，继续监听（不占轮次） | +| **TTS 合成失败** | 跳过缓存，运行时实时合成；实时合成也失败时仅打印错误 | +| **LLM 超时/报错** | 使用随机 fallback 回复（`["嗯，我在听。", "好的，我知道了。", ...]`） | +| **音频播放异常** | 用 `time.sleep` 等待剩余时间，不崩溃 | +| **状态内未捕获异常** | `_force_next()` 强制推进到下一状态，打印完整 traceback | +| **录音设备不可用** | `record_vad_with_timeout` 返回 `None`，走超时分支 | +| **缓存文件损坏** | 删除后自动重新生成 | + +## 九、开发期与生产期行为 + +| 行为 | 开发期 (`shutdown_enabled=false`) | 生产期 (`shutdown_enabled=true`) | +|------|----------------------------------|----------------------------------| +| SHUTDOWN 动作 | 打印 `[SHUTDOWN]` 日志 | 执行 `sudo halt` | +| 录音 | 实际录音（可听） | 同左 | +| TTS | 实际播放 | 同左 | +| 放松音频 | 实际播放 | 同左 | +| 缓存 | 写入 `cache/` | 同左 | +| 开机 | 流程完整运行 | 同左 | + +## 十、关键变量速查 + +| 变量 | 类型 | 位置 | 说明 | +|------|------|------|------| +| `self.state` | `SleepState` | `flow.py` | 当前状态枚举 | +| `self.chat_round` | `int` | `flow.py` | CHAT 阶段已完成的轮次（0~4） | +| `self._exit_flag` | `bool` | `flow.py` | 外部停止请求标志（`stop()` 方法设置） | +| `self.cache_dir` | `str` | `flow.py` | 预合成缓存目录路径 | +| `self.sleep_cfg` | `dict` | `flow.py` | `config["sleep_flow"]` 的快捷引用 | +| `self.gui` | `BotGUI` | `flow.py` | 可选的 GUI 实例引用 | + +## 十一、文件清单（档2 新增/修改） + +| 文件 | 操作 | 说明 | +|------|------|------| +| `flow.py` | **新增** | 状态机编排，~420 行 | +| `flow.md` | **新增** | 本文档 | +| `prompts.py` | 修改 | 新增 `get_chat_prompt()` 等 3 个函数 | +| `config.json` | 修改 | 新增 `sleep_flow` 配置段 | +| `.gitignore` | 修改 | 新增 `cache/` 忽略规则 | +| `main.py` | 参考 | 替换为主循环调用 `SleepFlow` | diff --git a/flow.py b/flow.py new file mode 100644 index 00000000..61212a25 --- /dev/null +++ b/flow.py @@ -0,0 +1,802 @@ +# ========================================================================= +# Be More Agent · 睡前情绪梳理状态机 +# 档2: 单向状态机编排，从 BOOT 到 SHUTDOWN 共 6 个状态。 +# 依赖 config / prompts，与 BotGUI 实例协作完成语音交互。 +# ========================================================================= + +import os +import time +import wave +import random +import traceback +import collections +from enum import Enum + +import numpy as np + +# 硬件相关库（在无音频硬件的环境缺失时可正常导入 flow.py） +try: + import sounddevice as sd + _HAS_SOUNDDEVICE = True +except ImportError: + sd = None + _HAS_SOUNDDEVICE = False + +try: + import webrtcvad + _HAS_WEBRTCVAD = True +except ImportError: + webrtcvad = None + _HAS_WEBRTCVAD = False + +try: + import ollama + _HAS_OLLAMA = True +except ImportError: + ollama = None + _HAS_OLLAMA = False + +try: + from config import ( + CURRENT_CONFIG, OLLAMA_OPTIONS, TEXT_MODEL, + choose_input_samplerate, INPUT_DEVICE_NAME, timed_block, + ) + from prompts import SYSTEM_PROMPT, get_chat_prompt + _HAS_CONFIG = True +except (ImportError, ModuleNotFoundError) as e: + # 在无硬件依赖的开发环境中，config.py 可能因 import sounddevice 失败 + print(f"[FLOW] config.py import 失败: {e}", flush=True) + print("[FLOW] 使用内置默认配置（纯日志模式）", flush=True) + + # 提供最小化 fallback 常量 + CURRENT_CONFIG = {} + OLLAMA_OPTIONS = {} + TEXT_MODEL = "" + INPUT_DEVICE_NAME = None + + def choose_input_samplerate(device, preferred=None): + return 16000 + + def timed_block(label): + import contextlib + @contextlib.contextmanager + def _inner(): + yield + return _inner() + + SYSTEM_PROMPT = "你是睡前陪伴机器人，帮助用户在睡前梳理情绪。说话温和、简短。" + get_chat_prompt = None # type: ignore + + _HAS_CONFIG = False + + +# ========================================================================= +# 1. STATE ENUM +# ========================================================================= + +class SleepState(Enum): + """睡前情绪梳理机器人状态枚举，严格单向转移，不可逆""" + BOOT = "boot" + CHAT = "chat" + TRANSITION = "transition" + AUDIO = "audio" + SLEEP = "sleep" + SHUTDOWN = "shutdown" + + +# ========================================================================= +# 2. VAD 录音（带超时） +# 与 BotGUI.record_voice_vad 同源，额外支持 idle-timeout。 +# ========================================================================= + +def check_hardware(): + """检查硬件/软件依赖是否可用，缺失时打印警告""" + if not _HAS_SOUNDDEVICE: + print("[HARDWARE] sounddevice 未安装，录音/播放功能不可用", flush=True) + if not _HAS_WEBRTCVAD: + print("[HARDWARE] webrtcvad 未安装，VAD 录音功能不可用", flush=True) + if not _HAS_OLLAMA: + print("[HARDWARE] ollama 未安装，LLM 对话功能不可用", flush=True) + + +def record_vad_with_timeout(timeout=None, config=None, exit_flag=None): + """ + 基于 webrtcvad 的免手录音，支持"安静等待"超时。 + + 行为： + - 在检测到人声起始前持续监听；超过 timeout 秒无人说话则返回 None + - 检测到人声后自动录音，直到尾部静音或达 max_record_ms，返回 wav 路径 + - timeout=None 时不会因安静而超时（即原版行为） + + Args: + timeout: 等待人声起始的超时秒数（对录音阶段不生效） + config: 配置字典（默认 CURRENT_CONFIG） + exit_flag: 可选 threading.Event，置位时提前退出返回 None + + Returns: + str: 录音文件路径，或 None（超时 / 错误 / 退出标志） + """ + if not _HAS_SOUNDDEVICE or not _HAS_WEBRTCVAD: + print("[VAD] 硬件依赖缺失，无法录音", flush=True) + return None + + if config is None: + config = CURRENT_CONFIG + + VAD_RATE = 16000 + FRAME_MS = 30 + frame_samples = int(VAD_RATE * FRAME_MS / 1000) # 480 samples @ 16kHz + + aggressiveness = int(config.get("vad_aggressiveness", 2)) + start_frames = max(1, int(config.get("vad_start_ms", 150) / FRAME_MS)) + silence_frames = max(1, int(config.get("vad_silence_ms", 900) / FRAME_MS)) + max_frames = max(1, int(config.get("vad_max_record_ms", 30000) / FRAME_MS)) + preroll_frames = max(0, int(config.get("vad_preroll_ms", 300) / FRAME_MS)) + skip_frames = int(200 / FRAME_MS) # 丢弃头部 ~200ms，避开上一句 TTS 的回声尾巴 + + vad = webrtcvad.Vad(aggressiveness) + + # 采样率协商 + input_rate = choose_input_samplerate(INPUT_DEVICE_NAME, VAD_RATE) + use_resampling = (input_rate != VAD_RATE) + read_size = int(input_rate * FRAME_MS / 1000) if use_resampling else frame_samples + + buffer = [] # 已确认录音的帧（int16, 16000Hz） + preroll = collections.deque(maxlen=preroll_frames) # 起始前回看缓冲 + recording = False + voiced_run = 0 + silence_run = 0 + total_frames = 0 + idle_start = time.time() + + filename = f"flow_input_{int(time.time())}.wav" + + try: + # 释放硬件，避免 Pi 上音频争用死锁 + sd.stop() + time.sleep(0.2) + + with sd.InputStream(samplerate=input_rate, channels=1, dtype='int16', + blocksize=read_size, device=INPUT_DEVICE_NAME) as stream: + while True: + # --- 外部退出标志检查 --- + if exit_flag is not None and exit_flag.is_set(): + print("[VAD] Exit flag set, stopping.", flush=True) + return None + + # --- 空闲超时检测（仅等待说话阶段）--- + if not recording and timeout is not None: + elapsed = time.time() - idle_start + if elapsed > timeout: + print(f"[VAD TIMEOUT] No speech for {elapsed:.1f}s", flush=True) + return None + + data, _overflow = stream.read(read_size) + frame = np.frombuffer(data, dtype=np.int16) + if frame.ndim > 1: + frame = frame.flatten() + + # 最近邻重采样到 480 样本/帧（如果设备不支持 16kHz 直采） + if use_resampling: + step = len(frame) / frame_samples + idx = np.arange(0, len(frame), step)[:frame_samples].astype(int) + frame = frame[idx] + if len(frame) != frame_samples: + continue + + # 丢弃头部帧，避开上一句 TTS 的房间回声尾巴 + if skip_frames > 0: + skip_frames -= 1 + continue + + is_speech = vad.is_speech(frame.tobytes(), VAD_RATE) + + if not recording: + preroll.append(frame.copy()) + if is_speech: + voiced_run += 1 + if voiced_run >= start_frames: + recording = True + # 预缓冲并入开头，避免吞掉第一个字 + buffer.extend(preroll) + preroll.clear() + total_frames = len(buffer) + silence_run = 0 + print("[VAD] Speech detected, recording...", flush=True) + else: + voiced_run = 0 + else: + buffer.append(frame.copy()) + total_frames += 1 + if is_speech: + silence_run = 0 + else: + silence_run += 1 + if silence_run >= silence_frames: + print("[VAD] Trailing silence, stop.", flush=True) + break + if total_frames >= max_frames: + print("[VAD] Max record time reached, stop.", flush=True) + break + except Exception as e: + print(f"[VAD ERROR] Recording failed: {e}", flush=True) + return None + + if not buffer: + return None + return _save_int16_buffer(buffer, filename, samplerate=VAD_RATE) + + +def _save_int16_buffer(buffer, filename, samplerate=16000): + """将 int16 音频帧列表保存为 wav 文件""" + try: + audio_data = np.concatenate(buffer, axis=0).flatten().astype(np.int16) + with wave.open(filename, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(samplerate) + wf.writeframes(audio_data.tobytes()) + return filename + except Exception as e: + print(f"[AUDIO] Save buffer error: {e}", flush=True) + return None + + +# ========================================================================= +# 3. 放松音频生成 +# ========================================================================= + +def _generate_white_noise(duration_sec, sample_rate=44100): + """生成白噪音音频 (samples, rate)""" + n = int(sample_rate * duration_sec) + noise = np.random.randn(n).astype(np.float32) * 0.3 + return noise, sample_rate + + +# ========================================================================= +# 4. SLEEP FLOW 状态机 +# ========================================================================= + +class SleepFlow: + """ + 睡前情绪梳理机器人状态机编排。 + + 单向状态转移（不可逆）:: + + BOOT → CHAT → TRANSITION → AUDIO → SLEEP → SHUTDOWN + + 使用方式:: + + flow = SleepFlow(config=CURRENT_CONFIG, gui=bot_gui) + flow.start() # 阻塞直到 SHUTDOWN + + 参数: + config: 配置字典（通常是 ``CURRENT_CONFIG``） + gui: BotGUI 实例（提供 speak / transcribe_audio / set_state / play_sound 等方法） + 为 None 时进入纯日志模式（无音频 I/O），适合调试。 + """ + + # --- LLM 回复 fallback 池 ------------------------------------------------ + FALLBACKS = [ + "嗯，我在听。", + "好的，我知道了。", + "放松一点，没事的。", + "我在听你说。", + ] + + def __init__(self, config, gui=None): + self.cfg = config + self.sleep_cfg = config.get("sleep_flow", {}) + self.gui = gui + + # --- 硬件检查 --- + check_hardware() + + # --- 状态 --- + self.state = SleepState.BOOT + self.chat_round = 0 + self._exit_flag = False + + # --- 预合成缓存目录 ---------------------------------------------------- + self.cache_dir = self.sleep_cfg.get("cache_dir", "cache") + os.makedirs(self.cache_dir, exist_ok=True) + + # --- 预合成固定话术 --- + self._presynth_all() + + print(f"[FLOW] SleepFlow initialized | cache={os.path.abspath(self.cache_dir)}", flush=True) + + # ========================================================================= + # 4a. 缓存管理 + # ========================================================================= + + def _cache_path(self, key): + """返回缓存 wav 的完整路径""" + return os.path.join(self.cache_dir, f"{key}.wav") + + def _presynth_all(self): + """遍历 pre_synthesize_texts，缺失的调用 gui._render 补全并落盘""" + texts = self.sleep_cfg.get("pre_synthesize_texts", {}) + if not texts: + return + + renderer = None + if self.gui and hasattr(self.gui, "_render"): + renderer = self.gui._render + + for key, text in texts.items(): + path = self._cache_path(key) + if os.path.exists(path): + print(f"[CACHE] Hit: {key}", flush=True) + continue + + print(f"[CACHE] Synthesizing: {key} ...", flush=True) + try: + if renderer is not None: + result = renderer(text) + if result is not None: + samples, rate = result + _save_float32_wav(samples, rate, path) + print(f"[CACHE] Saved: {key} -> {path}", flush=True) + continue + # 没有 renderer 或渲染失败 → 跳过缓存，运行时走实时 TTS + print(f"[CACHE] Skip (no renderer): {key}", flush=True) + except Exception as e: + print(f"[CACHE] Error synthesizing {key}: {e}", flush=True) + + def _play_cached(self, key, fallback_text=None): + """ + 播放缓存 wav；不存在则用 gui.speak(fallback_text) 实时合成。 + 返回 True 表示实际播放/说了一句话。 + """ + path = self._cache_path(key) + if os.path.exists(path) and self.gui: + print(f"[AUDIO] Playing cached: {key}", flush=True) + self.gui.play_sound(path) + return True + + # 缓存缺失 → 实时 TTS + if fallback_text is None: + texts = self.sleep_cfg.get("pre_synthesize_texts", {}) + fallback_text = texts.get(key, "") + if fallback_text and self.gui: + print(f"[AUDIO] Live TTS: {key}", flush=True) + self.gui.speak(fallback_text) + return True + + return False + + # ========================================================================= + # 4b. GUI / 日志辅助 + # ========================================================================= + + def _set_state(self, state_name, msg=""): + """更新 GUI 状态显示；无 GUI 时仅打印日志""" + if self.gui: + self.gui.set_state(state_name, msg) + print(f"[FLOW] [{self.state.value}] {msg}", flush=True) + + def _append_text(self, text): + """追加文字到 GUI 对话文本框""" + if self.gui: + self.gui.append_to_text(text) + + # ========================================================================= + # 4c. LLM 调用 + # ========================================================================= + + def _call_llm(self, user_text, system_override=None): + """ + 调用本地 Ollama LLM，返回回复文本；失败时返回随机 fallback。 + + Args: + user_text: 本轮用户语音转写文本 + system_override: 可选的 system prompt 覆盖（默认用 SYSTEM_PROMPT） + """ + system = system_override or SYSTEM_PROMPT + lang = self.cfg.get("whisper_lang", "zh") + lang_hint = "请用中文回答。" if lang == "zh" else "" + + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user_text + ("\n" + lang_hint if lang_hint else "")}, + ] + + try: + with timed_block(f"LLM chat (round {self.chat_round})"): + resp = ollama.chat( + model=TEXT_MODEL, + messages=messages, + stream=False, + options=OLLAMA_OPTIONS, + keep_alive=-1, + ) + return _extract_ollama_content(resp) + except Exception as e: + print(f"[LLM ERROR] {e}", flush=True) + traceback.print_exc() + + # LLM 超时/失败 → fallback + return random.choice(self.FALLBACKS) + + # ========================================================================= + # 4d. STT 转写 + # ========================================================================= + + def _transcribe(self, audio_file): + """ + 转写音频文件，失败时重试一次。 + + Returns: + str: 转写文本（去两端空白），或 None（两次均失败） + """ + if not audio_file or not os.path.exists(audio_file): + return None + + text = self._transcribe_once(audio_file) + if text: + return text + + print("[STT] First attempt failed, retrying once...", flush=True) + time.sleep(0.5) + text = self._transcribe_once(audio_file) + if text: + return text + + # 二次失败 → 提示用户重说 + print("[STT] Both attempts failed, prompting user...", flush=True) + if self.gui: + self.gui.speak("我没听清，可以再说一遍吗") + return None + + def _transcribe_once(self, audio_file): + """单次转写，失败返回空字符串""" + try: + if self.gui: + text = self.gui.transcribe_audio(audio_file) or "" + else: + text = "" + if text.strip(): + return text.strip() + except Exception as e: + print(f"[STT ERROR] {e}", flush=True) + return "" + + # ========================================================================= + # 4e. 放松音频播放 + # ========================================================================= + + def _play_relax_audio(self, audio_type, timeout): + """ + 播放放松音频（白噪音/轻音乐/冥想），持续 timeout 秒或音频放完。 + + 优先从 ``sounds/relax/.wav`` 循环播放； + 文件不存在则生成白噪音。 + """ + audio_path = os.path.join("sounds", "relax", f"{audio_type}.wav") + start = time.time() + + if os.path.exists(audio_path) and self.gui: + print(f"[AUDIO] Playing file: {audio_path}", flush=True) + while time.time() - start < timeout and not self._exit_flag: + self.gui.play_sound(audio_path) + return + + # 无文件 → 生成白噪音 + if not _HAS_SOUNDDEVICE: + print(f"[AUDIO] sounddevice 不可用，无法播放音频，静等超时", flush=True) + time.sleep(min(timeout, 10)) + return + print(f"[AUDIO] No file at {audio_path}, generating white noise", flush=True) + try: + SAMPLE_RATE = 44100 + # 每次生成最长 5 分钟，循环播放直到超时 + chunk_sec = min(timeout, 300) + noise, rate = _generate_white_noise(chunk_sec, SAMPLE_RATE) + + while time.time() - start < timeout and not self._exit_flag: + remaining = timeout - (time.time() - start) + play_sec = min(remaining, chunk_sec) + end = int(rate * play_sec) + sd.play(noise[:end], rate) + sd.wait() + except Exception as e: + print(f"[AUDIO] Playback error: {e}", flush=True) + # 音频播放失败 → 静等剩余时间 + elapsed = time.time() - start + remaining = timeout - elapsed + if remaining > 0: + time.sleep(min(remaining, 10)) + + # ========================================================================= + # 5. PUBLIC API + # ========================================================================= + + def start(self): + """ + 启动状态机主循环，阻塞直到 SHUTDOWN。 + + 状态转移: BOOT → CHAT → TRANSITION → AUDIO → SLEEP → SHUTDOWN + """ + print("=" * 50, flush=True) + print(" 睡前情绪梳理机器人 Sleep Flow", flush=True) + print("=" * 50, flush=True) + + while self.state != SleepState.SHUTDOWN and not self._exit_flag: + try: + if self.state == SleepState.BOOT: + self._run_boot() + elif self.state == SleepState.CHAT: + self._run_chat() + elif self.state == SleepState.TRANSITION: + self._run_transition() + elif self.state == SleepState.AUDIO: + self._run_audio() + elif self.state == SleepState.SLEEP: + self._run_sleep() + else: + break + except Exception as e: + print(f"[FLOW CRITICAL] 状态 {self.state.value} 异常: {e}", flush=True) + traceback.print_exc() + # 出错后强制推进到下一状态，防止卡死 + self._force_next() + + self._run_shutdown() + + def stop(self): + """安全停止状态机（可在另一线程调用）""" + self._exit_flag = True + + # ========================================================================= + # 6. 各状态执行方法 + # ========================================================================= + + # ------------------------------------------------------------------ + # BOOT + # ------------------------------------------------------------------ + + def _run_boot(self): + """播放开机问候，自动转入 CHAT""" + self._set_state("greeting", "晚上好") + self._play_cached("greeting") + self._transition_to(SleepState.CHAT) + + # ------------------------------------------------------------------ + # CHAT + # ------------------------------------------------------------------ + + def _run_chat(self): + """ + 多轮对话状态。 + + 流程:: + + 录音（VAD + 静默超时）→ STT → LLM → TTS（缓存或实时） + ↑ 失败重试1次，仍失败提示后继续下一轮 + └── 超时无人说话 → 软收尾 → 跳到 TRANSITION + + 第 max_chat_rounds 轮播放收尾话术后强制转入 TRANSITION。 + """ + max_rounds = self.sleep_cfg.get("max_chat_rounds", 5) + silence_timeout = self.sleep_cfg.get("silence_timeout_chat", 75) + + self.chat_round = 0 + + while self.chat_round < max_rounds: + if self._exit_flag: + return + + round_label = f"第 {self.chat_round+1}/{max_rounds} 轮" + self._set_state("listening", f"我在听… {round_label}") + + # --- 录音（带安静超时） --- + audio_file = record_vad_with_timeout( + timeout=silence_timeout, + config=self.cfg, + exit_flag=None, # 用 self._exit_flag 在外部线程控制 + ) + + if self._exit_flag: + return + + if audio_file is None: + # 安静超时 → 软收尾后过渡 + print("[CHAT] 安静超时，软收尾...", flush=True) + self._play_cached("soft_close") + self._transition_to(SleepState.TRANSITION) + return + + # --- STT 转写 --- + user_text = self._transcribe(audio_file) + if user_text is None: + # 两次 STT 均失败，提示后继续监听（不占用轮次） + continue + + # --- 追加对话记录 --- + self._append_text(f"你: {user_text}") + + # --- 构造带轮次信息的 system prompt --- + chat_system = get_chat_prompt(self.chat_round, max_rounds) + + # --- LLM 回复 --- + self._set_state("thinking", "思考中…") + reply = self._call_llm(user_text, system_override=chat_system) + + if not reply: + reply = random.choice(self.FALLBACKS) + + # --- TTS 播放 --- + self._set_state("speaking", "回复中…") + self._append_text(f"机器人: {reply}") + if self.gui: + self.gui.speak(reply) + + # --- 轮次推进 --- + self.chat_round += 1 + + # 已满最大轮次 → 播放收尾话术后过渡 + if self.chat_round >= max_rounds: + print(f"[CHAT] 达到最大轮次 {max_rounds}，强制过渡", flush=True) + self._play_cached("round5_close") + self._transition_to(SleepState.TRANSITION) + return + + # while 正常结束（理论上不会走到这里，因为 while 条件就是 chat_round < max_rounds） + if self.state == SleepState.CHAT: + self._transition_to(SleepState.TRANSITION) + + # ------------------------------------------------------------------ + # TRANSITION + # ------------------------------------------------------------------ + + def _run_transition(self): + """播放过渡脚本，转入 AUDIO""" + self._set_state("idle", "准备放松") + self._play_cached("transition") + self._transition_to(SleepState.AUDIO) + + # ------------------------------------------------------------------ + # AUDIO + # ------------------------------------------------------------------ + + def _run_audio(self): + """ + 播放放松音频，用户在此阶段入眠。 + + - 音频类型由 ``sleep_flow.audio_type`` 指定 + - 持续 ``sleep_flow.shutdown_timeout`` 秒后自动结束 + - 期间不检测用户说话，不打扰 + """ + self._set_state("sleep", "放松中…") + audio_type = self.sleep_cfg.get("audio_type", "white_noise") + timeout = self.sleep_cfg.get("shutdown_timeout", 2700) # 45 分钟 + + print(f"[AUDIO] 开始播放: {audio_type} 持续时间: {timeout}s", flush=True) + self._play_relax_audio(audio_type, timeout) + + self._transition_to(SleepState.SLEEP) + + # ------------------------------------------------------------------ + # SLEEP + # ------------------------------------------------------------------ + + def _run_sleep(self): + """短暂停留后进入 SHUTDOWN""" + self._set_state("sleep", "晚安") + print("[SLEEP] 用户已在放松音频中入眠，5 秒后关机", flush=True) + time.sleep(5) + self._transition_to(SleepState.SHUTDOWN) + + # ========================================================================= + # 7. 状态转移 / 关机 + # ========================================================================= + + def _transition_to(self, new_state): + """单向状态转移（打印日志 + 记入内存）""" + prev = self.state.value + self.state = new_state + print(f"[FLOW] {prev} -> {new_state.value}", flush=True) + + def _force_next(self): + """出错时的紧急推进：无论如何跳到下一个状态""" + order = list(SleepState) + try: + idx = order.index(self.state) + if idx < len(order) - 1: + self.state = order[idx + 1] + print(f"[FLOW] ⚠ 强制前进到 {self.state.value}", flush=True) + else: + self.state = SleepState.SHUTDOWN + except ValueError: + self.state = SleepState.SHUTDOWN + + def _run_shutdown(self): + """关机 / 退出""" + shutdown_enabled = self.sleep_cfg.get("shutdown_enabled", False) + + if shutdown_enabled: + print("[SHUTDOWN] 执行系统关机...", flush=True) + self._set_state("sleep", "关机中") + try: + import subprocess + subprocess.run(["sudo", "halt"], check=True, timeout=10) + except Exception as e: + print(f"[SHUTDOWN] 关机命令失败: {e}", flush=True) + print("[SHUTDOWN] 回退: 退出进程", flush=True) + else: + print("[SHUTDOWN] 开发模式：此处会执行关机", flush=True) + print("[SHUTDOWN] 生产环境请设置 shutdown_enabled=true", flush=True) + + # 安全退出 GUI + if self.gui: + try: + self.gui.safe_exit() + except Exception as e: + print(f"[SHUTDOWN] safe_exit: {e}", flush=True) + + print("[FLOW] 晚安 🌙", flush=True) + + +# ========================================================================= +# 5. 工具函数 +# ========================================================================= + +def _extract_ollama_content(response): + """ + 从 ollama.chat 响应中安全提取文本内容。 + 兼容 ollama 库的 dict-style 与 object-style 两种接口。 + """ + if response is None: + return "" + # dict-style + if isinstance(response, dict): + return response.get("message", {}).get("content", "") + # object-style (pydantic BaseModel) + try: + return response.message.content + except AttributeError: + return "" + + +def _save_float32_wav(samples, rate, path): + """将 float32 [-1,1] 音频保存为 16-bit wav 文件""" + samples_int16 = (samples * 32767).clip(-32768, 32767).astype(np.int16) + with wave.open(path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(rate) + wf.writeframes(samples_int16.tobytes()) + + +# ========================================================================= +# 6. 独立测试入口 +# ========================================================================= + +if __name__ == "__main__": + # 纯日志模式（无 GUI、无音频硬件），验证初始化与状态流转 + print(">>> 测试 SleepFlow 纯日志模式 <<<") + test_cfg = { + "text_model": "gemma3:1b", + "whisper_lang": "zh", + "sleep_flow": { + "max_chat_rounds": 2, + "silence_timeout_chat": 10, + "audio_type": "white_noise", + "shutdown_timeout": 30, + "shutdown_enabled": False, + "cache_dir": "cache", + "pre_synthesize_texts": { + "greeting": "晚上好，今天过得怎么样？有什么想说的吗？", + "transition": "好的，已经记下了。现在让我们慢慢放松，准备休息吧。", + "soft_close": "如果你没什么想说的了，我们就开始放松吧。", + "round5_close":"我们先到这里，准备休息吧。", + }, + }, + } + flow = SleepFlow(config=test_cfg, gui=None) + print(">>> 初始化完成（无异常）<<<") + print(">>> cache/ 目录已创建，预合成结果如下：") + for f in sorted(os.listdir(flow.cache_dir)): + print(f" {f}") + print(">>> 测试结束 <<<") diff --git a/main.py b/main.py new file mode 100644 index 00000000..3bd8abed --- /dev/null +++ b/main.py @@ -0,0 +1,58 @@ +# ========================================================================= +# Be More Agent · 睡前情绪梳理机器人入口 +# 用 SleepFlow 状态机替换原有的开放式问答主循环。 +# +# 依赖: agent.py (BotGUI), flow.py (SleepFlow), config.py, prompts.py +# +# 启动方式: python main.py +# 注: 保留 agent.py 不动，agent.py 的 if __name__ == "__main__" 在新流程中不再使用。 +# ========================================================================= + +import tkinter as tk +import threading + +from config import CURRENT_CONFIG +from agent import BotGUI + + +def main(): + print("--- 睡前情绪梳理机器人 STARTING ---", flush=True) + + # 1. 创建 Tk 窗口和 GUI + root = tk.Tk() + app = BotGUI(root) + + # 2. 创建 SleepFlow 状态机，传入 BotGUI 实例 + # SleepFlow 会自动接管对话循环，不需要 BotGUI.safe_main_execution + from flow import SleepFlow + flow = SleepFlow(config=CURRENT_CONFIG, gui=app) + + # 3. 在后台线程启动状态机（不阻塞 Tk 主循环） + threading.Thread(target=flow.start, daemon=True).start() + + # 4. Tk 主循环在前台运行（响应用户按键等） + root.mainloop() + + +# ========================================================================= +# 修改说明 +# +# 原 entry point (agent.py 末尾): +# if __name__ == "__main__": +# root = tk.Tk() +# app = BotGUI(root) +# root.mainloop() +# +# 改为: +# python main.py ← SleepFlow 状态机自动接管 +# +# 关键变化: +# 1. BotGUI 不再启动 safe_main_execution 线程 +# 2. SleepFlow 代替 safe_main_execution 控制交互流程 +# 3. BotGUI 的 set_state / speak / transcribe_audio 等方法被 SleepFlow 调用 +# 4. 原有唤醒词 / PTT / 开放式对话全部停用 +# 5. agent.py 本身不做任何修改 +# ========================================================================= + +if __name__ == "__main__": + main() diff --git a/piper.tar.gz b/piper.tar.gz new file mode 100644 index 00000000..e69de29b diff --git a/prompts.py b/prompts.py new file mode 100644 index 00000000..26af9fa1 --- /dev/null +++ b/prompts.py @@ -0,0 +1,101 @@ +"""prompts.py — 系统提示词。 + +只放提示词文本，依赖 config.CURRENT_CONFIG。 +档2 的“每状态窄 prompt + few-shot”都放在这里， +让负责调 prompt 的人独占此文件、不与改 agent.py 的人冲突。 +""" + +from config import CURRENT_CONFIG + + +# BASE_SYSTEM_PROMPT：内置兜底人设。 +# 当 config.json 没有提供 system_prompt 时使用。 +BASE_SYSTEM_PROMPT = """你是一个睡前情绪梳理机器人，只服务于“睡前把今天轻轻放下”这个场景。 + +【目标】 +- 让用户在睡前把今天说出来、被听见，然后慢慢安静下来。 +- 成功不是聊得久，而是用户更容易停止思考、进入休息。 + +【说话方式】 +- 全程使用中文。 +- 温和、安静、像小夜灯一样陪着用户。 +- 每次最多两句话；每句话尽量短。 +- 只回应用户刚刚说的内容，不主动开启新话题。 + +【必须做】 +- 先接住用户的感受，再轻轻收束。 +- 用户提到烦心事、压力、冲突、后悔、担心、待办时，把它“寄存到明天”。 +- 接近结束时，不再追问，直接引导用户放松休息。 + +【禁止做】 +- 不给解决方案、行动计划、任务清单、分析框架。 +- 不深挖原因，不连续追问，不让用户越聊越清醒。 +- 不讲大道理，不输出心理学长解释，不泛泛鸡汤。 +- 不假装自己能诊断、治疗或替代专业帮助。 + +【安全边界】 +- 如果用户表达自伤、自杀、伤害他人或已经处于危险中：不要继续普通安抚；请简短、明确地让用户立刻联系当地紧急救援、身边可信任的人或危机热线，并尽量不要独处。 +""" + + +# 每状态窄 prompt：CHAT 阶段只负责“接住这一句”，不要替状态机做流程决策。 +CHAT_STAGE_PROMPT = """\n【当前阶段：睡前梳理】 +你只需要回应用户这一句话。结构优先为： +1. 用一句话接住情绪； +2. 用一句话把事情轻轻放下或寄存到明天。 +除非用户明确说“没什么了”，否则最多只问一个很轻的问题。""" + + +# few-shot：用示范约束小模型，重点是“短、接住、不解决、会收尾”。 +CHAT_FEW_SHOTS = """\n【回应示例】 +用户：今天工作又被说了，我觉得自己很没用。 +助手：被这样说一定不好受，你已经把这份委屈带到这里了。今晚先不评判自己，我们把它暂时放下。 + +用户：我和朋友吵架了，越想越生气。 +助手：这股生气还在身体里，确实很难一下子安静。先让它停在这里，明天清醒一点再看也可以。 + +用户：我今天说错话了，一直后悔。 +助手：我听见你在反复责怪自己。今晚先不用把这件事想明白，你已经记下了，明天再处理。 + +用户：那我明天到底该怎么办？ +助手：这个问题可以留给明天更清醒的你。今晚我们先让身体休息，不急着做决定。 + +用户：嗯……也没什么了。 +助手：好，那今天就先到这里。接下来我们慢慢放松，准备休息吧。""" + + +NEAR_END_PROMPT = """\n【轮次提示】 +对话接近尾声。不要再提出新的问题，可以开始自然收束，引导用户放松。""" + +FINAL_ROUND_PROMPT = """\n【轮次提示】 +这是本轮最后一次交流。必须自然收尾，告诉用户今天先到这里，接下来准备放松休息。""" + + +# SYSTEM_PROMPT：实际生效的系统提示。 +# 规则：config.json 的 system_prompt（无则用兜底）+ system_prompt_extras。 +SYSTEM_PROMPT = ( + CURRENT_CONFIG.get("system_prompt") or BASE_SYSTEM_PROMPT +) + "\n\n" + CURRENT_CONFIG.get("system_prompt_extras", "") + + +def get_chat_prompt(round_num, max_rounds=5): + """返回第 N 轮聊天使用的 system prompt，并按轮次追加窄 prompt / few-shot / 收尾提示。""" + lines = [SYSTEM_PROMPT, CHAT_STAGE_PROMPT, CHAT_FEW_SHOTS] + + if round_num >= max_rounds - 1: + lines.append(FINAL_ROUND_PROMPT) + elif round_num >= max_rounds - 2: + lines.append(NEAR_END_PROMPT) + + lines.append(f"\n（当前第 {round_num + 1}/{max_rounds} 轮）") + return "".join(lines) + + +def get_transition_prompt(): + """返回过渡步骤的 prompt 预留位；当前使用固定过渡语，不调用 LLM。""" + return "" + + +def get_boot_prompt(): + """返回开机问候的 prompt 预留位；当前使用固定问候语，不调用 LLM。""" + return "" diff --git a/requirements.txt b/requirements.txt index 8c4ebc02..f978508e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ sounddevice numpy scipy +setuptools<81 +webrtcvad openwakeword onnxruntime ollama diff --git a/setup.sh b/setup.sh old mode 100644 new mode 100755 diff --git a/test_agent.py b/test_agent.py new file mode 100644 index 00000000..69ef2109 --- /dev/null +++ b/test_agent.py @@ -0,0 +1,41 @@ +import re +<<<<<<< HEAD +======= +import json +>>>>>>> 1f88e85856a3fcf12e037e6633a5cf6ee5113ada + + +# --- 测试1：中文 bug --- +# 修复前，TTS 队列过滤正则要求句子含英文字母或数字，导致中文句子全部被丢弃 +def test_chinese_tts_filter_bug(): + chinese = "你好，今天天气很好。" + old_regex = r'[a-zA-Z0-9]' + new_regex = r'[\w一-鿿]' + assert not re.search(old_regex, chinese), "旧正则不应匹配纯中文（即旧代码会丢弃此句）" + assert re.search(new_regex, chinese), "新正则应匹配中文（修复后此句能进入 TTS 队列）" + + +# --- 测试2：speak() 文本清理保留中文标点 --- +# 修复前的正则会删掉，。！？等中文标点，导致 TTS 停顿异常 +def test_speak_clean_keeps_chinese_punct(): + text = "你好！今天天气，很好。" + clean = re.sub(r"[^\w\s,.!?:-，。！？、；：]", "", text) + assert "，" in clean, "逗号应被保留" + assert "。" in clean, "句号应被保留" + assert "！" in clean, "感叹号应被保留" + + +# --- 测试3：config 缺字段时有合理默认值 --- +# 保证旧的 config.json 不加新字段也能正常运行（向后兼容） +def test_whisper_config_defaults(): + config = {} + model = config.get("whisper_model", "ggml-base.en.bin") + lang = config.get("whisper_lang", "en") + assert model == "ggml-base.en.bin" + assert lang == "en" +# --- 测试4：JSON 动作解析 --- +def test_action_json_parse(): + text = '{"action": "get_time", "value": "now"}' + data = json.loads(text) + assert data["action"] == "get_time" + assert data["value"] == "now" diff --git a/voices/bmo.onnx.json b/voices/bmo.onnx.json new file mode 100644 index 00000000..9c46117b --- /dev/null +++ b/voices/bmo.onnx.json @@ -0,0 +1,497 @@ +{ + "dataset": "ko_voice_dojo", + "audio": { + "sample_rate": 22050, + "quality": "training_folder" + }, + "espeak": { + "voice": "en-us" + }, + "language": { + "code": "en-us" + }, + "inference": { + "noise_scale": 0.667, + "length_scale": 1, + "noise_w": 0.8 + }, + "phoneme_type": "espeak", + "phoneme_map": {}, + "phoneme_id_map": { + " ": [ + 3 + ], + "!": [ + 4 + ], + "\"": [ + 150 + ], + "#": [ + 149 + ], + "$": [ + 2 + ], + "'": [ + 5 + ], + "(": [ + 6 + ], + ")": [ + 7 + ], + ",": [ + 8 + ], + "-": [ + 9 + ], + ".": [ + 10 + ], + "0": [ + 130 + ], + "1": [ + 131 + ], + "2": [ + 132 + ], + "3": [ + 133 + ], + "4": [ + 134 + ], + "5": [ + 135 + ], + "6": [ + 136 + ], + "7": [ + 137 + ], + "8": [ + 138 + ], + "9": [ + 139 + ], + ":": [ + 11 + ], + ";": [ + 12 + ], + "?": [ + 13 + ], + "X": [ + 156 + ], + "^": [ + 1 + ], + "_": [ + 0 + ], + "a": [ + 14 + ], + "b": [ + 15 + ], + "c": [ + 16 + ], + "d": [ + 17 + ], + "e": [ + 18 + ], + "f": [ + 19 + ], + "g": [ + 154 + ], + "h": [ + 20 + ], + "i": [ + 21 + ], + "j": [ + 22 + ], + "k": [ + 23 + ], + "l": [ + 24 + ], + "m": [ + 25 + ], + "n": [ + 26 + ], + "o": [ + 27 + ], + "p": [ + 28 + ], + "q": [ + 29 + ], + "r": [ + 30 + ], + "s": [ + 31 + ], + "t": [ + 32 + ], + "u": [ + 33 + ], + "v": [ + 34 + ], + "w": [ + 35 + ], + "x": [ + 36 + ], + "y": [ + 37 + ], + "z": [ + 38 + ], + "æ": [ + 39 + ], + "ç": [ + 40 + ], + "ð": [ + 41 + ], + "ø": [ + 42 + ], + "ħ": [ + 43 + ], + "ŋ": [ + 44 + ], + "œ": [ + 45 + ], + "ǀ": [ + 46 + ], + "ǁ": [ + 47 + ], + "ǂ": [ + 48 + ], + "ǃ": [ + 49 + ], + "ɐ": [ + 50 + ], + "ɑ": [ + 51 + ], + "ɒ": [ + 52 + ], + "ɓ": [ + 53 + ], + "ɔ": [ + 54 + ], + "ɕ": [ + 55 + ], + "ɖ": [ + 56 + ], + "ɗ": [ + 57 + ], + "ɘ": [ + 58 + ], + "ə": [ + 59 + ], + "ɚ": [ + 60 + ], + "ɛ": [ + 61 + ], + "ɜ": [ + 62 + ], + "ɞ": [ + 63 + ], + "ɟ": [ + 64 + ], + "ɠ": [ + 65 + ], + "ɡ": [ + 66 + ], + "ɢ": [ + 67 + ], + "ɣ": [ + 68 + ], + "ɤ": [ + 69 + ], + "ɥ": [ + 70 + ], + "ɦ": [ + 71 + ], + "ɧ": [ + 72 + ], + "ɨ": [ + 73 + ], + "ɪ": [ + 74 + ], + "ɫ": [ + 75 + ], + "ɬ": [ + 76 + ], + "ɭ": [ + 77 + ], + "ɮ": [ + 78 + ], + "ɯ": [ + 79 + ], + "ɰ": [ + 80 + ], + "ɱ": [ + 81 + ], + "ɲ": [ + 82 + ], + "ɳ": [ + 83 + ], + "ɴ": [ + 84 + ], + "ɵ": [ + 85 + ], + "ɶ": [ + 86 + ], + "ɸ": [ + 87 + ], + "ɹ": [ + 88 + ], + "ɺ": [ + 89 + ], + "ɻ": [ + 90 + ], + "ɽ": [ + 91 + ], + "ɾ": [ + 92 + ], + "ʀ": [ + 93 + ], + "ʁ": [ + 94 + ], + "ʂ": [ + 95 + ], + "ʃ": [ + 96 + ], + "ʄ": [ + 97 + ], + "ʈ": [ + 98 + ], + "ʉ": [ + 99 + ], + "ʊ": [ + 100 + ], + "ʋ": [ + 101 + ], + "ʌ": [ + 102 + ], + "ʍ": [ + 103 + ], + "ʎ": [ + 104 + ], + "ʏ": [ + 105 + ], + "ʐ": [ + 106 + ], + "ʑ": [ + 107 + ], + "ʒ": [ + 108 + ], + "ʔ": [ + 109 + ], + "ʕ": [ + 110 + ], + "ʘ": [ + 111 + ], + "ʙ": [ + 112 + ], + "ʛ": [ + 113 + ], + "ʜ": [ + 114 + ], + "ʝ": [ + 115 + ], + "ʟ": [ + 116 + ], + "ʡ": [ + 117 + ], + "ʢ": [ + 118 + ], + "ʦ": [ + 155 + ], + "ʰ": [ + 145 + ], + "ʲ": [ + 119 + ], + "ˈ": [ + 120 + ], + "ˌ": [ + 121 + ], + "ː": [ + 122 + ], + "ˑ": [ + 123 + ], + "˞": [ + 124 + ], + "ˤ": [ + 146 + ], + "̃": [ + 141 + ], + "̧": [ + 140 + ], + "̩": [ + 144 + ], + "̪": [ + 142 + ], + "̯": [ + 143 + ], + "̺": [ + 152 + ], + "̻": [ + 153 + ], + "β": [ + 125 + ], + "ε": [ + 147 + ], + "θ": [ + 126 + ], + "χ": [ + 127 + ], + "ᵻ": [ + 128 + ], + "↑": [ + 151 + ], + "↓": [ + 148 + ], + "ⱱ": [ + 129 + ] + }, + "num_symbols": 256, + "num_speakers": 1, + "speaker_id_map": {}, + "piper_version": "1.0.0" +} \ No newline at end of file diff --git "a/\344\273\243\347\240\201\350\257\264\346\230\216\346\226\207\346\241\243.md" "b/\344\273\243\347\240\201\350\257\264\346\230\216\346\226\207\346\241\243.md" new file mode 100644 index 00000000..ac1d6638 --- /dev/null +++ "b/\344\273\243\347\240\201\350\257\264\346\230\216\346\226\207\346\241\243.md" @@ -0,0 +1,279 @@ +# 代码说明文档 · agent / config / prompts + +本项目（睡前情绪梳理机器人）当前由三个 Python 文件组成。本文档说明每个文件的职责、 +导出的函数/常量、以及它们之间的调用关系，供多人协作时查阅。 + +``` +config.py ← 纯配置 / 设备解析 / 计时工具 / 状态枚举（无 GUI、无运行时状态） +prompts.py ← 系统提示词（依赖 config） +agent.py ← 主程序：GUI + 音频管线 + LLM 对话 + 记忆（依赖 config / prompts） +``` + +依赖方向是单向的：`agent → prompts → config`，`config` 不反向依赖任何内部模块。 +**新增模块（如档2 的 `flow.py`）应只依赖 `config` / `prompts`，不要让 `config` 反向 import。** + +启动入口是 `python agent.py`（文件末尾的 `if __name__ == "__main__"`）。 + +--- + +## 一、config.py — 配置与常量 + +纯配置模块。导入时会执行一次性的配置加载与音频设备解析（有 module-level 副作用）， +因此 **import config 时就会去查询声卡**（在没有声卡的机器上会打印告警但不报错）。 + +### 常量 + +| 名称 | 含义 | +|------|------| +| `CONFIG_FILE` | 用户配置文件名 `"config.json"` | +| `MEMORY_FILE` | 对话历史文件名 `"memory.json"` | +| `WAKE_WORD_MODEL` | 唤醒词模型路径 `"./wakeword.onnx"` | +| `WAKE_WORD_THRESHOLD` | 唤醒触发分数阈值 `0.5` | +| `DEFAULT_CONFIG` | 配置默认值字典（缺失项的兜底） | +| `OLLAMA_OPTIONS` | 传给 `ollama.chat` 的推理参数（`keep_alive`/线程数/温度等） | +| `CURRENT_CONFIG` | **最终生效配置** = `DEFAULT_CONFIG` 被 `config.json` 覆盖后的结果 | +| `TEXT_MODEL` | 当前文本模型名（取自 `CURRENT_CONFIG["text_model"]`） | +| `INPUT_DEVICE_NAME` | 解析后的输入设备索引（`None` = 系统默认） | + +### 函数 + +**`timed_block(label)`** — 上下文管理器，计时并打印 `[TIMER]` 日志。 +```python +from config import timed_block +with timed_block("STT whisper-cli"): + run_whisper() # 退出时打印耗时 +``` + +**`load_config() -> dict`** +读 `config.json` 合并进 `DEFAULT_CONFIG`，返回最终配置。读取失败时回退默认值。 +模块加载时已调用一次，结果存入 `CURRENT_CONFIG`，**正常不需要再手动调用**。 + +**`resolve_input_device(config) -> int | None`** +把 `config["input_device"]`（可填索引数字或设备名子串）解析成 sounddevice 的设备索引。 +填 `None`/`""`/`"default"` 或找不到时返回 `None`（用系统默认设备）。 +模块加载时已调用一次，结果存入 `INPUT_DEVICE_NAME`。 + +**`choose_input_samplerate(device, preferred=None) -> int`** +为指定输入设备协商一个可用采样率。按 `preferred → 设备默认 → 48000/44100/32000/16000` +顺序逐个 `check_input_settings` 试探，返回第一个能用的。录音/唤醒前都会调用它， +以避免 ALSA 采样率不匹配报错。 + +### 类 + +**`BotStates`** — 状态字符串枚举（无实例，直接取类属性）。 +取值：`IDLE` / `LISTENING` / `THINKING` / `SPEAKING` / `ERROR` / `WARMUP`。 +用于驱动 GUI 表情动画和状态栏文字。 + +> 注意：原 `CAPTURING`（拍照）状态已随摄像头功能一起删除。 + +--- + +## 二、prompts.py — 系统提示词 + +只放提示词文本，依赖 `config.CURRENT_CONFIG`。档2 的“每状态窄 prompt + few-shot”都加在这里，让负责调 prompt 的人独占此文件、不与改 `agent.py` 的人冲突。 + +| 名称 | 含义 | +|------|------| +| `BASE_SYSTEM_PROMPT` | 内置兜底人设：睡前情绪梳理、温和简短、只接住不出主意，并包含安全边界 | +| `CHAT_STAGE_PROMPT` | CHAT 阶段窄 prompt：只回应用户这一句，先接住情绪，再寄存到明天 | +| `CHAT_FEW_SHOTS` | few-shot 示例：工作压力、人际冲突、后悔自责、索要方案、自然收尾 | +| `NEAR_END_PROMPT` | 倒数第二轮提示：减少追问，开始自然收束 | +| `FINAL_ROUND_PROMPT` | 最后一轮提示：必须收尾，引导放松休息 | +| `SYSTEM_PROMPT` | **实际生效的系统提示** = `config.json` 的 `system_prompt`（无则用兜底）+ `system_prompt_extras` | + +用法： + +```python +from prompts import SYSTEM_PROMPT +messages = [{"role": "system", "content": SYSTEM_PROMPT}, ...] +``` + +睡前状态机中应优先使用带轮次信息的版本： + +```python +from prompts import get_chat_prompt +chat_system = get_chat_prompt(round_num, max_rounds) +``` + +--- + +## 三、agent.py — 主程序 + +整个程序就一个类 `BotGUI`，承担 GUI 显示、音频采集、唤醒词、STT、LLM 对话、TTS、记忆。 +`__init__` 在 Tk 主线程构建界面，并启动一个后台守护线程 `safe_main_execution` 跑主循环。 + +### 线程模型（先理解这个再看函数） + +- **Tk 主线程**：只负责界面（`mainloop`）。所有改 UI 的操作都通过 `self.master.after(0, ...)` + 投递回主线程，因此 `set_state` / `append_to_text` / `_stream_to_text` 可从后台线程安全调用。 +- **主逻辑线程** `safe_main_execution`：唤醒→录音→转写→对话 的串行循环。 +- **TTS 两级流水线线程**：`_synth_worker`（合成）+ `_play_worker`（播放）两个独立线程， + 与 LLM 生成并行。合成线程把 `tts_queue` 里的句子**提前渲染**成音频压入 `audio_queue`， + 播放线程只管取出来播——这样第 N 句播放期间第 N+1 句已在合成，消除句间静音空挡。 +- **线程间通信靠这些共享状态**（都是 `self.xxx`）： + `ptt_event` / `recording_active` / `interrupted`（`threading.Event`）， + `tts_queue` + `tts_queue_lock`（待合成文本队列），`audio_queue` + `audio_queue_lock` + （已渲染音频队列，背压上限 `audio_queue_max`），`synth_active` / `play_active` + （两级各自的"忙"标志），`current_audio_process`。 + +### 3.1 构造与生命周期 + +| 方法 | 说明 | +|------|------| +| `__init__(self, master)` | 绑定按键（回车=PTT，空格=打断，Esc=退出）、初始化全部共享状态、加载唤醒词模型与（可选）sherpa TTS、建好 GUI 控件、启动主逻辑线程 | +| `safe_exit(self)` | 关机收尾：停音频进程、保存对话历史、卸载 ollama 模型、退出 Tk。`atexit` 已注册，幂等（`self.exiting` 防重入） | +| `exit_fullscreen(self, event=None)` | Esc 触发：退出全屏并 `safe_exit` | + +### 3.2 GUI / 状态显示 + +| 方法 | 说明 | +|------|------| +| `load_animations(self)` | 从 `faces//*.png` 读取各状态的逐帧图；缺图时回退到 idle 或蓝屏占位 | +| `update_animation(self)` | 定时切帧的动画循环（说话时 50ms 快切、其余 500ms）。`__init__` 启动后自循环 | +| `set_state(self, state, msg="")` | **切换状态的统一入口**。改 `current_state`（驱动表情）+ 状态栏文字 + 打印 `[STATE]` 日志。线程安全 | +| `append_to_text(self, text, newline=True)` | 往对话文本框追加整行（如 `YOU: ...`）。线程安全 | +| `_stream_to_text(self, chunk)` | 流式追加 LLM 输出的小片段（不换行）。线程安全 | +| `toggle_hud_visibility(self, event=None)` | 点屏幕：显示/隐藏文本框、状态栏、退出按钮 | + +### 3.3 输入触发与按键 + +| 方法 | 说明 | +|------|------| +| `handle_ptt_toggle(self, event=None)` | 回车键：切换"按下说话"(PTT)录音的开/关，带 0.5s 防抖。**当前主循环走 VAD 不再使用 PTT**，此绑定仅在切回唤醒词模式时生效 | +| `handle_speaking_interrupt(self, event=None)` | 空格键：在思考/说话时打断——置 `interrupted`、清空 `tts_queue` 与 `audio_queue` 两个队列、`sd.stop()` 停播放、杀音频进程、回到 IDLE | + +### 3.4 主循环 + +**`safe_main_execution(self)`** — 后台主线程入口。**已改为全程免手持续监听**（助眠场景），流程： +``` +warm_up_logic() → 启动合成/播放两个 TTS 线程 → 无限循环{ + record_voice_vad() # 持续监听：VAD 自动检测说话起止并录音 + transcribe_audio() # STT 转文字 + chat_and_respond() # LLM 回应 + TTS（内部阻塞到 TTS 放完） +} +``` +不再经过唤醒词/PTT 触发闸门：竖耳朵 → webrtcvad 检测到人声起始就自动录音 → 尾部静音 +就自动停 → 转写回复 → 回到监听。全程不碰按键、不喊唤醒词。 +没听到（返回 `None`）时安静地继续监听，不报错停顿。 +任何未捕获异常都会落到 `ERROR` 状态，不会让线程静默死掉。 + +> 因 `chat_and_respond()` 内部 `wait_for_tts()` 会阻塞到 TTS 全部放完才返回，主循环是串行的， +> 回到监听时机器人已不在说话，所以"录到自己 TTS 声音"的回声问题天然不存在。 + +**`warm_up_logic(self)`** — 开机预热。跑一次真实 `ollama.chat`（用 `permanent_memory` + "你好"） +把第1轮要用的 KV 前缀提前评估好，避免首轮 LLM 首 Token 慢（实测可从 ~16s 降到个位数）； +丢弃输出、不写入 memory。然后说一句中文开场白。 + +### 3.5 唤醒词（已停用，保留备查） + +> **现状**：主循环已改用 `record_voice_vad()` 持续监听，下列唤醒词方法与 `__init__` 里的 +> 唤醒词模型加载（`self.oww_model`）**均保留但不再被调用**，便于回滚/对照。如需切回唤醒词 +> 模式，把 `safe_main_execution` 改回调用 `detect_wake_word_or_ptt()` 即可。 + +| 方法 | 说明 | +|------|------| +| `detect_wake_word_or_ptt(self)` | 阻塞直到：检测到唤醒词→返回 `"WAKE"`；按了回车→`"PTT"`；终端回车→`"CLI"`。无唤醒模型时退化为纯 PTT。内含采样率协商 + 失败重试（松弛参数）两层兜底 | +| `_listen_loop(self, stream_args, input_chunk_size, target_chunk_size, use_resampling)` | 实际的音频读取+唤醒推理循环。用最近邻切片做快速重采样到 16kHz（省 CPU 防溢出），只在音量超阈值时才跑模型。命中唤醒/PTT/CLI 时用 `StopIteration` 跳出 | + +### 3.6 录音与转写 + +| 方法 | 说明 | +|------|------| +| `record_voice_vad(self, filename="input.wav")` | **当前主输入路径**。基于 webrtcvad 的免手持续监听：检测到人声起始（连续 `vad_start_ms`）自动开始录音，尾部静音 `vad_silence_ms` 或超 `vad_max_record_ms` 自动停。阻塞直到捕获完整一句，返回 wav 路径；没听到返回 `None`。详见下方说明 | +| `record_voice_adaptive(self, filename="input.wav")` | *（已停用，保留）* 唤醒词触发后用。能量阈值检测静音结束（1.5s 或最长 30s） | +| `record_voice_ptt(self, filename="input.wav")` | *（已停用，保留）* PTT 触发后用。一直录到 `recording_active` 被清除（再按一次回车） | +| `save_audio_buffer(self, buffer, filename, samplerate=16000, already_int16=False)` | 把录音缓冲区写成 16-bit 单声道 wav。`already_int16=True` 时（VAD 路径）直接落盘，跳过 `float×32767` 换算；缺省走原归一化路径 | +| `transcribe_audio(self, filename) -> str` | 调用 `whisper.cpp/build/bin/whisper-cli` 子进程转写，模型/语言取自配置。失败返回空串 | + +**`record_voice_vad` 要点**： +- webrtcvad 只吃 8/16/32/48kHz、16 位单声道、10/20/30ms 帧。这里用 16kHz×30ms=480 样本/帧。 +- 采样率：优先 `choose_input_samplerate` 协商出 16000Hz 直采；设备只能跑 44100/48000 时按原生率采集、用最近邻切片把每帧重采样到 480 样本（复用唤醒词循环同款技巧）。 +- 帧驱动状态机：`WAITING`（维护 `vad_preroll_ms` 预缓冲，连续语音达 `vad_start_ms` → 开始录音并把预缓冲并入开头，避免吞掉第一个字）→ `RECORDING`（累计尾部非语音达 `vad_silence_ms` → 停止）。 +- 录音前 `sd.stop()` 释放硬件（防 Pi 音频争用死锁），并丢弃头部 ~200ms 帧避开上一句 TTS 回声尾巴。 +- buffer 全程存 int16，`save_audio_buffer(..., already_int16=True)` 落盘。 +- **不依赖回车**：纯免手，无任何按键兜底。 + +### 3.7 LLM 对话（核心） + +**`chat_and_respond(self, text)`** — 一轮完整对话： +1. 命中"清空记忆/forget everything"等关键词 → 重置 memory 并提示，直接返回。 +2. 组装 `messages = permanent_memory + session_memory + [本轮 user]`（中文模式追加"请用中文回答"）。 +3. 流式 `ollama.chat`：边收 token 边 `_stream_to_text` 显示，按句标点切分后丢进 `tts_queue` 让 TTS 流水线（合成+播放两线程）并行朗读。 +4. 记录 `LLM 首Token延迟` 计时日志；`interrupted` 置位时中断。 +5. 整段回应写入 `session_memory`，等 TTS 放完回到 `IDLE`。 + +> 注意：原工具调用 / 拍照 / 联网搜索的 action-router 分支已删除，这里是纯聊天路径。 + +### 3.8 文本转语音（TTS） + +**两级流水线**：合成与播放解耦成两个线程，让"播第 N 句"和"合成第 N+1 句"重叠进行， +消除原先串行合成时听到的句间静音空挡（详见线程模型小节）。数据流： +`tts_queue`（文本）→ `_synth_worker` → `audio_queue`（音频缓冲）→ `_play_worker` → 声卡。 + +| 方法 | 说明 | +|------|------| +| `_synth_worker(self)` | **合成线程**：从 `tts_queue` 取句子调 `_render` 渲染成音频，压入 `audio_queue`。`audio_queue` 满（达 `audio_queue_max`）时背压等待，避免提前渲染堆积。靠 `synth_active` 标记"正在合成" | +| `_play_worker(self)` | **播放线程**：从 `audio_queue` 取已渲染音频调 `_play_samples` 播放。靠 `play_active` 标记"正在播" | +| `speak(self, text)` | **同步**合成并播放一句（阻塞）。用于开场问候等流水线 worker 启动前的场景；内部就是 `_render` + `_play_samples` | +| `_render(self, text)` | 合成总入口：清洗文本后按是否加载 sherpa 选 `_render_sherpa` / `_render_piper`，返回 `(samples float32, rate)` 或 `None`，**不播放** | +| `_init_sherpa_tts(self)` | 当 `config.tts_engine == "sherpa"` 时加载 sherpa-onnx 中文 VITS 模型，设 `num_threads`（取 `sherpa_num_threads`）吃满多核；失败则置 `None` 回退 piper | +| `_render_sherpa(self, text)` | 用 sherpa 合成中文，输出归一化后经 `_fit_samplerate` 适配采样率，返回音频缓冲。出错回退 `_render_piper` | +| `_render_piper(self, text)` | 用 `./piper/piper` 子进程合成（`communicate` 取全量 PCM），转 float32 后经 `_fit_samplerate` 适配，返回音频缓冲 | +| `_fit_samplerate(self, samples, rate)` | 声卡支持模型原生采样率就直接用；否则用 `scipy.signal.resample_poly`（多相，比 FFT 法 `resample` 快很多）重采样到声卡默认率 | +| `_play_samples(self, samples, rate)` | 播放一段已渲染音频（`sd.play` + 轮询等播完），`interrupted` 置位时 `sd.stop()` 立即停 | +| `wait_for_tts(self)` | 阻塞等待两级都空闲（`tts_queue`/`audio_queue` 皆空且 `synth_active`/`play_active` 都不忙；被打断则提前返回） | +| `play_sound(self, file_path)` | 通用 wav 播放器（必要时重采样）。**档2 放松音频会复用它** | + +> TIMER 日志：合成段打 `[TIMER] TTS sherpa synth [...]` / `TTS piper synth [...]`，播放段打 +> `[TIMER] TTS play [...]`，可分别看合成与播放各占多少耗时。 + +### 3.9 记忆持久化 + +| 方法 | 说明 | +|------|------| +| `load_chat_history(self) -> list` | 启动时读 `memory.json`，前置一条 `system` 消息（用 `SYSTEM_PROMPT`）。`system` 是配置不存盘 | +| `save_chat_history(self)` | 退出时存对话轮次到 `memory.json`，只留最近 10 轮，剔除 `system` 消息 | + +`permanent_memory`（跨会话历史）与 `session_memory`（本次会话）分开： +对话时拼在一起喂模型，存盘时合并后裁剪。 + +--- + +## 四、配置项速查（config.json） + +`config.json` 缺失时全用 `DEFAULT_CONFIG`；存在时逐项覆盖。常用键： + +| 键 | 作用 | 默认 | +|----|------|------| +| `text_model` | ollama 文本模型 | `gemma3:1b` | +| `voice_model` | piper 语音模型路径 | `piper/en_GB-semaine-medium.onnx` | +| `whisper_model` / `whisper_lang` | whisper 模型与语言（`zh` 会触发中文回答提示） | `ggml-base.en.bin` / `en` | +| `input_device` / `input_sample_rate` | 麦克风设备（索引或名字子串）/ 采样率 | `None` | +| `system_prompt` / `system_prompt_extras` | 覆盖人设 / 追加人设 | — | +| `tts_engine` | 设为 `"sherpa"` 启用中文 sherpa TTS | 不设=piper | +| `sherpa_model_dir` / `sherpa_speaker_id` / `sherpa_speed` | sherpa 模型目录 / 说话人 / 语速 | 见 `_init_sherpa_tts` | +| `sherpa_num_threads` | sherpa 合成线程数（设为 CPU 核数可大幅提速，Pi 填 4） | `4` | +| `vad_aggressiveness` | webrtcvad 灵敏度 0~3，越大越严格、越不易把噪声当人声 | `3` | +| `vad_start_ms` | 连续多少毫秒判定为人声才算"开始说话"（防瞬时噪声误触发） | `150` | +| `vad_silence_ms` | 尾部静音多久判定"说完"并停止录音 | `900` | +| `vad_max_record_ms` | 单次最长录音时长 | `30000` | +| `vad_preroll_ms` | 起始前回看缓冲，避免吞掉第一个字 | `300` | + +> **VAD 调参**：嫌接话慢→调小 `vad_silence_ms`（如 600）；环境吵、老被噪声误触发→ +> `vad_aggressiveness` 提到 3（已是最大）；开头第一个字被吞→调大 `vad_preroll_ms`。 + +--- + +## 五、给协作者的提示 + +- **改提示词** → 只动 `prompts.py`。 +- **改配置项/默认值/设备逻辑** → 只动 `config.py`。 +- **改对话/音频/GUI 行为** → 动 `agent.py` 对应小节。 +- **写档2 状态机** → 新建 `flow.py`，只 import `config` / `prompts`，把睡前流程编排独立出来， + 尽量不把新逻辑塞进已经很满的 `chat_and_respond`。 +- 三个文件改完用 `python -m py_compile config.py prompts.py agent.py` 做语法自检； + 但真正能否跑通（声卡 / ollama / whisper / piper）**必须在树莓派上 `python agent.py` 实测**。 +- **依赖踩坑（VAD）**：免手监听用了 `webrtcvad`，它在导入时会 `import pkg_resources`， + 而 setuptools≥81 已移除该模块，故 `requirements.txt` 固定 `setuptools<81`。Python 3.12+ + 的全新 venv 默认不装 setuptools，记得 `pip install -r requirements.txt` 一并装上。 +``` diff --git "a/\347\235\241\345\211\215\346\203\205\347\273\252\346\242\263\347\220\206\346\234\272\345\231\250\344\272\272 \302\267 \345\256\236\346\226\275\350\256\241\345\210\222.md" "b/\347\235\241\345\211\215\346\203\205\347\273\252\346\242\263\347\220\206\346\234\272\345\231\250\344\272\272 \302\267 \345\256\236\346\226\275\350\256\241\345\210\222.md" new file mode 100644 index 00000000..f1a88716 --- /dev/null +++ "b/\347\235\241\345\211\215\346\203\205\347\273\252\346\242\263\347\220\206\346\234\272\345\231\250\344\272\272 \302\267 \345\256\236\346\226\275\350\256\241\345\210\222.md" @@ -0,0 +1,97 @@ +# 睡前情绪梳理机器人 · 实施计划 + +## 目标 + +课设项目（树莓派离线语音陪伴机器人，fork 自 be-more-agent），下周交。 +把产品从"泛泛陪伴"**收窄为"睡前梳理情绪"**，服从一条单向主线： + +> 通电开机 → 说出今天 → 情绪落地 → 放松音频 → 睡着 → 自动关机 + +成功指标是"用户睡着"，不是"用得久"。 + +代码结构与各函数用法见 [代码说明文档.md](代码说明文档.md)。 + +--- + +## 进度速览 + +| 模块 | 状态 | +|------|------| +| 清场删除（摄像头 / 联网搜索 / 英文音效） | ✅ 完成 | +| 轻量拆分（[config.py](config.py) / [prompts.py](prompts.py) 抽出，agent 瘦身到 ~845 行） | ✅ 完成 | +| 首轮延迟修复（`warm_up_logic` 真实预热前缀） | ✅ 完成 | +| 档2 主线（状态机 / 窄 prompt / 放松音频 / 沉默 / 关机） | ⏳ 未完成 | +| 档3 加分（当日摘要、晨间冥想） | ⏳ 未完成 | + +> 拆分决策：只做轻拆，**不**照搬七文件方案。audio/stt/tts/GUI 与运行时状态深度耦合， +> 交付前一周拆出去性价比为负，留在 `agent.py`。新流程独立成 `flow.py`，只 import config/prompts。 + +--- + +## 交互设计规格 + +### 主流程状态机 + +- **开场（脚本）**：开机后播固定问候；若存在"近一两天的昨天摘要"，可选择性接一句"昨天你说…今天呢？"。 +- **梳理（LLM）**：听用户说今天，温和接住。前几句可引导用户说出想法 / 情绪 / 身体感受（具体事件时）或其他合理回应。 + - **不深挖、不在睡前出主意解决问题**：烦心事一律"寄存"（记下了，明天再想）。 + - 建议三句左右收尾进入过渡，**五句封顶（硬上限）**。 +- **过渡（脚本）** → 进入放松音频。 +- **放松音频**：三种——语音引导冥想 / 白噪音（如雨声）/ 轻音乐合集，**默认白噪音，可配置**（自定义音频留作 future work）。 + 单条音频、做好音量平衡；无淡入淡出、无 LED。进入后**绝不要求用户回应、绝不"你还在吗"叫醒**。 +- **关机**：音频自然放完 **或** 满 45 分钟，先到为准 → 自动 `halt`。 + +### 沉默与关机（分阶段） + +- **梳理阶段**沉默 60–90s → 不直接关机，轻收一句、滑入音频。 +- **音频阶段**沉默 = 正常（人睡了），不处理、不打扰。 + +### 记录今日感受（存 / 读） + +- **存**：会话结束时 LLM 多跑一次，把今天压成"一句话摘要 + 日期"，只存这一句；**异步执行，不阻塞睡眠流程**。原始转录仅留作调试，不进读取链路。 +- **读**：仅当存在"近一两天"的昨天摘要时开场接上；隔太久不主动提。 + +### 核心难点：怎么让 LLM 听话 + +问题：把"短、不出主意、该收就收"写进 system prompt **无效**——模型本能是"有用 = 多说、给建议、接话题"，与需求对冲。 + +**原则：不靠模型自律，用代码他律。** 四个手段（看情况组合）： + +1. **`num_predict` 硬卡（约 60）**：物理上保证回应短。 +2. **回合上限由编排代码定**：梳理约 3–5 个来回到顶，到点由状态机**强制转音频**，不让模型自己决定要不要继续。 +3. **few-shot 范例**：prompt 里放 3–4 段目标对话（用户吐槽 → 只接住+寄存，不出主意）；示范是强约束，远胜文字描述。 +4. **拆 prompt**：固定话术写死，LLM 只负责"接住用户这一句"；每个状态各配一个窄小 prompt，而非一个什么都管的巨型 prompt。 + +--- + +## 待办 · 档2（特色主线，务必跑通） + +按主线顺序，核心是新写 `flow.py` 把开放循环换成单向状态机。 + +| # | 任务 | 落到哪 | +|---|------|--------| +| 1 | **状态机 + 回合计数**：开场→梳理(≤5句强制转)→过渡→音频→关机 | 新 `flow.py` | +| 2 | **窄 prompt + few-shot**：吐槽时只接住+寄存，不出主意；每状态一个 prompt | [prompts.py](prompts.py) | +| 3 | **回应短**：`OLLAMA_OPTIONS` 加 `'num_predict': 60` | [config.py](config.py) | +| 4 | **放松音频播放**：默认白噪音，三选一可配置（播放复用 `play_sound`） | 新模块 + 配置项 | +| 5 | **阶段化沉默**：梳理静默 60–90s 软收一句滑入音频；音频阶段不打扰 | 改 `record_voice_adaptive` | +| 6 | **自动关机**：音频放完 或 满 45min → `sudo halt`（开发期用开关位避免真关机） | `flow.py` | +| 7 | **入口去唤醒词**：开机即开场（当前仍加载 openWakeWord） | `agent.py` / `flow.py` | + +依赖项：放松音频素材（无版权白噪音/轻音乐）**需第三人收集**，代码先按可配置路径做接口。 + +## 待办 · 档3（加分，能砍） + +- **当日一句话摘要**：会话结束 LLM 压成一句+日期异步存；开场仅当存在"近一两天"摘要时接上。 +- **夜间休眠 + 晨间冥想**：起床后切到晨间冥想流程（探索性，不确定能否实现）。 + +--- + +## 验收清单 + +- **首轮延迟**：真机连跑两轮，对比 `LLM 首Token延迟` 日志，第1轮应接近第2轮（目标 16s → 个位数）。 +- **保命线**：上电开场为中文、全程无英文音效。 +- **主线链路**：走通"说今天 → 3–5 轮 → 过渡 → 白噪音"，梳理 ≤5 句被强制转音频。 +- **沉默**：梳理静默 60–90s 触发软收滑入音频；音频阶段静默不打扰。 +- **关机**：音频放完或 45min 触发 `halt`（开发期用日志代替真关机）。 +- 全程用 `[TIMER]` 日志（[config.py](config.py) 的 `timed_block`）核对各段耗时。