import os import time import cv2 import numpy as np import gradio as gr from reachy_mini import ReachyMini from reachy_mini.utils import create_head_pose from reachy_mini.media.media_manager import MediaManager import sys import pyaudio import threading import math import main as camp_main class StdoutRedirector: def __init__(self): self.buffer = [] self.original_stdout = sys.stdout self.original_stderr = sys.stderr def write(self, message): self.original_stdout.write(message) self.original_stdout.flush() if message: self.buffer.append(message) if len(self.buffer) > 200: # limit to 200 lines self.buffer.pop(0) def flush(self): self.original_stdout.flush() def get_text(self): return "".join(self.buffer) def clear(self): self.buffer = [] def __getattr__(self, name): # 代理所有其他屬性/方法至原始的 stdout return getattr(self.original_stdout, name) stdout_redirector = StdoutRedirector() sys.stdout = stdout_redirector sys.stderr = stdout_redirector class AudioLevelMonitor: def __init__(self): self.running = False self.thread = None self.history = [0.0] * 30 self.p = None self.stream = None def start(self): if self.running: return self.running = True self.thread = threading.Thread(target=self._run, daemon=True) self.thread.start() def stop(self): self.running = False if self.thread: self.thread.join(timeout=1.0) self.thread = None self.history = [0.0] * 30 def _run(self): self.p = pyaudio.PyAudio() try: self.stream = self.p.open( format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024 ) except Exception as e: print(f"⚠️ Cannot open microphone for waveform monitoring: {e}") self.p.terminate() return while self.running: try: data = self.stream.read(1024, exception_on_overflow=False) audio_data = np.frombuffer(data, dtype=np.int16) # Calculate RMS rms = np.sqrt(np.mean(np.square(audio_data))) # Normalize to a nice range normalized = min(40.0, (rms / 200.0) * 40.0) if normalized < 2.0: normalized = 2.0 + np.random.uniform(0, 1.5) self.history.pop(0) self.history.append(normalized) except Exception: pass time.sleep(0.05) try: self.stream.stop_stream() self.stream.close() except Exception: pass self.p.terminate() def get_svg_wave(self): svg = '' svg += """ """ bar_width = 4 gap = 2 for i, val in enumerate(self.history): x = i * (bar_width + gap) + 10 height = max(3.0, val) y = 30 - height / 2 svg += f'' svg += '' return svg audio_monitor = AudioLevelMonitor() class ManualVoiceRecorder: def __init__(self): self.frames = [] self.is_recording = False self.thread = None self.p = None self.stream = None def start(self): if self.is_recording: return self.frames = [] self.is_recording = True self.thread = threading.Thread(target=self._record_loop, daemon=True) self.thread.start() def _record_loop(self): self.p = pyaudio.PyAudio() try: self.stream = self.p.open( format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024 ) except Exception as e: print(f"❌ Manual recording failed to open microphone: {e}") self.is_recording = False self.p.terminate() return while self.is_recording: try: data = self.stream.read(1024, exception_on_overflow=False) self.frames.append(data) # Synchronize audio_monitor's history update to animate the waveform audio_data = np.frombuffer(data, dtype=np.int16) rms = np.sqrt(np.mean(np.square(audio_data))) normalized = min(40.0, (rms / 200.0) * 40.0) if normalized < 2.0: normalized = 2.0 + np.random.uniform(0, 1.5) audio_monitor.history.pop(0) audio_monitor.history.append(normalized) except Exception as e: print(f"❌ Failed to read recording data: {e}") break try: self.stream.stop_stream() self.stream.close() except Exception: pass self.p.terminate() def stop_and_save(self): if not self.is_recording: return False self.is_recording = False if self.thread: self.thread.join(timeout=2.0) self.thread = None if not self.frames: print("❌ No audio recorded.") return False record_dir = os.path.join(camp_main.SCRIPT_DIR, "record") os.makedirs(record_dir, exist_ok=True) wav_filename = os.path.join(record_dir, "latest_voice.wav") mp3_filename = os.path.join(record_dir, "latest_voice.mp3") # Write WAV import wave try: wf = wave.open(wav_filename, 'wb') wf.setnchannels(1) wf.setsampwidth(2) # paInt16 is 2 bytes wf.setframerate(16000) wf.writeframes(b''.join(self.frames)) wf.close() except Exception as e: print(f"❌ Failed to write temporary WAV file: {e}") return False # 轉換為 MP3 converted = False # 嘗試使用 pydub try: from pydub import AudioSegment audio_segment = AudioSegment.from_wav(wav_filename) audio_segment.export(mp3_filename, format="mp3") converted = True except Exception: pass # 嘗試使用 ffmpeg if not converted: try: import subprocess subprocess.run( ["ffmpeg", "-y", "-i", wav_filename, "-codec:a", "libmp3lame", "-qscale:a", "2", mp3_filename], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True ) converted = True except Exception: pass # 刪除暫存 WAV 檔 if os.path.exists(wav_filename): try: os.remove(wav_filename) except Exception: pass if converted: print(f"✨ [Manual recording saved successfully] -> record/latest_voice.mp3") return True else: print("❌ Failed to convert recording to MP3.") return False manual_voice_recorder = ManualVoiceRecorder() # ----------------- Monkey Patch 處理 SSH Tunnel 遠端連線問題 ----------------- # 備份原始的 MediaManager 初始化函數,便於在 UI 中切換是否啟用 SSH 隧道模式 original_media_manager_init = MediaManager.__init__ # 建立一個 RobotManager 來管理 persistent (持久) 的 ReachyMini 連線 class RobotManager: def __init__(self): self.mini = None self.last_frame = None self.placeholder_frame = self.make_placeholder_frame("Disconnected") def make_placeholder_frame(self, message="Disconnected"): """Generate a dark placeholder frame with status message""" frame = np.zeros((480, 640, 3), dtype=np.uint8) # Use dark slate blue background frame[:, :] = [26, 28, 38] # Draw status text using OpenCV font = cv2.FONT_HERSHEY_SIMPLEX text_size = cv2.getTextSize(message, font, 1.0, 2)[0] text_x = (640 - text_size[0]) // 2 text_y = (480 + text_size[1]) // 2 cv2.putText(frame, message, (text_x, text_y), font, 1.0, (140, 140, 160), 2, cv2.LINE_AA) # Return RGB format for Gradio return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) def connect(self, host, connection_mode, media_backend, force_localhost_signalling): if self.mini is not None: return "🤖 Reachy Mini is already connected." # Apply monkey patch based on SSH Tunnel enablement if force_localhost_signalling: def mocked_media_manager_init(mm_self, *args, **kwargs): if kwargs.get("signalling_host") not in (None, "localhost", "127.0.0.1"): kwargs["signalling_host"] = "localhost" original_media_manager_init(mm_self, *args, **kwargs) MediaManager.__init__ = mocked_media_manager_init else: MediaManager.__init__ = original_media_manager_init try: # Initialize ReachyMini self.mini = ReachyMini( host=host, connection_mode=connection_mode, media_backend=media_backend ) # Invoke context manager's enter method self.mini.__enter__() # Wait for camera initialization and fetch the first frame to verify connection start_time = time.time() frame = None while frame is None and time.time() - start_time < 5.0: frame = self.mini.media.get_frame() if frame is None: time.sleep(0.1) if frame is not None: self.last_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) return "✅ Connection successful! Camera frame retrieved." else: return "⚠️ Connected, but camera module initialization timed out. No image received." except Exception as e: self.disconnect() return f"❌ Connection failed: {str(e)}" def disconnect(self): if self.mini is not None: try: self.mini.__exit__(None, None, None) except Exception as e: print(f"Error during disconnect exit: {e}") self.mini = None self.last_frame = None return "🔌 Disconnected." def get_frame(self): if self.mini is None: return self.placeholder_frame try: frame = self.mini.media.get_frame() if frame is not None: frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) self.last_frame = frame_rgb return frame_rgb elif self.last_frame is not None: return self.last_frame else: return self.make_placeholder_frame("Waiting for camera frame...") except Exception as e: print(f"Error fetching frame: {e}") return self.make_placeholder_frame("Camera connection lost") def get_frame_stream(self): while True: if self.mini is None: yield self.placeholder_frame time.sleep(0.5) else: try: frame = self.mini.media.get_frame() if frame is not None: frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) self.last_frame = frame_rgb yield frame_rgb elif self.last_frame is not None: yield self.last_frame else: yield self.make_placeholder_frame("Waiting for camera frame...") except Exception as e: print(f"Error fetching frame in stream: {e}") yield self.make_placeholder_frame("Camera connection lost") time.sleep(0.1) def move_head(self, yaw, pitch, roll, antennas_left, antennas_right, body_yaw): if self.mini is None: return "❌ Please connect the robot before controlling movements." try: head_pose = create_head_pose( yaw=yaw, pitch=pitch, roll=roll, degrees=True, mm=True ) antennas = np.deg2rad([antennas_left, antennas_right]).tolist() body_yaw_rad = np.deg2rad(body_yaw) self.mini.goto_target( head=head_pose, antennas=antennas, body_yaw=body_yaw_rad, duration=1.5, method="minjerk" ) return "✅ Movement command sent successfully!" except Exception as e: return f"❌ Movement control failed: {str(e)}" def set_motors_torque(self, enable=True): if self.mini is None: return "❌ Robot not connected yet." try: if enable: self.mini.enable_motors() return "✅ Motor torque enabled (Torque ON)" else: self.mini.disable_motors() return "✅ Motor torque disabled (Torque OFF - Robot relaxed)" except Exception as e: return f"❌ Motor setting failed: {str(e)}" # 實例化管理器 robot_manager = RobotManager() # ----------------- Gradio UI 建立 ----------------- custom_css = """ body { background-color: #0c0d14; color: #f3f4f6; font-family: 'Outfit', 'Inter', sans-serif; } .gradio-container { background-color: #0c0d14 !important; } .header-box { text-align: center; padding: 30px; background: linear-gradient(135deg, #1e1e38 0%, #0f0f1b 100%); border-radius: 16px; box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4); border: 1px solid rgba(255, 255, 255, 0.1); margin-bottom: 24px; } .header-title { background: linear-gradient(90deg, #6366f1, #06b6d4); -webkit-background-clip: text; -webkit-text-fill-color: transparent; font-size: 2.2rem !important; font-weight: 800 !important; margin: 0; } .header-subtitle { color: #9ca3af; font-size: 1rem; margin-top: 8px; } .custom-card { background-color: #111322 !important; border: 1px solid rgba(255, 255, 255, 0.05) !important; border-radius: 12px !important; padding: 16px !important; } .pulse-dot { width: 12px; height: 12px; background-color: #ef4444; border-radius: 50%; display: inline-block; } .pulse-active { background-color: #10b981 !important; animation: pulse 1.5s infinite; } @keyframes pulse { 0% { transform: scale(0.95); box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); } 70% { transform: scale(1.05); box-shadow: 0 0 0 8px rgba(16, 185, 129, 0); } 100% { transform: scale(0.95); box-shadow: 0 0 0 0 rgba(16, 185, 129, 0); } } .manual-btn { background: linear-gradient(135deg, #1b1c2b 0%, #131422 100%) !important; border: 1px solid rgba(255, 255, 255, 0.08) !important; color: #e2e8f0 !important; transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important; font-weight: 600 !important; border-radius: 10px !important; box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1) !important; padding: 10px 14px !important; text-align: center !important; } .manual-btn:hover { background: linear-gradient(135deg, #312e81 0%, #1e1b4b 100%) !important; border-color: #6366f1 !important; box-shadow: 0 0 15px rgba(99, 102, 241, 0.4) !important; transform: translateY(-2px) !important; color: #ffffff !important; } .manual-btn:active { transform: translateY(1px) !important; } .manual-group { background-color: rgba(255, 255, 255, 0.01) !important; border: 1px solid rgba(255, 255, 255, 0.05) !important; border-radius: 14px !important; padding: 16px !important; margin-top: 12px !important; margin-bottom: 12px !important; } """ with gr.Blocks(title="Reachy Mini Camping Smart Assistant & Control Panel", css=custom_css) as demo: # Header Section gr.HTML("""

🏕️ Reachy Mini Camping Smart Voice & Guard Control Platform

Integrating multimodal vision recognition (MiniCPM-V), smart voice dialogue (Whisper/Gemini/ChatTTS), security guard patrol, and geolocation reports for a comprehensive smart camping assistant experience

You need to connect to Reachy via Reachy Mini Control first.

""") with gr.Row(): # Left column: Connection and settings management with gr.Column(scale=3, elem_classes=["custom-card"]): gr.Markdown("### 🔌 Connection & Settings") host_input = gr.Textbox( label="Host IP (Host)", value="localhost", placeholder="e.g., localhost, reachy-mini.local, or 192.168.1.175" ) with gr.Row(): conn_mode = gr.Dropdown( label="Connection Mode", choices=["network", "local"], value="network" ) backend_mode = gr.Dropdown( label="Video Backend", choices=["default", "webrtc", "no_media"], value="default" ) tunnel_mode = gr.Checkbox( label="Enable SSH Tunnel Mode (forces signalling_host to localhost)", value=True ) with gr.Row(): btn_connect = gr.Button("🔗 Connect Reachy Mini", variant="primary") btn_disconnect = gr.Button("🔌 Disconnect", variant="stop") # Removed motor torque control and system status textbox # Middle column: Real-time video stream window with gr.Column(scale=5, elem_classes=["custom-card"]): stream_title_html = gr.HTML("""

📷 Reachy Mini Head View

""") # Setup image output component and use every to fetch get_frame periodically image_output = gr.Image( label="Live Stream", type="numpy" ) # Use demo.load to load generator stream image, compatible with all Gradio versions stream_event = demo.load( fn=robot_manager.get_frame_stream, outputs=image_output ) btn_snapshot = gr.Button("📸 Capture Frame & Save", variant="secondary") snapshot_status = gr.Textbox(label="Capture Status", interactive=False) # Right column: Camp AI Voice Control Panel with gr.Column(scale=4, elem_classes=["custom-card"]): # === Camp AI Voice Control Panel === camp_pulse_dot = gr.HTML(value='

🏕️ Camp AI Voice Control System

') with gr.Row(): btn_camp_start = gr.Button("🎤 Start Camp AI", variant="primary") btn_camp_stop = gr.Button("🛑 Stop Camp AI", variant="stop", interactive=False) with gr.Group(elem_classes=["manual-group"]): gr.Markdown("

⚡ Manual Mode

") with gr.Row(): btn_location = gr.Button("📍 Location Report", variant="secondary", elem_classes=["manual-btn"]) btn_voice = gr.Button("💬 Voice Dialogue", variant="secondary", elem_classes=["manual-btn"]) with gr.Row(): btn_guardian = gr.Button("🛡️ Guardian Mode", variant="secondary", elem_classes=["manual-btn"]) btn_species = gr.Button("🔍 Species Identification", variant="secondary", elem_classes=["manual-btn"]) btn_stop_voice = gr.Button("⏹️ Stop Recording & Run Dialogue", variant="stop", visible=False) camp_status = gr.Textbox(label="Voice Service Status", value="Not Started", interactive=False) gr.Markdown("🔊 **Microphone Input Waveform**") camp_wave_html = gr.HTML(value=audio_monitor.get_svg_wave()) camp_log_output = gr.Textbox( label="Camp AI Execution Logs (stdout)", value="Voice service not started yet.", lines=8, max_lines=12, interactive=False ) # --- 互動邏輯綁定 --- # Connect and disconnect callbacks def on_connect(host, conn, backend, tunnel): res = robot_manager.connect(host, conn, backend, tunnel) print(res) # When connection is successful, set the pulse dot indicator to active green if "successful" in res.lower(): return gr.update(value='

📷 Reachy Mini Head View

') return gr.update(value='

📷 Reachy Mini Head View

') def on_disconnect(): res = robot_manager.disconnect() print(res) return gr.update(value='

📷 Reachy Mini Head View

') btn_connect.click( fn=on_connect, inputs=[host_input, conn_mode, backend_mode, tunnel_mode], outputs=stream_title_html ) btn_disconnect.click( fn=on_disconnect, outputs=stream_title_html ) # Snapshot functionality def take_snapshot(): if robot_manager.mini is None: return "❌ Please connect first before capturing a frame." frame = robot_manager.get_frame() if frame is not None: record_dir = os.path.join(camp_main.SCRIPT_DIR, "record") os.makedirs(record_dir, exist_ok=True) filename = os.path.join(record_dir, f"reachy_snapshot_{int(time.time())}.jpg") # Convert to BGR for saving frame_bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) cv2.imwrite(filename, frame_bgr) return f"📸 Frame saved to: {filename}" return "❌ Capture failed. No image signal." btn_snapshot.click(fn=take_snapshot, outputs=snapshot_status) # === Camp AI 語音控制邏輯與綁定 === global camp_thread, manual_service_running camp_thread = None manual_service_running = False def start_manual_service(script_name, service_label): global manual_service_running if camp_main.CAMP_AI_RUNNING or manual_service_running: return "⚠️ Another service is currently running. Please wait or stop the current service." # Fallback handling for Guardian Mode script name (guard.py vs gurad.py) if script_name == "guard.py": script_path = os.path.join(camp_main.SCRIPT_DIR, "guard.py") if not os.path.exists(script_path): script_name = "gurad.py" manual_service_running = True stdout_redirector.clear() def run_thread(): global manual_service_running try: print(f"🎬 [Manual Start] Running {service_label} ({script_name})...", flush=True) camp_main.run_command_script(script_name) print(f"🏁 [Manual Start] {service_label} finished execution.", flush=True) except Exception as e: print(f"❌ Execution error: {e}", flush=True) finally: manual_service_running = False threading.Thread(target=run_thread, daemon=True).start() return f"🟢 Manually executing: {service_label}..." def check_default_input_device(): try: p = pyaudio.PyAudio() p.get_default_input_device_info() p.terminate() return True except Exception as e: print(f"⚠️ Failed to check microphone device: {e}") return False def start_camp_ai(): global camp_thread if camp_main.CAMP_AI_RUNNING: return "⚠️ Camp AI service is already running.", gr.update(interactive=False), gr.update(interactive=True) # Check if there is a default microphone input device if not check_default_input_device(): camp_main.CAMP_AI_RUNNING = False return "❌ Start failed: No default microphone input device detected. Please make sure the microphone is plugged in, or allow the terminal/environment to access the microphone in macOS 'System Settings' -> 'Privacy & Security'.", gr.update(interactive=True), gr.update(interactive=False) # Clear old logs stdout_redirector.clear() # Start voice monitoring thread camp_main.CAMP_AI_RUNNING = True camp_thread = threading.Thread(target=camp_main.main, daemon=True) camp_thread.start() # Start microphone waveform monitor audio_monitor.start() return "🟢 Camp AI voice service started. Listening...", gr.update(interactive=False), gr.update(interactive=True) def stop_camp_ai(): global camp_thread if not camp_main.CAMP_AI_RUNNING: return "⚠️ Camp AI service not started yet.", gr.update(interactive=True), gr.update(interactive=False) # Stop voice monitoring thread and waveform monitor camp_main.CAMP_AI_RUNNING = False audio_monitor.stop() camp_thread = None return "🔴 Camp AI voice service stopped.", gr.update(interactive=True), gr.update(interactive=False) def stream_camp_status(): while True: is_running = camp_main.CAMP_AI_RUNNING or manual_service_running or manual_voice_recorder.is_recording pulse_active = " pulse-active" if is_running else "" pulse_html = f'

🏕️ Camp AI Voice Control System

' wave_svg = audio_monitor.get_svg_wave() logs = stdout_redirector.get_text() yield pulse_html, wave_svg, logs # 若服務已停止,且波形已經歸零(所有元素均為 0),則退出生成器釋放資源 if not is_running and all(h == 0.0 for h in audio_monitor.history): break time.sleep(0.25) def start_voice_recording(): global manual_service_running if camp_main.CAMP_AI_RUNNING or manual_service_running or manual_voice_recorder.is_recording: return "⚠️ A service or recording is already running. Please wait or stop the current service.", gr.update(visible=False) manual_voice_recorder.start() return "🎙️ Recording... click [⏹️ Stop Recording & Run Dialogue] below when finished to run dialogue.", gr.update(visible=True) def stop_voice_recording_and_run(): if not manual_voice_recorder.is_recording: return "⚠️ Recording not started or already finished.", gr.update(visible=False) success = manual_voice_recorder.stop_and_save() if not success: return "❌ Recording failed or no audio captured. Please try again.", gr.update(visible=False) status_msg = start_manual_service("ask.py", "Voice Dialogue") return status_msg, gr.update(visible=False) btn_camp_start.click( fn=start_camp_ai, outputs=[camp_status, btn_camp_start, btn_camp_stop] ).then( fn=stream_camp_status, outputs=[camp_pulse_dot, camp_wave_html, camp_log_output] ) btn_camp_stop.click( fn=stop_camp_ai, outputs=[camp_status, btn_camp_start, btn_camp_stop] ) btn_location.click( fn=lambda: start_manual_service("location.py", "Location Report"), outputs=[camp_status] ).then( fn=stream_camp_status, outputs=[camp_pulse_dot, camp_wave_html, camp_log_output] ) btn_voice.click( fn=start_voice_recording, outputs=[camp_status, btn_stop_voice] ).then( fn=stream_camp_status, outputs=[camp_pulse_dot, camp_wave_html, camp_log_output] ) btn_stop_voice.click( fn=stop_voice_recording_and_run, outputs=[camp_status, btn_stop_voice] ).then( fn=stream_camp_status, outputs=[camp_pulse_dot, camp_wave_html, camp_log_output] ) btn_guardian.click( fn=lambda: start_manual_service("guard.py", "Guardian Mode"), outputs=[camp_status] ).then( fn=stream_camp_status, outputs=[camp_pulse_dot, camp_wave_html, camp_log_output] ) btn_species.click( fn=lambda: start_manual_service("look_that.py", "Species Identification"), outputs=[camp_status] ).then( fn=stream_camp_status, outputs=[camp_pulse_dot, camp_wave_html, camp_log_output] ) if __name__ == "__main__": # Launch Gradio interface demo.queue().launch(server_name="0.0.0.0", show_api=False)