import os import time import cv2 import numpy as np import gradio as gr from reachy_mini import ReachyMini from reachy_mini.utils import create_head_pose from reachy_mini.media.media_manager import MediaManager import sys import pyaudio import threading import math import main as camp_main class StdoutRedirector: def __init__(self): self.buffer = [] self.original_stdout = sys.stdout self.original_stderr = sys.stderr def write(self, message): self.original_stdout.write(message) self.original_stdout.flush() if message: self.buffer.append(message) if len(self.buffer) > 200: # limit to 200 lines self.buffer.pop(0) def flush(self): self.original_stdout.flush() def get_text(self): return "".join(self.buffer) def clear(self): self.buffer = [] def __getattr__(self, name): # 代理所有其他屬性/方法至原始的 stdout return getattr(self.original_stdout, name) stdout_redirector = StdoutRedirector() sys.stdout = stdout_redirector sys.stderr = stdout_redirector class AudioLevelMonitor: def __init__(self): self.running = False self.thread = None self.history = [0.0] * 30 self.p = None self.stream = None def start(self): if self.running: return self.running = True self.thread = threading.Thread(target=self._run, daemon=True) self.thread.start() def stop(self): self.running = False if self.thread: self.thread.join(timeout=1.0) self.thread = None self.history = [0.0] * 30 def _run(self): self.p = pyaudio.PyAudio() try: self.stream = self.p.open( format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024 ) except Exception as e: print(f"⚠️ Cannot open microphone for waveform monitoring: {e}") self.p.terminate() return while self.running: try: data = self.stream.read(1024, exception_on_overflow=False) audio_data = np.frombuffer(data, dtype=np.int16) # Calculate RMS rms = np.sqrt(np.mean(np.square(audio_data))) # Normalize to a nice range normalized = min(40.0, (rms / 200.0) * 40.0) if normalized < 2.0: normalized = 2.0 + np.random.uniform(0, 1.5) self.history.pop(0) self.history.append(normalized) except Exception: pass time.sleep(0.05) try: self.stream.stop_stream() self.stream.close() except Exception: pass self.p.terminate() def get_svg_wave(self): svg = '' return svg audio_monitor = AudioLevelMonitor() class ManualVoiceRecorder: def __init__(self): self.frames = [] self.is_recording = False self.thread = None self.p = None self.stream = None def start(self): if self.is_recording: return self.frames = [] self.is_recording = True self.thread = threading.Thread(target=self._record_loop, daemon=True) self.thread.start() def _record_loop(self): self.p = pyaudio.PyAudio() try: self.stream = self.p.open( format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024 ) except Exception as e: print(f"❌ Manual recording failed to open microphone: {e}") self.is_recording = False self.p.terminate() return while self.is_recording: try: data = self.stream.read(1024, exception_on_overflow=False) self.frames.append(data) # Synchronize audio_monitor's history update to animate the waveform audio_data = np.frombuffer(data, dtype=np.int16) rms = np.sqrt(np.mean(np.square(audio_data))) normalized = min(40.0, (rms / 200.0) * 40.0) if normalized < 2.0: normalized = 2.0 + np.random.uniform(0, 1.5) audio_monitor.history.pop(0) audio_monitor.history.append(normalized) except Exception as e: print(f"❌ Failed to read recording data: {e}") break try: self.stream.stop_stream() self.stream.close() except Exception: pass self.p.terminate() def stop_and_save(self): if not self.is_recording: return False self.is_recording = False if self.thread: self.thread.join(timeout=2.0) self.thread = None if not self.frames: print("❌ No audio recorded.") return False record_dir = os.path.join(camp_main.SCRIPT_DIR, "record") os.makedirs(record_dir, exist_ok=True) wav_filename = os.path.join(record_dir, "latest_voice.wav") mp3_filename = os.path.join(record_dir, "latest_voice.mp3") # Write WAV import wave try: wf = wave.open(wav_filename, 'wb') wf.setnchannels(1) wf.setsampwidth(2) # paInt16 is 2 bytes wf.setframerate(16000) wf.writeframes(b''.join(self.frames)) wf.close() except Exception as e: print(f"❌ Failed to write temporary WAV file: {e}") return False # 轉換為 MP3 converted = False # 嘗試使用 pydub try: from pydub import AudioSegment audio_segment = AudioSegment.from_wav(wav_filename) audio_segment.export(mp3_filename, format="mp3") converted = True except Exception: pass # 嘗試使用 ffmpeg if not converted: try: import subprocess subprocess.run( ["ffmpeg", "-y", "-i", wav_filename, "-codec:a", "libmp3lame", "-qscale:a", "2", mp3_filename], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True ) converted = True except Exception: pass # 刪除暫存 WAV 檔 if os.path.exists(wav_filename): try: os.remove(wav_filename) except Exception: pass if converted: print(f"✨ [Manual recording saved successfully] -> record/latest_voice.mp3") return True else: print("❌ Failed to convert recording to MP3.") return False manual_voice_recorder = ManualVoiceRecorder() # ----------------- Monkey Patch 處理 SSH Tunnel 遠端連線問題 ----------------- # 備份原始的 MediaManager 初始化函數,便於在 UI 中切換是否啟用 SSH 隧道模式 original_media_manager_init = MediaManager.__init__ # 建立一個 RobotManager 來管理 persistent (持久) 的 ReachyMini 連線 class RobotManager: def __init__(self): self.mini = None self.last_frame = None self.placeholder_frame = self.make_placeholder_frame("Disconnected") def make_placeholder_frame(self, message="Disconnected"): """Generate a dark placeholder frame with status message""" frame = np.zeros((480, 640, 3), dtype=np.uint8) # Use dark slate blue background frame[:, :] = [26, 28, 38] # Draw status text using OpenCV font = cv2.FONT_HERSHEY_SIMPLEX text_size = cv2.getTextSize(message, font, 1.0, 2)[0] text_x = (640 - text_size[0]) // 2 text_y = (480 + text_size[1]) // 2 cv2.putText(frame, message, (text_x, text_y), font, 1.0, (140, 140, 160), 2, cv2.LINE_AA) # Return RGB format for Gradio return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) def connect(self, host, connection_mode, media_backend, force_localhost_signalling): if self.mini is not None: return "🤖 Reachy Mini is already connected." # Apply monkey patch based on SSH Tunnel enablement if force_localhost_signalling: def mocked_media_manager_init(mm_self, *args, **kwargs): if kwargs.get("signalling_host") not in (None, "localhost", "127.0.0.1"): kwargs["signalling_host"] = "localhost" original_media_manager_init(mm_self, *args, **kwargs) MediaManager.__init__ = mocked_media_manager_init else: MediaManager.__init__ = original_media_manager_init try: # Initialize ReachyMini self.mini = ReachyMini( host=host, connection_mode=connection_mode, media_backend=media_backend ) # Invoke context manager's enter method self.mini.__enter__() # Wait for camera initialization and fetch the first frame to verify connection start_time = time.time() frame = None while frame is None and time.time() - start_time < 5.0: frame = self.mini.media.get_frame() if frame is None: time.sleep(0.1) if frame is not None: self.last_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) return "✅ Connection successful! Camera frame retrieved." else: return "⚠️ Connected, but camera module initialization timed out. No image received." except Exception as e: self.disconnect() return f"❌ Connection failed: {str(e)}" def disconnect(self): if self.mini is not None: try: self.mini.__exit__(None, None, None) except Exception as e: print(f"Error during disconnect exit: {e}") self.mini = None self.last_frame = None return "🔌 Disconnected." def get_frame(self): if self.mini is None: return self.placeholder_frame try: frame = self.mini.media.get_frame() if frame is not None: frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) self.last_frame = frame_rgb return frame_rgb elif self.last_frame is not None: return self.last_frame else: return self.make_placeholder_frame("Waiting for camera frame...") except Exception as e: print(f"Error fetching frame: {e}") return self.make_placeholder_frame("Camera connection lost") def get_frame_stream(self): while True: if self.mini is None: yield self.placeholder_frame time.sleep(0.5) else: try: frame = self.mini.media.get_frame() if frame is not None: frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) self.last_frame = frame_rgb yield frame_rgb elif self.last_frame is not None: yield self.last_frame else: yield self.make_placeholder_frame("Waiting for camera frame...") except Exception as e: print(f"Error fetching frame in stream: {e}") yield self.make_placeholder_frame("Camera connection lost") time.sleep(0.1) def move_head(self, yaw, pitch, roll, antennas_left, antennas_right, body_yaw): if self.mini is None: return "❌ Please connect the robot before controlling movements." try: head_pose = create_head_pose( yaw=yaw, pitch=pitch, roll=roll, degrees=True, mm=True ) antennas = np.deg2rad([antennas_left, antennas_right]).tolist() body_yaw_rad = np.deg2rad(body_yaw) self.mini.goto_target( head=head_pose, antennas=antennas, body_yaw=body_yaw_rad, duration=1.5, method="minjerk" ) return "✅ Movement command sent successfully!" except Exception as e: return f"❌ Movement control failed: {str(e)}" def set_motors_torque(self, enable=True): if self.mini is None: return "❌ Robot not connected yet." try: if enable: self.mini.enable_motors() return "✅ Motor torque enabled (Torque ON)" else: self.mini.disable_motors() return "✅ Motor torque disabled (Torque OFF - Robot relaxed)" except Exception as e: return f"❌ Motor setting failed: {str(e)}" # 實例化管理器 robot_manager = RobotManager() # ----------------- Gradio UI 建立 ----------------- custom_css = """ body { background-color: #0c0d14; color: #f3f4f6; font-family: 'Outfit', 'Inter', sans-serif; } .gradio-container { background-color: #0c0d14 !important; } .header-box { text-align: center; padding: 30px; background: linear-gradient(135deg, #1e1e38 0%, #0f0f1b 100%); border-radius: 16px; box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4); border: 1px solid rgba(255, 255, 255, 0.1); margin-bottom: 24px; } .header-title { background: linear-gradient(90deg, #6366f1, #06b6d4); -webkit-background-clip: text; -webkit-text-fill-color: transparent; font-size: 2.2rem !important; font-weight: 800 !important; margin: 0; } .header-subtitle { color: #9ca3af; font-size: 1rem; margin-top: 8px; } .custom-card { background-color: #111322 !important; border: 1px solid rgba(255, 255, 255, 0.05) !important; border-radius: 12px !important; padding: 16px !important; } .pulse-dot { width: 12px; height: 12px; background-color: #ef4444; border-radius: 50%; display: inline-block; } .pulse-active { background-color: #10b981 !important; animation: pulse 1.5s infinite; } @keyframes pulse { 0% { transform: scale(0.95); box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); } 70% { transform: scale(1.05); box-shadow: 0 0 0 8px rgba(16, 185, 129, 0); } 100% { transform: scale(0.95); box-shadow: 0 0 0 0 rgba(16, 185, 129, 0); } } .manual-btn { background: linear-gradient(135deg, #1b1c2b 0%, #131422 100%) !important; border: 1px solid rgba(255, 255, 255, 0.08) !important; color: #e2e8f0 !important; transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important; font-weight: 600 !important; border-radius: 10px !important; box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1) !important; padding: 10px 14px !important; text-align: center !important; } .manual-btn:hover { background: linear-gradient(135deg, #312e81 0%, #1e1b4b 100%) !important; border-color: #6366f1 !important; box-shadow: 0 0 15px rgba(99, 102, 241, 0.4) !important; transform: translateY(-2px) !important; color: #ffffff !important; } .manual-btn:active { transform: translateY(1px) !important; } .manual-group { background-color: rgba(255, 255, 255, 0.01) !important; border: 1px solid rgba(255, 255, 255, 0.05) !important; border-radius: 14px !important; padding: 16px !important; margin-top: 12px !important; margin-bottom: 12px !important; } """ with gr.Blocks(title="Reachy Mini Camping Smart Assistant & Control Panel", css=custom_css) as demo: # Header Section gr.HTML("""
Integrating multimodal vision recognition (MiniCPM-V), smart voice dialogue (Whisper/Gemini/ChatTTS), security guard patrol, and geolocation reports for a comprehensive smart camping assistant experience
You need to connect to Reachy via Reachy Mini Control first.
⚡ Manual Mode
") with gr.Row(): btn_location = gr.Button("📍 Location Report", variant="secondary", elem_classes=["manual-btn"]) btn_voice = gr.Button("💬 Voice Dialogue", variant="secondary", elem_classes=["manual-btn"]) with gr.Row(): btn_guardian = gr.Button("🛡️ Guardian Mode", variant="secondary", elem_classes=["manual-btn"]) btn_species = gr.Button("🔍 Species Identification", variant="secondary", elem_classes=["manual-btn"]) btn_stop_voice = gr.Button("⏹️ Stop Recording & Run Dialogue", variant="stop", visible=False) camp_status = gr.Textbox(label="Voice Service Status", value="Not Started", interactive=False) gr.Markdown("🔊 **Microphone Input Waveform**") camp_wave_html = gr.HTML(value=audio_monitor.get_svg_wave()) camp_log_output = gr.Textbox( label="Camp AI Execution Logs (stdout)", value="Voice service not started yet.", lines=8, max_lines=12, interactive=False ) # --- 互動邏輯綁定 --- # Connect and disconnect callbacks def on_connect(host, conn, backend, tunnel): res = robot_manager.connect(host, conn, backend, tunnel) print(res) # When connection is successful, set the pulse dot indicator to active green if "successful" in res.lower(): return gr.update(value='