commit

app.py

@@ -1,12 +1,18 @@
-# app.py – Prüfungsrechts-Chatbot (RAG + Sprache, UI kiểu ChatGPT) với các tính năng nâng cao
-#
+# app.py – Prüfungsrechts-Chatbot với OpenAI Realtime API và Voice Agents
 import os
 import time
+import json
+import asyncio
+import threading
 from dataclasses import dataclass, field
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, List
 import gradio as gr
 from gradio_pdf import PDF
 import numpy as np
+import queue
+
+from openai import OpenAI
+from openai._streaming import AssistantEventHandler
 
 from load_documents import load_all_documents
 from split_documents import split_documents
@@ -14,30 +20,29 @@ from vectorstore import build_vectorstore
 from retriever import get_retriever
 from llm import load_llm
 from rag_pipeline import answer
-from speech_io import transcribe_audio, synthesize_speech, transcribe_with_openai, detect_voice_activity
 
-# Cấu hình môi trường
-ASR_LANGUAGE_HINT = os.getenv("ASR_LANGUAGE", "de")
-USE_OPENAI = os.getenv("USE_OPENAI", "false").lower() == "true"
-USE_REALTIME = os.getenv("USE_REALTIME", "false").lower() == "true"
-REALTIME_SERVER_URL = os.getenv("REALTIME_SERVER_URL", "ws://localhost:8000/ws")
-ENABLE_VAD = os.getenv("ENABLE_VAD", "true").lower() == "true"
-VAD_THRESHOLD = float(os.getenv("VAD_THRESHOLD", "0.3"))
+# =====================================================
+# CONFIGURATION
+# =====================================================
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+if not OPENAI_API_KEY:
+    raise RuntimeError("OPENAI_API_KEY is required for Realtime API")
+
+# Initialize OpenAI client
+openai_client = OpenAI(api_key=OPENAI_API_KEY)
 
 # =====================================================
-# STATE MANAGEMENT - Quản lý trạng thái hội thoại liền mạch
+# STATE MANAGEMENT
 # =====================================================
 @dataclass
 class ConversationState:
     """Quản lý trạng thái hội thoại"""
     messages: list = field(default_factory=list)
-    last_audio_time: float = field(default_factory=time.time)
-    is_listening: bool = False
-    vad_confidence: float = 0.0
+    is_streaming: bool = False
+    realtime_session_id: Optional[str] = None
+    audio_queue: queue.Queue = field(default_factory=queue.Queue)
+    text_queue: queue.Queue = field(default_factory=queue.Queue)
     conversation_context: str = ""
-    whisper_model: str = field(default_factory=lambda: os.getenv("WHISPER_MODEL", "base"))
-    language: str = field(default_factory=lambda: ASR_LANGUAGE_HINT)
-    current_audio_path: Optional[str] = None
     
     def add_message(self, role: str, content: str):
         """Thêm message vào hội thoại"""
@@ -62,32 +67,26 @@ class ConversationState:
         context_parts = []
         for msg in self.messages[-5:]:  # Giữ 5 message gần nhất
             prefix = "User" if msg["role"] == "user" else "Assistant"
-            context_parts.append(f"{prefix}: {msg['content'][:200]}")  # Giới hạn độ dài
+            context_parts.append(f"{prefix}: {msg['content'][:200]}")
         self.conversation_context = "\n".join(context_parts)
     
-    def get_recent_context(self, num_messages: int = 3) -> str:
-        """Lấy context gần đây"""
-        if not self.messages or num_messages <= 0:
-            return ""
-        
-        recent = self.messages[-num_messages:] if len(self.messages) >= num_messages else self.messages
-        return "\n".join([f"{m['role']}: {m['content']}" for m in recent])
-    
     def reset(self):
         """Reset trạng thái hội thoại"""
         self.messages = []
         self.conversation_context = ""
-        self.is_listening = False
-        self.vad_confidence = 0.0
-        self.current_audio_path = None
+        self.is_streaming = False
+        self.realtime_session_id = None
+        while not self.audio_queue.empty():
+            self.audio_queue.get()
+        while not self.text_queue.empty():
+            self.text_queue.get()
 
 # Khởi tạo state
 state = ConversationState()
 
 # =====================================================
-# INITIALISIERUNG (global)
+# INITIALIZATION - RAG Components
 # =====================================================
-
 print("📚 Lade Dokumente…")
 docs = load_all_documents()
 
@@ -109,70 +108,400 @@ hg_meta = next(d.metadata for d in docs if d.metadata.get("type") == "hg")
 hg_url = hg_meta.get("viewer_url")
 
 # =====================================================
-# VOICE ACTIVITY DETECTION
+# OPENAI REALTIME API HANDLERS
 # =====================================================
-def handle_voice_activity(audio_data: Optional[np.ndarray], sample_rate: int) -> Dict[str, Any]:
-    """Xử lý phát hiện hoạt động giọng nói"""
-    if audio_data is None or len(audio_data) == 0:
-        return {"is_speech": False, "confidence": 0.0, "status": "No audio data"}
+class RealtimeEventHandler(AssistantEventHandler):
+    """Handler cho OpenAI Realtime API events"""
     
+    def __init__(self, state_ref):
+        super().__init__()
+        self.state = state_ref
+        self.current_text = ""
+        
+    def on_text_delta(self, delta, snapshot):
+        """Xử lý text delta từ Realtime API"""
+        if delta.value:
+            self.current_text += delta.value
+            # Thêm vào text queue để hiển thị
+            self.state.text_queue.put({
+                "type": "text_delta",
+                "content": delta.value
+            })
+    
+    def on_audio_transcript_delta(self, delta, snapshot):
+        """Xử lý audio transcript từ Realtime API"""
+        if delta.text:
+            # Thêm vào text queue
+            self.state.text_queue.put({
+                "type": "transcript",
+                "content": delta.text
+            })
+    
+    def on_audio_delta(self, delta, snapshot):
+        """Xử lý audio data từ Realtime API"""
+        if delta.data:
+            # Thêm vào audio queue để phát
+            self.state.audio_queue.put({
+                "type": "audio",
+                "data": delta.data
+            })
+    
+    def on_response_created(self, response):
+        """Khi response được tạo"""
+        print("DEBUG: Response created")
+    
+    def on_response_done(self, response):
+        """Khi response hoàn thành"""
+        print(f"DEBUG: Response done, final text: {self.current_text[:100]}...")
+        if self.current_text:
+            # Thêm message vào history
+            self.state.add_message("assistant", self.current_text)
+            
+            # Signal end of response
+            self.state.text_queue.put({
+                "type": "response_end",
+                "content": self.current_text
+            })
+        
+        self.current_text = ""
+        self.state.is_streaming = False
+    
+    def on_error(self, error):
+        """Xử lý lỗi"""
+        print(f"DEBUG: Realtime API error: {error}")
+        self.state.is_streaming = False
+        self.state.text_queue.put({
+            "type": "error",
+            "content": f"Error: {str(error)}"
+        })
+
+def start_realtime_conversation():
+    """Bắt đầu cuộc hội thoại Realtime API"""
     try:
-        vad_result = detect_voice_activity(audio_data, sample_rate, threshold=VAD_THRESHOLD)
+        # Tạo Realtime session
+        session = openai_client.realtime.sessions.create(
+            model="gpt-4o-realtime-preview",
+            voice="shimmer",  # Có thể chọn: alloy, echo, fable, onyx, nova, shimmer
+            modalities=["text", "audio"],
+            instructions="""Du bist ein juristischer Assistent für Prüfungsrecht.
+            Du hilfst Studenten mit Fragen zu Prüfungsordnung und Hochschulgesetz NRW.
+            Antworte präzise, freundlich und professionell.
+            Bei unsicheren Fragen, verweise auf die offiziellen Dokumente."""
+        )
         
-        # Cập nhật state
-        state.is_listening = vad_result["is_speech"]
-        if vad_result["is_speech"]:
-            state.last_audio_time = time.time()
-            state.vad_confidence = vad_result["confidence"]
+        state.realtime_session_id = session.id
+        state.is_streaming = True
         
-        return {
-            "is_speech": vad_result["is_speech"],
-            "confidence": vad_result["confidence"],
-            "status": f"Speech detected: {vad_result['is_speech']} (conf: {vad_result['confidence']:.2f})"
-        }
+        print(f"DEBUG: Realtime session started: {session.id}")
+        
+        # Bắt đầu streaming
+        with openai_client.realtime.connect(
+            session_id=session.id,
+            event_handler=RealtimeEventHandler(state)
+        ) as connection:
+            # Keep connection alive
+            while state.is_streaming:
+                time.sleep(0.1)
+                
+    except Exception as e:
+        print(f"DEBUG: Error in realtime conversation: {e}")
+        state.is_streaming = False
+
+def stop_realtime_conversation():
+    """Dừng cuộc hội thoại Realtime"""
+    state.is_streaming = False
+    if state.realtime_session_id:
+        try:
+            openai_client.realtime.sessions.delete(state.realtime_session_id)
+        except:
+            pass
+        state.realtime_session_id = None
+
+def send_text_to_realtime(text: str):
+    """Gửi text đến Realtime API"""
+    if not state.is_streaming or not state.realtime_session_id:
+        return False
+    
+    try:
+        # Tạo Realtime client mới để gửi message
+        with openai_client.realtime.connect(session_id=state.realtime_session_id) as connection:
+            connection.send({
+                "type": "response.create",
+                "response": {
+                    "modalities": ["text", "audio"],
+                    "instructions": f"Antworte auf: {text}"
+                }
+            })
+        return True
     except Exception as e:
-        print(f"VAD error: {e}")
-        return {"is_speech": False, "confidence": 0.0, "status": f"VAD error: {e}"}
+        print(f"DEBUG: Error sending to realtime: {e}")
+        return False
 
 # =====================================================
-# TRANSCRIBE WITH OPTIMIZED PIPELINE
+# TOOLS (FUNCTION CALLS) FOR VOICE AGENT
 # =====================================================
-def transcribe_audio_optimized(audio_path: str, language: Optional[str] = None) -> str:
-    """Transcribe audio với pipeline tối ưu"""
-    if not audio_path or not os.path.exists(audio_path):
-        return ""
+def search_documents_tool(query: str) -> Dict[str, Any]:
+    """Tool để tìm kiếm tài liệu"""
+    try:
+        # Sử dụng retriever để tìm tài liệu liên quan
+        docs = retriever.invoke(query)
+        
+        if not docs:
+            return {
+                "success": False,
+                "message": "Keine relevanten Dokumente gefunden.",
+                "documents": []
+            }
+        
+        # Format kết quả
+        results = []
+        for i, doc in enumerate(docs[:3], 1):
+            meta = doc.metadata
+            source_type = meta.get("type", "unknown")
+            
+            if source_type == "pdf":
+                source_info = {
+                    "type": "Prüfungsordnung",
+                    "page": meta.get("page"),
+                    "url": meta.get("pdf_url")
+                }
+            elif source_type == "hg":
+                source_info = {
+                    "type": "Hochschulgesetz NRW",
+                    "paragraph": meta.get("title"),
+                    "url": meta.get("viewer_url")
+                }
+            else:
+                source_info = {"type": "unknown"}
+            
+            results.append({
+                "id": i,
+                "content": doc.page_content[:500] + "...",
+                "source": source_info
+            })
+        
+        return {
+            "success": True,
+            "message": f"{len(results)} Dokumente gefunden",
+            "documents": results
+        }
         
-    if USE_OPENAI:
-        return transcribe_with_openai(audio_path, language=language)
-    return transcribe_audio(audio_path, language=language)
+    except Exception as e:
+        return {
+            "success": False,
+            "message": f"Fehler bei der Suche: {str(e)}",
+            "documents": []
+        }
+
+def get_legal_advice_tool(question: str) -> Dict[str, Any]:
+    """Tool để nhận tư vấn pháp lý từ RAG"""
+    try:
+        # Sử dụng RAG pipeline
+        ans, sources = answer(question, retriever, llm)
+        
+        # Format sources
+        formatted_sources = []
+        for src in sources:
+            formatted_sources.append({
+                "source": src["source"],
+                "page": src.get("page"),
+                "url": src["url"]
+            })
+        
+        return {
+            "success": True,
+            "answer": ans,
+            "sources": formatted_sources,
+            "has_relevant_info": len(sources) > 0
+        }
+        
+    except Exception as e:
+        return {
+            "success": False,
+            "answer": f"Fehler: {str(e)}",
+            "sources": [],
+            "has_relevant_info": False
+        }
 
 # =====================================================
-# CONVERSATIONAL INTELLIGENCE
+# VOICE AGENT WITH TOOLS
 # =====================================================
-def enhance_conversation_context(user_input: str, history: list) -> str:
-    """Tăng cường context hội thoại"""
-    if not user_input:
-        return user_input
-    
-    # Thêm context đơn giản từ history
-    if history and len(history) > 0:
-        # Lấy 3 tin nhắn gần nhất từ history
-        recent_history = history[-3:] if len(history) >= 3 else history
-        context_parts = ["Previous conversation:"]
-        for msg in recent_history:
-            role = "User" if msg.get("role") == "user" else "Assistant"
-            content = msg.get("content", "")[:100]  # Giới hạn độ dài
-            context_parts.append(f"{role}: {content}")
+class VoiceAgent:
+    """Voice Agent sử dụng OpenAI Realtime API với Tools"""
+    
+    def __init__(self, openai_client):
+        self.client = openai_client
+        self.session_id = None
+        self.is_active = False
         
-        context = "\n".join(context_parts)
-        return f"{context}\n\nCurrent question: {user_input}"
+    def start_session(self):
+        """Bắt đầu session với tools"""
+        try:
+            # Tạo session với tools definition
+            session = self.client.realtime.sessions.create(
+                model="gpt-4o-realtime-preview-2024-12-17",
+                voice="shimmer",
+                modalities=["text", "audio"],
+                instructions="""Du bist ein juristischer Voice Agent.
+                Du kannst:
+                1. Dokumente durchsuchen (search_documents)
+                2. Rechtliche Beratung geben (get_legal_advice)
+                
+                Sei präzise, freundlich und hilfreich.
+                Verweise immer auf die Quellen.""",
+                tools=[
+                    {
+                        "type": "function",
+                        "name": "search_documents",
+                        "description": "Durchsucht die Prüfungsordnung und das Hochschulgesetz nach relevanten Informationen",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "query": {
+                                    "type": "string",
+                                    "description": "Suchbegriff oder Frage"
+                                }
+                            },
+                            "required": ["query"]
+                        }
+                    },
+                    {
+                        "type": "function",
+                        "name": "get_legal_advice",
+                        "description": "Gibt juristische Beratung basierend auf den Dokumenten",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "question": {
+                                    "type": "string",
+                                    "description": "Juristische Frage"
+                                }
+                            },
+                            "required": ["question"]
+                        }
+                    }
+                ],
+                tool_choice="auto"
+            )
+            
+            self.session_id = session.id
+            self.is_active = True
+            
+            # Start event handling thread
+            threading.Thread(target=self._handle_events, daemon=True).start()
+            
+            return True
+            
+        except Exception as e:
+            print(f"DEBUG: Error starting voice agent: {e}")
+            return False
+    
+    def _handle_events(self):
+        """Xử lý events từ Realtime API"""
+        try:
+            with self.client.realtime.connect(
+                session_id=self.session_id,
+                event_handler=VoiceAgentEventHandler(self)
+            ) as connection:
+                
+                while self.is_active:
+                    time.sleep(0.1)
+                    
+        except Exception as e:
+            print(f"DEBUG: Error in event handler: {e}")
+            self.is_active = False
+    
+    def stop_session(self):
+        """Dừng session"""
+        self.is_active = False
+        if self.session_id:
+            try:
+                self.client.realtime.sessions.delete(self.session_id)
+            except:
+                pass
+            self.session_id = None
+    
+    def process_tool_call(self, tool_name: str, arguments: Dict) -> Dict:
+        """Xử lý tool calls"""
+        try:
+            if tool_name == "search_documents":
+                query = arguments.get("query", "")
+                return search_documents_tool(query)
+                
+            elif tool_name == "get_legal_advice":
+                question = arguments.get("question", "")
+                return get_legal_advice_tool(question)
+                
+            else:
+                return {
+                    "success": False,
+                    "message": f"Unbekanntes Tool: {tool_name}"
+                }
+                
+        except Exception as e:
+            return {
+                "success": False,
+                "message": f"Tool Fehler: {str(e)}"
+            }
+
+class VoiceAgentEventHandler(AssistantEventHandler):
+    """Event handler cho Voice Agent"""
+    
+    def __init__(self, agent):
+        super().__init__()
+        self.agent = agent
+        self.current_text = ""
+    
+    def on_text_delta(self, delta, snapshot):
+        """Xử lý text delta"""
+        if delta.value:
+            self.current_text += delta.value
+            # Thêm vào state text queue
+            state.text_queue.put({
+                "type": "agent_text",
+                "content": delta.value
+            })
+    
+    def on_function_call_arguments_delta(self, delta, snapshot):
+        """Xử lý function call arguments"""
+        print(f"DEBUG: Function call arguments: {delta}")
+    
+    def on_function_call_done(self, function_call, snapshot):
+        """Khi function call hoàn thành"""
+        try:
+            tool_name = function_call.name
+            arguments = json.loads(function_call.arguments)
+            
+            print(f"DEBUG: Processing tool call: {tool_name}, args: {arguments}")
+            
+            # Process tool call
+            result = self.agent.process_tool_call(tool_name, arguments)
+            
+            # Gửi kết quả trở lại
+            with openai_client.realtime.connect(session_id=self.agent.session_id) as conn:
+                conn.send({
+                    "type": "response.function_call_arguments",
+                    "function_call_id": function_call.id,
+                    "output": json.dumps(result)
+                })
+                
+        except Exception as e:
+            print(f"DEBUG: Error processing function call: {e}")
     
-    return user_input
+    def on_response_done(self, response):
+        """Khi response hoàn thành"""
+        if self.current_text:
+            state.add_message("assistant", self.current_text)
+        self.current_text = ""
+
+# Khởi tạo Voice Agent
+voice_agent = VoiceAgent(openai_client)
 
 # =====================================================
-# Quellen formatieren – Markdown für Chat
+# GRADIO UI COMPONENTS
 # =====================================================
 def format_sources(src):
+    """Format sources cho display"""
     if not src:
         return ""
 
@@ -186,232 +515,60 @@ def format_sources(src):
 
     return "\n".join(out)
 
-# =====================================================
-# CORE CHAT-FUNKTION với tất cả tính năng mới
-# =====================================================
-def chat_fn(text_input, audio_path, history, lang_sel, use_vad):
-    """
-    Main chat function với xử lý VAD và transcription
-    """
-    print(f"DEBUG: chat_fn called - text_input: '{text_input}', audio_path: {audio_path}, history length: {len(history) if history else 0}")
-    
-    # Khởi tạo history nếu None
-    if history is None:
+def update_chat_display(history, new_text=""):
+    """Cập nhật chat display với streaming text"""
+    if not history:
         history = []
     
-    # Biến để lưu text cần xử lý
-    text_to_process = ""
-    
-    # Xử lý audio nếu có
-    if audio_path and os.path.exists(audio_path):
-        print(f"DEBUG: Processing audio file: {audio_path}")
-        
-        # Lưu đường dẫn audio vào state
-        state.current_audio_path = audio_path
-        
-        # Kiểm tra VAD nếu được bật
-        if use_vad and ENABLE_VAD:
-            try:
-                import soundfile as sf
-                audio_data, sample_rate = sf.read(audio_path)
-                print(f"DEBUG: Audio loaded - shape: {audio_data.shape}, sample_rate: {sample_rate}")
-                
-                vad_result = handle_voice_activity(audio_data, sample_rate)
-                print(f"DEBUG: VAD result: {vad_result}")
-                
-                # Nếu VAD phát hiện có giọng nói, hoặc nếu VAD không bật, tiến hành transcribe
-                if vad_result.get("is_speech", True):
-                    # Transcribe audio
-                    transcribed_text = transcribe_audio_optimized(audio_path, language=lang_sel)
-                    if transcribed_text and transcribed_text.strip():
-                        text_to_process = transcribed_text.strip()
-                        print(f"DEBUG: Transcribed text: {text_to_process}")
-                else:
-                    print("DEBUG: VAD detected no speech, skipping transcription")
-            except Exception as e:
-                print(f"DEBUG: Error in VAD/transcription: {e}")
-                # Fallback: transcribe ngay cả khi có lỗi
-                transcribed_text = transcribe_audio_optimized(audio_path, language=lang_sel)
-                if transcribed_text and transcribed_text.strip():
-                    text_to_process = transcribed_text.strip()
+    if new_text:
+        # Nếu last message là của assistant, append text
+        if history and history[-1]["role"] == "assistant":
+            history[-1]["content"] += new_text
         else:
-            # Nếu VAD không bật, transcribe trực tiếp
-            transcribed_text = transcribe_audio_optimized(audio_path, language=lang_sel)
-            if transcribed_text and transcribed_text.strip():
-                text_to_process = transcribed_text.strip()
-                print(f"DEBUG: Transcribed text (no VAD): {text_to_process}")
-    
-    # Nếu có text input từ textbox, ưu tiên sử dụng nó
-    if text_input and text_input.strip():
-        text_to_process = text_input.strip()
-        print(f"DEBUG: Using text input: {text_to_process}")
-    
-    # Nếu không có gì để xử lý
-    if not text_to_process:
-        print("DEBUG: No text to process")
-        # Trả về history hiện tại và status
-        status_text = f"Bereit | VAD: {'On' if use_vad and ENABLE_VAD else 'Off'} | Model: {state.whisper_model}"
-        if history is None:
-            history = []
-        return history, "", None, status_text
-    
-    print(f"DEBUG: Processing text: {text_to_process}")
-    
-    # Tăng cường context cho câu hỏi
-    enhanced_question = enhance_conversation_context(text_to_process, history)
+            history.append({"role": "assistant", "content": new_text})
     
-    try:
-        # RAG-Antwort berechnen
-        ans, sources = answer(enhanced_question, retriever, llm)
-        bot_msg = ans + format_sources(sources)
-        
-        # Thêm vào state
-        state.add_message("user", text_to_process)
-        state.add_message("assistant", ans)
-        
-        # History aktualisieren (ChatGPT-Style)
-        history.append({"role": "user", "content": text_to_process})
-        history.append({"role": "assistant", "content": bot_msg})
-        
-        print(f"DEBUG: Answer generated, history length: {len(history)}")
-        
-    except Exception as e:
-        print(f"DEBUG: Error in RAG pipeline: {e}")
-        # Fallback response
-        error_msg = "Entschuldigung, es gab einen Fehler bei der Verarbeitung Ihrer Anfrage. Bitte versuchen Sie es erneut."
-        history.append({"role": "user", "content": text_to_process})
-        history.append({"role": "assistant", "content": error_msg})
-    
-    status_text = f"Bereit | VAD: {'On' if use_vad and ENABLE_VAD else 'Off'} | Model: {state.whisper_model}"
-    return history, "", None, status_text
-
-# =====================================================
-# FUNCTIONS FOR UI CONTROLS
-# =====================================================
-def toggle_vad(use_vad):
-    """Toggle Voice Activity Detection"""
-    global ENABLE_VAD
-    ENABLE_VAD = use_vad
-    status = "EIN" if use_vad else "AUS"
-    return f"Voice Activity Detection: {status} | Model: {state.whisper_model}"
-
-def change_whisper_model(model_size):
-    """Đổi Whisper model"""
-    state.whisper_model = model_size
-    os.environ["WHISPER_MODEL"] = model_size
-    return f"Whisper Model: {model_size} | VAD: {'On' if ENABLE_VAD else 'Off'}"
-
-def clear_conversation():
-    """Xóa hội thoại"""
-    state.reset()
-    return [], "Konversation gelöscht | Bereit"
-
-def update_vad_indicator():
-    """Cập nhật VAD indicator"""
-    if state.is_listening:
-        indicator_html = """
-        <div style="display: flex; align-items: center; gap: 8px;">
-            <div style="width: 12px; height: 12px; border-radius: 50%; background-color: #10b981; box-shadow: 0 0 10px #10b981; animation: pulse 1.5s infinite;"></div>
-            <span style="color: #10b981; font-weight: bold;">Sprache erkannt</span>
-        </div>
-        <style>
-        @keyframes pulse {
-            0% { opacity: 0.7; }
-            50% { opacity: 1; }
-            100% { opacity: 0.7; }
-        }
-        </style>
-        """
-    else:
-        indicator_html = """
-        <div style="display: flex; align-items: center; gap: 8px;">
-            <div style="width: 12px; height: 12px; border-radius: 50%; background-color: #6b7280;"></div>
-            <span>Bereit</span>
-        </div>
-        """
-    
-    return indicator_html
+    return history
 
-# =====================================================
-# AUDIO STREAMING HANDLER
-# =====================================================
-def handle_audio_stream(audio_path, use_vad):
-    """Xử lý audio streaming real-time"""
-    if not audio_path or not os.path.exists(audio_path):
-        return "", update_vad_indicator(), "Keine Audiodatei"
+def process_queue_updates():
+    """Process queue updates cho streaming"""
+    updates = []
     
-    try:
-        import soundfile as sf
-        audio_data, sample_rate = sf.read(audio_path)
-        
-        # Cập nhật VAD indicator
-        vad_html = update_vad_indicator()
-        
-        if use_vad and ENABLE_VAD:
-            vad_result = handle_voice_activity(audio_data, sample_rate)
-            
-            if vad_result.get("is_speech", False):
-                # Nếu phát hiện giọng nói, transcribe
-                text = transcribe_audio_optimized(audio_path, language=state.language)
-                status = f"Sprache erkannt ({vad_result.get('confidence', 0):.2f})"
-                return text, vad_html, status
-            else:
-                status = "Keine Sprache erkannt"
-                return "", vad_html, status
-        else:
-            # Nếu VAD không bật, vẫn transcribe nhưng hiển thị trạng thái khác
-            text = transcribe_audio_optimized(audio_path, language=state.language)
-            status = "Transkription (VAD aus)"
-            return text, vad_html, status
-            
-    except Exception as e:
-        print(f"Error in audio stream handler: {e}")
-        return "", update_vad_indicator(), f"Fehler: {str(e)[:50]}"
-
-# =====================================================
-# TTS FUNCTION
-# =====================================================
-def read_last_answer(history):
-    """Đọc câu trả lời cuối cùng"""
-    if not history:
-        print("DEBUG: No history for TTS")
-        return None
-    
-    # Tìm câu trả lời cuối cùng của assistant
-    for msg in reversed(history):
-        if isinstance(msg, dict) and msg.get("role") == "assistant":
-            content = msg.get("content", "")
-            # Loại bỏ phần sources từ câu trả lời
-            if "## 📚 Quellen" in content:
-                content = content.split("## 📚 Quellen")[0].strip()
-            
-            print(f"DEBUG: Synthesizing speech for: {content[:100]}...")
-            audio_result = synthesize_speech(content)
-            if audio_result:
-                print("DEBUG: TTS successful")
-                return audio_result
-    
-    print("DEBUG: No assistant message found for TTS")
-    return None
+    # Process text queue
+    while not state.text_queue.empty():
+        try:
+            item = state.text_queue.get_nowait()
+            updates.append(("text", item.get("content", "")))
+        except queue.Empty:
+            break
+    
+    # Process audio queue (simplified - trong thực tế cần xử lý audio)
+    while not state.audio_queue.empty():
+        try:
+            item = state.audio_queue.get_nowait()
+            # Có thể xử lý audio data ở đây
+            pass
+        except queue.Empty:
+            break
+    
+    return updates
 
 # =====================================================
-# UI – GRADIO với tất cả tính năng mới
+# UI – GRADIO INTERFACE
 # =====================================================
-with gr.Blocks(title="Prüfungsrechts-Chatbot (RAG + Sprache) - Enhanced") as demo:
-    # CSS Styling nâng cao
+with gr.Blocks(title="Prüfungsrechts-Chatbot mit OpenAI Realtime API", theme=gr.themes.Soft()) as demo:
+    # CSS Styling
     gr.HTML("""
     <style>
     .gradio-container {
         max-width: 1200px;
         margin: 0 auto;
-        padding: 20px;
         font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
     }
     
     .header {
         text-align: center;
         margin-bottom: 30px;
-        padding: 20px;
+        padding: 25px;
         background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
         border-radius: 15px;
         color: white;
@@ -425,306 +582,278 @@ with gr.Blocks(title="Prüfungsrechts-Chatbot (RAG + Sprache) - Enhanced") as de
         border: 1px solid #e2e8f0;
     }
     
-    .chat-container {
-        background: white;
-        border-radius: 15px;
-        padding: 20px;
-        box-shadow: 0 4px 20px rgba(0,0,0,0.1);
-        margin-bottom: 20px;
+    .status-indicator {
+        padding: 10px 15px;
+        border-radius: 10px;
+        font-weight: 500;
+        display: inline-flex;
+        align-items: center;
+        gap: 8px;
     }
     
-    .input-row {
-        background: #f8fafc;
-        border-radius: 25px;
-        padding: 10px 20px;
-        border: 2px solid #e2e8f0;
-        transition: all 0.3s ease;
-        display: flex;
-        align-items: center;
-        gap: 10px;
+    .status-active {
+        background: #d1fae5;
+        color: #065f46;
     }
     
-    .input-row:focus-within {
-        border-color: #667eea;
-        box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1);
+    .status-inactive {
+        background: #f3f4f6;
+        color: #6b7280;
     }
     
-    .send-btn {
-        background: linear-gradient(135deg, #667eea 0%, #764ba2 100%) !important;
-        color: white !important;
-        border: none !important;
-        border-radius: 50% !important;
-        width: 44px !important;
-        height: 44px !important;
-        display: flex !important;
-        align-items: center !important;
-        justify-content: center !important;
-        cursor: pointer !important;
+    .voice-btn {
+        padding: 12px 24px;
+        border-radius: 25px;
+        font-weight: 600;
+        transition: all 0.3s;
+        border: none;
     }
     
-    .send-btn:hover {
-        transform: scale(1.05);
-        box-shadow: 0 4px 15px rgba(102, 126, 234, 0.4) !important;
+    .voice-btn-start {
+        background: linear-gradient(135deg, #10b981 0%, #059669 100%);
+        color: white;
     }
     
-    .vad-indicator-container {
-        padding: 10px;
-        background: #f1f5f9;
-        border-radius: 10px;
-        margin: 10px 0;
-        display: flex;
-        align-items: center;
-        gap: 10px;
+    .voice-btn-stop {
+        background: linear-gradient(135deg, #ef4444 0%, #dc2626 100%);
+        color: white;
     }
     
-    .feature-badge {
-        display: inline-block;
-        padding: 4px 12px;
-        background: #e0e7ff;
-        color: #4f46e5;
-        border-radius: 20px;
-        font-size: 12px;
-        font-weight: 500;
-        margin: 2px;
+    .chat-message {
+        padding: 15px;
+        border-radius: 15px;
+        margin: 10px 0;
+        max-width: 85%;
     }
     
-    .chatbot {
-        min-height: 400px;
-        max-height: 500px;
-        overflow-y: auto;
+    .user-message {
+        background: #e0e7ff;
+        margin-left: auto;
     }
     
-    /* Responsive design */
-    @media (max-width: 768px) {
-        .gradio-container {
-            padding: 10px;
-        }
-        
-        .input-row {
-            flex-direction: column;
-            gap: 10px;
-        }
-        
-        .send-btn {
-            width: 100% !important;
-            height: 44px !important;
-            border-radius: 10px !important;
-        }
+    .assistant-message {
+        background: #f3f4f6;
+        margin-right: auto;
     }
     </style>
     """)
     
     # Header
     with gr.Column(elem_classes=["header"]):
-        gr.Markdown("# 🧑‍⚖️ Prüfungsrechts-Chatbot")
-        gr.Markdown("### Intelligent Voice Interface with Advanced Features")
-        
-        # Feature badges
-        gr.HTML("""
-        <div style="text-align: center; margin: 10px 0;">
-            <span class="feature-badge">🎤 Voice Activity Detection</span>
-            <span class="feature-badge">⚡ Fast Transcription</span>
-            <span class="feature-badge">🧠 Conversational AI</span>
-            <span class="feature-badge">📚 Document RAG</span>
-        </div>
-        """)
+        gr.Markdown("# 🎤 Voice Agent mit OpenAI Realtime API")
+        gr.Markdown("### Live Sprachkonversation mit juristischer Beratung")
     
     # Control Panel
     with gr.Column(elem_classes=["control-panel"]):
         with gr.Row():
-            with gr.Column(scale=2):
-                # Model Selection
-                model_selector = gr.Dropdown(
-                    choices=["tiny", "base", "small", "medium"],
-                    value=state.whisper_model,
-                    label="Whisper Model",
-                    info="Wählen Sie das Modell für Spracherkennung"
-                )
-                
-                # VAD Control
-                vad_toggle = gr.Checkbox(
-                    value=ENABLE_VAD,
-                    label="Voice Activity Detection aktivieren",
-                    info="Automatische Spracherkennung"
-                )
-                
-                # Language Selection
-                lang_selector = gr.Dropdown(
-                    choices=["de", "en", "auto"],
-                    value=ASR_LANGUAGE_HINT,
-                    label="Spracherkennung Sprache"
-                )
+            # Status Display
+            status_display = gr.HTML(
+                value='<div class="status-indicator status-inactive">🔴 Voice Agent inaktiv</div>',
+                label="Status"
+            )
             
+            # Voice Controls
             with gr.Column(scale=1):
-                # Status Display
-                status_display = gr.Textbox(
-                    label="System Status",
-                    value="Bereit",
-                    interactive=False
+                start_voice_btn = gr.Button(
+                    "🎤 Start Voice Conversation",
+                    variant="primary",
+                    elem_classes=["voice-btn", "voice-btn-start"]
                 )
-                
-                # Clear Conversation Button
-                clear_btn = gr.Button("🗑️ Konversation löschen", variant="secondary", size="sm")
-                
-                # VAD Indicator
-                vad_indicator = gr.HTML(value=update_vad_indicator(), label="VAD Status")
+                stop_voice_btn = gr.Button(
+                    "⏹️ Stop Voice Conversation",
+                    variant="secondary",
+                    elem_classes=["voice-btn", "voice-btn-stop"],
+                    visible=False
+                )
+        
+        # Mode Selection
+        with gr.Row():
+            mode_selector = gr.Radio(
+                choices=["Voice Agent (Live Conversation)", "Text Chat (RAG)"],
+                value="Text Chat (RAG)",
+                label="Modus"
+            )
     
     # Main Chat Interface
-    with gr.Column(elem_classes=["chat-container"]):
+    with gr.Column():
         # Chatbot Display
         chatbot = gr.Chatbot(
             label="Konversation",
-            height=400,
-            avatar_images=(None, "🤖")
+            height=500,
+            avatar_images=(None, "🤖"),
+            bubble_full_width=False,
+            show_copy_button=True
         )
         
-        # Input Row với VAD Indicator
-        with gr.Row(elem_classes=["input-row"]):
-            # Text Input
-            chat_text = gr.Textbox(
-                label=None,
-                placeholder="Stellen Sie eine Frage oder sprechen Sie ins Mikrofon...",
-                lines=1,
+        # Input Area (cho Text Mode)
+        with gr.Row(visible=True) as text_input_row:
+            chat_input = gr.Textbox(
+                label="Ihre Frage",
+                placeholder="Stellen Sie eine juristische Frage...",
+                lines=2,
                 max_lines=4,
-                scale=8,
-                container=False,
-                show_label=False
+                scale=8
             )
-            
-            # Audio Input
-            chat_audio = gr.Audio(
-                sources=["microphone"],
-                type="filepath",
-                format="wav",
-                streaming=True,
-                interactive=True,
-                show_label=False,
-                scale=1,
-                elem_id="audio-input"
-            )
-            
-            # Send Button
-            send_btn = gr.Button("➤", variant="primary", elem_classes=["send-btn"], scale=1)
+            send_btn = gr.Button("Senden", variant="primary", scale=1)
         
-        # TTS Controls
-        with gr.Row():
-            tts_btn = gr.Button("🔊 Antwort vorlesen", variant="secondary", size="sm")
-            tts_audio = gr.Audio(label="Audio Ausgabe", interactive=False, visible=False)
-            tts_status = gr.Textbox(label="TTS Status", interactive=False, visible=False)
+        # Voice Interface (cho Voice Mode)
+        with gr.Row(visible=False) as voice_interface:
+            gr.Markdown("### 🎤 Sprechen Sie jetzt...")
+            voice_status = gr.Textbox(
+                label="Status",
+                value="Bereit für Sprachaufnahme",
+                interactive=False
+            )
+            voice_output = gr.Textbox(label="Transkription", interactive=False)
     
     # Documents Section
-    with gr.Accordion("📚 Quellen & Dokumente", open=False):
+    with gr.Accordion("📚 Dokumente & Quellen", open=False):
         with gr.Tabs():
-            with gr.TabItem("📄 Prüfungsordnung (PDF)"):
-                PDF(pdf_meta["pdf_url"], height=300)
+            with gr.TabItem("📄 Prüfungsordnung"):
+                PDF(pdf_meta["pdf_url"], height=400)
             
             with gr.TabItem("📘 Hochschulgesetz NRW"):
-                if isinstance(hg_url, str) and hg_url.startswith("http"):
-                    gr.Markdown(f"### [Im Viewer öffnen]({hg_url})")
-                    gr.HTML(f'<iframe src="{hg_url}" width="100%" height="500px" style="border: 1px solid #ddd; border-radius: 8px;"></iframe>')
-                else:
-                    gr.Markdown("Viewer-Link nicht verfügbar.")
+                if hg_url:
+                    gr.HTML(f'''
+                    <div style="padding: 20px;">
+                        <h3>Hochschulgesetz NRW Viewer</h3>
+                        <a href="{hg_url}" target="_blank" style="display: inline-block; padding: 10px 20px; background: #3b82f6; color: white; text-decoration: none; border-radius: 5px; margin-bottom: 15px;">
+                            Im Viewer öffnen
+                        </a>
+                        <iframe src="{hg_url}" width="100%" height="500px" style="border: 1px solid #ddd; border-radius: 8px;"></iframe>
+                    </div>
+                    ''')
+    
+    # Queue Processing (cho streaming updates)
+    interval_component = gr.Interval(value=100, interactive=False)
     
     # =====================================================
     # EVENT HANDLERS
     # =====================================================
     
-    # Model Selection
-    model_selector.change(
-        change_whisper_model,
-        inputs=[model_selector],
-        outputs=[status_display]
-    )
+    def toggle_mode(mode):
+        """Chuyển đổi giữa Voice và Text mode"""
+        if "Voice Agent" in mode:
+            return (
+                gr.Row(visible=False),  # text_input_row
+                gr.Row(visible=True),   # voice_interface
+                '<div class="status-indicator status-inactive">🔴 Bitte Voice Agent starten</div>'
+            )
+        else:
+            stop_voice_agent()
+            return (
+                gr.Row(visible=True),   # text_input_row
+                gr.Row(visible=False),  # voice_interface
+                '<div class="status-indicator status-inactive">🔴 Text Mode aktiv</div>'
+            )
     
-    # VAD Toggle
-    vad_toggle.change(
-        toggle_vad,
-        inputs=[vad_toggle],
-        outputs=[status_display]
-    )
+    def start_voice_agent():
+        """Bắt đầu Voice Agent"""
+        success = voice_agent.start_session()
+        if success:
+            state.is_streaming = True
+            return (
+                gr.Button(visible=False),  # start_voice_btn
+                gr.Button(visible=True),   # stop_voice_btn
+                '<div class="status-indicator status-active">🟢 Voice Agent aktiv - Sprechen Sie jetzt</div>',
+                "Voice Agent gestartet. Sie können jetzt sprechen..."
+            )
+        else:
+            return (
+                gr.Button(visible=True),
+                gr.Button(visible=False),
+                '<div class="status-indicator status-inactive">🔴 Fehler beim Starten</div>',
+                "Fehler beim Starten des Voice Agents"
+            )
     
-    # Clear Conversation
-    clear_btn.click(
-        clear_conversation,
-        outputs=[chatbot, status_display]
-    ).then(
-        lambda: update_vad_indicator(),
-        outputs=[vad_indicator]
-    )
+    def stop_voice_agent():
+        """Dừng Voice Agent"""
+        voice_agent.stop_session()
+        state.is_streaming = False
+        state.reset()
+        return (
+            gr.Button(visible=True),   # start_voice_btn
+            gr.Button(visible=False),  # stop_voice_btn
+            '<div class="status-indicator status-inactive">🔴 Voice Agent gestoppt</div>',
+            "Voice Agent gestoppt"
+        )
     
-    # Main Chat Function
-    def process_chat(text_input, audio_path, history, lang_sel, use_vad):
-        """Wrapper function để xử lý chat"""
+    def process_text_chat(message, history):
+        """Xử lý text chat với RAG"""
+        if not message:
+            return history, ""
+        
+        # Thêm user message
+        history.append({"role": "user", "content": message})
+        
         try:
-            return chat_fn(text_input, audio_path, history, lang_sel, use_vad)
+            # Get RAG answer
+            ans, sources = answer(message, retriever, llm)
+            full_response = ans + format_sources(sources)
+            
+            # Add assistant message
+            history.append({"role": "assistant", "content": full_response})
+            
+            # Add to state
+            state.add_message("user", message)
+            state.add_message("assistant", ans)
+            
         except Exception as e:
-            print(f"Error in process_chat: {e}")
-            error_msg = f"Fehler: {str(e)}"
-            if history is None:
-                history = []
-            return history, "", None, error_msg
+            error_msg = f"Fehler: {str(e)[:100]}"
+            history.append({"role": "assistant", "content": error_msg})
+        
+        return history, ""
     
-    # Send Button Click
-    send_btn.click(
-        process_chat,
-        inputs=[chat_text, chat_audio, chatbot, lang_selector, vad_toggle],
-        outputs=[chatbot, chat_text, chat_audio, status_display]
-    ).then(
-        lambda: update_vad_indicator(),
-        outputs=[vad_indicator]
+    def update_streaming_display(history):
+        """Cập nhật display với streaming text"""
+        updates = process_queue_updates()
+        
+        if not updates:
+            return history
+        
+        for update_type, content in updates:
+            if update_type == "text" and content:
+                history = update_chat_display(history, content)
+        
+        return history
+    
+    # Mode toggle
+    mode_selector.change(
+        toggle_mode,
+        inputs=[mode_selector],
+        outputs=[text_input_row, voice_interface, status_display]
     )
     
-    # Text Submit (Enter key)
-    chat_text.submit(
-        process_chat,
-        inputs=[chat_text, chat_audio, chatbot, lang_selector, vad_toggle],
-        outputs=[chatbot, chat_text, chat_audio, status_display]
-    ).then(
-        lambda: update_vad_indicator(),
-        outputs=[vad_indicator]
+    # Voice Agent controls
+    start_voice_btn.click(
+        start_voice_agent,
+        outputs=[start_voice_btn, stop_voice_btn, status_display, voice_status]
     )
     
-    # Audio Change Handler
-    def on_audio_change(audio_path, use_vad):
-        """Xử lý khi audio thay đổi"""
-        if audio_path:
-            print(f"DEBUG: Audio changed: {audio_path}")
-            # Xử lý streaming
-            text, vad_html, status = handle_audio_stream(audio_path, use_vad)
-            return text, vad_html, status
-        return "", update_vad_indicator(), "Bereit"
-    
-    chat_audio.change(
-        on_audio_change,
-        inputs=[chat_audio, vad_toggle],
-        outputs=[chat_text, vad_indicator, status_display]
+    stop_voice_btn.click(
+        stop_voice_agent,
+        outputs=[start_voice_btn, stop_voice_btn, status_display, voice_status]
     )
     
-    # Audio Streaming
-    chat_audio.stream(
-        on_audio_change,
-        inputs=[chat_audio, vad_toggle],
-        outputs=[chat_text, vad_indicator, status_display]
+    # Text chat
+    send_btn.click(
+        process_text_chat,
+        inputs=[chat_input, chatbot],
+        outputs=[chatbot, chat_input]
     )
     
-    # TTS Button
-    def handle_tts(history):
-        """Xử lý TTS"""
-        audio_result = read_last_answer(history)
-        if audio_result:
-            return audio_result, "Audio wird abgespielt..."
-        return None, "Keine Antwort zum Vorlesen gefunden"
+    chat_input.submit(
+        process_text_chat,
+        inputs=[chat_input, chatbot],
+        outputs=[chatbot, chat_input]
+    )
     
-    tts_btn.click(
-        handle_tts,
+    # Streaming updates
+    interval_component.timer(
+        update_streaming_display,
         inputs=[chatbot],
-        outputs=[tts_audio, tts_status]
-    ).then(
-        lambda: gr.Audio(visible=True),
-        outputs=[tts_audio]
-    ).then(
-        lambda: gr.Textbox(visible=True),
-        outputs=[tts_status]
+        outputs=[chatbot],
+        every=0.1
     )
 
 if __name__ == "__main__":

realtime_server.py

@@ -1,97 +0,0 @@
-"""
-realtime_server.py — v0.1 (2025-12-08)
-
-Realtime signaling & streaming server (WebSocket-based) for live audio chat.
-This module is optional and preserves backward compatibility with existing
-Gradio UI. When enabled, clients can stream microphone audio chunks to
-`/ws` and receive live transcripts (OpenAI Whisper API) and bot replies.
-
-NOTE: A full WebRTC peer-to-peer relay with SDP/ICE is scaffolded via
-`/webrtc/offer` but returns 501 until the upstream Realtime API is wired.
-"""
-
-import os
-import asyncio
-import json
-from typing import Optional
-from fastapi import FastAPI, WebSocket, WebSocketDisconnect
-from fastapi.responses import JSONResponse
-
-# Minimal import guard for OpenAI
-try:
-    from openai import OpenAI
-    OPENAI_AVAILABLE = True
-except Exception:
-    OPENAI_AVAILABLE = False
-
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
-
-app = FastAPI()
-
-
-def _openai_transcribe_file(path: str, language: Optional[str] = None) -> str:
-    """Transcribe a local WAV chunk via OpenAI Whisper-1.
-    Returns empty string on failure to keep the stream resilient."""
-    if not (OPENAI_AVAILABLE and OPENAI_API_KEY and path and os.path.exists(path)):
-        return ""
-    try:
-        client = OpenAI(api_key=OPENAI_API_KEY)
-        with open(path, "rb") as f:
-            resp = client.audio.transcriptions.create(
-                model="whisper-1",
-                file=f,
-                language=language if language and language != "auto" else None,
-            )
-        txt = getattr(resp, "text", "") or (resp.get("text") if isinstance(resp, dict) else "")
-        return (txt or "").strip()
-    except Exception:
-        return ""
-
-
-@app.get("/health")
-async def health():
-    """Basic health endpoint."""
-    return JSONResponse({"status": "ok"})
-
-
-@app.post("/webrtc/offer")
-async def webrtc_offer(body: dict):
-    """SDP offer scaffold (not fully implemented).
-    Returns 501 until Realtime API relay is wired (to keep backward compatibility)."""
-    return JSONResponse({"error": "not_implemented"}, status_code=501)
-
-
-@app.websocket("/ws")
-async def ws_stream(ws: WebSocket):
-    """WebSocket bidirectional streaming.
-    Client sends JSON frames:
-      {"type":"audio_chunk","path":"/tmp/chunk.wav","lang":"de"}
-    Server responds with transcript frames:
-      {"type":"transcript","text":"..."}
-    and bot reply frames (if desired in future).
-    """
-    await ws.accept()
-    try:
-        while True:
-            raw = await ws.receive_text()
-            try:
-                msg = json.loads(raw)
-            except Exception:
-                await ws.send_text(json.dumps({"type": "error", "message": "invalid_json"}))
-                continue
-
-            if msg.get("type") == "audio_chunk":
-                path = msg.get("path")
-                lang = msg.get("lang")
-                text = _openai_transcribe_file(path, language=lang)
-                await ws.send_text(json.dumps({"type": "transcript", "text": text}))
-            else:
-                await ws.send_text(json.dumps({"type": "error", "message": "unknown_type"}))
-    except WebSocketDisconnect:
-        pass
-    except Exception:
-        try:
-            await ws.close()
-        except Exception:
-            pass
-