feat: implement real-time text-to-speech highlighting and document management within chat interface

backend/app/api/api_v1/endpoints/chat.py

@@ -303,20 +303,35 @@ async def speak(request: Request, speak_data: SpeakRequest):
     
     # Internal stop signal for THIS specific request
     import threading
+    import asyncio
     disconnect_event = threading.Event()
 
+    async def watch_disconnect():
+        try:
+            while not disconnect_event.is_set():
+                if await request.is_disconnected():
+                    disconnect_event.set()
+                    break
+                await asyncio.sleep(0.1)
+        except asyncio.CancelledError:
+            pass
+
+    watch_task = asyncio.create_task(watch_disconnect())
+
     # Generator wrapper to monitor disconnection
     async def disconnect_monitor_gen():
         generator = stream_tts_wav_chunks(clean_text, disconnect_event)
         try:
             for chunk in generator:
-                if await request.is_disconnected():
-                    disconnect_event.set()
+                if disconnect_event.is_set():
                     break
                 yield chunk
         except Exception as e:
             disconnect_event.set()
             raise e
+        finally:
+            disconnect_event.set()
+            watch_task.cancel()
 
     return StreamingResponse(
         disconnect_monitor_gen(),

backend/app/services/tts.py

@@ -12,6 +12,7 @@ SPEECH_SPEED = 1  # Slower than default 1.0
 
 # Global event to stop any ongoing TTS generation across streams
 stop_tts_event = threading.Event()
+tts_generation_lock = threading.Lock()
 
 def clean_text_for_speech(text: str) -> str:
     """Removes citations and other non-spoken markers from text, and formats dates."""
@@ -109,7 +110,7 @@ def stream_tts_wav_chunks(text, cancel_event=None):
     
     if not raw_sentences:
         return
-
+ 
     import soundfile as sf
     import base64
     import json
@@ -127,12 +128,13 @@ def stream_tts_wav_chunks(text, cancel_event=None):
                 if not s_clean:
                     continue
 
-                generator = pipeline(s_clean, voice=voice, speed=SPEECH_SPEED)
-                sentence_audio_chunks = []
-                for gs, ps, audio in generator:
-                    if (cancel_event and cancel_event.is_set()) or stop_tts_event.is_set() or local_stop_signal.is_set():
-                        break
-                    sentence_audio_chunks.append(audio)
+                with tts_generation_lock:
+                    generator = pipeline(s_clean, voice=voice, speed=SPEECH_SPEED)
+                    sentence_audio_chunks = []
+                    for gs, ps, audio in generator:
+                        if (cancel_event and cancel_event.is_set()) or stop_tts_event.is_set() or local_stop_signal.is_set():
+                            break
+                        sentence_audio_chunks.append(audio)
                 
                 if sentence_audio_chunks:
                     full_audio = np.concatenate(sentence_audio_chunks)

frontend-react/src/App.jsx

@@ -1,4 +1,4 @@
-import { useState, useEffect } from 'react';
+import { useState, useEffect, useCallback } from 'react';
 import Auth from './components/Auth';
 import Sidebar from './components/Sidebar';
 import ChatWindow from './components/ChatWindow';
@@ -27,14 +27,14 @@ function App() {
   const [refreshSessions, setRefreshSessions] = useState(0);
   const [isSidebarCollapsed, setIsSidebarCollapsed] = useState(false);
 
-  const handleLoginSuccess = (newToken, newEmail) => {
+  const handleLoginSuccess = useCallback((newToken, newEmail) => {
     setToken(newToken);
     setEmail(newEmail);
     localStorage.setItem('rag_token', newToken);
     localStorage.setItem('rag_email', newEmail);
-  };
+  }, []);
 
-  const handleLogout = () => {
+  const handleLogout = useCallback(() => {
     setToken('');
     setEmail('');
     setMessages([]);
@@ -42,26 +42,30 @@ function App() {
     setCurrentView('chat');
     localStorage.removeItem('rag_token');
     localStorage.removeItem('rag_email');
-  };
+  }, []);
 
-  const handleNewChat = () => {
+  const handleNewChat = useCallback(() => {
     setCurrentSessionId(generateId());
     setMessages([]);
     setSessionDocuments([]);
     setCurrentView('chat');
     if (window.innerWidth <= 768) setIsSidebarCollapsed(true);
-  };
+  }, []);
 
-  const handleSelectSession = (sessionId) => {
+  const handleSelectSession = useCallback((sessionId) => {
     setCurrentSessionId(sessionId);
     setCurrentView('chat');
     if (window.innerWidth <= 768) setIsSidebarCollapsed(true);
-  };
+  }, []);
 
-  const handleViewLibrary = () => {
+  const handleViewLibrary = useCallback(() => {
     setCurrentView('library');
     if (window.innerWidth <= 768) setIsSidebarCollapsed(true);
-  };
+  }, []);
+
+  const handleRefreshSessions = useCallback(() => {
+    setRefreshSessions(prev => prev + 1);
+  }, []);
 
   if (!token) {
     return <Auth onLoginSuccess={handleLoginSuccess} />;
@@ -112,9 +116,9 @@ function App() {
             messages={messages}
             setMessages={setMessages}
             sessionId={currentSessionId}
-            onFirstMessage={() => setRefreshSessions(prev => prev + 1)}
+            onFirstMessage={handleRefreshSessions}
             setIsUploading={setIsUploading}
-            onUploadSuccess={() => setRefreshSessions(prev => prev + 1)}
+            onUploadSuccess={handleRefreshSessions}
             isSidebarCollapsed={isSidebarCollapsed}
             setIsSidebarCollapsed={setIsSidebarCollapsed}
             sessionDocuments={sessionDocuments}

frontend-react/src/components/ChatWindow.jsx

@@ -405,8 +405,16 @@ export default function ChatWindow({
   const handleSpeak = async (text, index) => {
     // If clicking same button while playing, STOP everything
     if (speakingIdx === index) {
-      if (audioRef.current) audioRef.current.pause();
-      if (abortControllerRef.current) abortControllerRef.current.abort();
+      if (audioRef.current) {
+        audioRef.current.onplay = null;
+        audioRef.current.onended = null;
+        audioRef.current.pause();
+        audioRef.current.src = "";
+        audioRef.current.load();
+      }
+      if (abortControllerRef.current) {
+        abortControllerRef.current.abort();
+      }
       if (highlightTimerRef.current) clearTimeout(highlightTimerRef.current);
       
       audioQueueRef.current.forEach(item => URL.revokeObjectURL(item.url));
@@ -418,7 +426,13 @@ export default function ChatWindow({
     }
 
     // Stop current if any
-    if (audioRef.current) audioRef.current.pause();
+    if (audioRef.current) {
+      audioRef.current.onplay = null;
+      audioRef.current.onended = null;
+      audioRef.current.pause();
+      audioRef.current.src = "";
+      audioRef.current.load();
+    }
     if (abortControllerRef.current) abortControllerRef.current.abort();
     if (highlightTimerRef.current) clearTimeout(highlightTimerRef.current);
 
@@ -478,13 +492,13 @@ export default function ChatWindow({
               playNextInQueue();
             }
           } catch (e) {
-            console.error("Error parsing audio chunk", e);
+            console.error("Error parsing audio chunk JSON:", e);
           }
         }
       }
     } catch (err) {
       if (err.name !== 'AbortError') {
-        console.error("TTS Stream error", err);
+        console.error("TTS Stream error:", err);
         setSpeakingIdx(null);
       }
     }