Update app.py

app.py

@@ -355,68 +355,67 @@ async def stream_text_to_speech_cloning(
     """
     if not hasattr(app.state, 'tts_wrapper'):
         raise HTTPException(status_code=503, detail="Service unavailable: Model not loaded")
-    
-    # This async generator is the final, correct implementation.
+
     async def stream_generator():
         loop = asyncio.get_event_loop()
         q = asyncio.Queue(maxsize=2)
 
-        # The PRODUCER is now an async task that runs in the background.
+        # The PRODUCER's job is to quickly schedule work, not wait for it.
         async def producer():
             try:
-                # The one-time setup cost: convert and encode the reference voice.
-                # This is done before the loop to ensure the voice is ready.
                 converted_wav_buffer = await convert_to_wav_in_memory(reference_audio)
                 ref_audio_bytes = converted_wav_buffer.getvalue()
                 audio_hash = hashlib.sha256(ref_audio_bytes).hexdigest()
-                ref_s = await loop.run_in_executor(
-                    tts_executor, 
-                    app.state.tts_wrapper._get_or_create_reference_encoding, 
-                    audio_hash, 
-                    ref_audio_bytes
-                )
+                
+                # Check cache for reference encoding
+                if audio_hash in app.state.tts_wrapper.encoding_cache:
+                    logger.info(f"Streaming Cache HIT for hash: {audio_hash[:10]}...")
+                    ref_s = app.state.tts_wrapper.encoding_cache[audio_hash]
+                else:
+                    logger.info(f"Streaming Cache MISS for hash: {audio_hash[:10]}...")
+                    ref_s = await loop.run_in_executor(
+                        tts_executor, 
+                        app.state.tts_wrapper.get_reference_encoding, 
+                        ref_audio_bytes
+                    )
+                    app.state.tts_wrapper.encoding_cache[audio_hash] = ref_s
                 
                 sentences = app.state.tts_wrapper._split_text_into_chunks(text)
                 
-                for sentence in sentences:
-                    # Define the blocking work for a single chunk
-                    def process_chunk():
-                        with torch.no_grad():
-                            audio_chunk = app.state.tts_wrapper.tts_model.infer(sentence, ref_s, reference_text)
-                        return app.state.tts_wrapper._convert_to_streamable_format(audio_chunk, output_format)
+                # This function does the heavy lifting for one chunk.
+                def process_chunk(sentence_text):
+                    with torch.no_grad():
+                        audio_chunk = app.state.tts_wrapper.tts_model.infer(sentence_text, ref_s, reference_text)
+                    return app.state.tts_wrapper._convert_to_streamable_format(audio_chunk, output_format)
 
-                    # Offload the blocking work to the thread pool
-                    mp3_bytes = await loop.run_in_executor(tts_executor, process_chunk)
-                    # Put the finished MP3 chunk into the async queue
-                    await q.put(mp3_bytes)
+                # Schedule all chunks to be processed in the background.
+                for sentence in sentences:
+                    task = loop.run_in_executor(tts_executor, process_chunk, sentence)
+                    await q.put(task) # Put the FUTURE, not the result, in the queue.
 
             except Exception as e:
                 logger.error(f"Error in producer task: {e}")
                 await q.put(e)
             finally:
-                # Signal that production is finished
-                await q.put(None)
+                await q.put(None) # Signal that all tasks have been scheduled.
 
-        # Start the producer as a background task. It starts working immediately.
         producer_task = asyncio.create_task(producer())
 
-        # The main loop now acts as the CONSUMER.
+        # The CONSUMER's job is to wait for each result and yield it.
         while True:
-            # Await the next finished MP3 chunk from the queue.
             result = await q.get()
-
             if result is None:
                 break
             
+            # Check if the item in the queue is a task (future) or an exception
             if isinstance(result, Exception):
                 logger.error(f"Terminating stream due to producer error: {result}")
                 raise result
             
-            # Yield the chunk to the user. While the network sends this,
-            # the producer is already working on the next chunk in the background.
-            yield result
+            # Await the result of the background task
+            chunk_bytes = await result
+            yield chunk_bytes
             
-        # Ensure the producer task is cleaned up.
         await producer_task
 
     return StreamingResponse(