Update app.py

app.py

@@ -1,26 +1,27 @@
 import asyncio
+import os
 import numpy as np
+import torch
 import gradio as gr
+from fastapi import FastAPI
 from fastrtc import AsyncStreamHandler, Stream, wait_for_item
-from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer
-import torch
+from transformers import pipeline, AutoTokenizer
 
-# 1. Initialize Open Source Models
-# Using Whisper for STT, Phi-3 for LLM (fast), and a local TTS pipeline
+# 1. Hardware & Modell-Initialisierung
 device = "cuda" if torch.cuda.is_available() else "cpu"
+# Wir nutzen 4-bit Quantisierung falls CUDA verfügbar ist, um Latenz zu sparen
+model_kwargs = {"torch_dtype": torch.float16, "load_in_4bit": True} if device == "cuda" else {}
 
-# STT: Whisper Tiny is fast enough for real-time
+# STT: Whisper-tiny für minimale Latenz
 stt_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-tiny", device=device)
 
-# LLM: Phi-3-mini is lightweight and powerful
+# LLM: Phi-3-mini (Instruction-tuned)
 llm_model = "microsoft/Phi-3-mini-4k-instruct"
 tokenizer = AutoTokenizer.from_pretrained(llm_model)
-llm_pipe = pipeline("text-generation", model=llm_model, tokenizer=tokenizer, device=device)
+llm_pipe = pipeline("text-generation", model=llm_model, tokenizer=tokenizer, device=device, model_kwargs=model_kwargs)
 
-# TTS: We use a simple gTTS or local pipeline (using a placeholder for logic here)
-# For a pure local OS experience, Bark or Parler-TTS are great but heavy.
-# Here we use a standard TTS pipeline.
-tts_pipe = pipeline("text-to-speech", model="facebook/mms-tts-deu", device=device) # German
+# TTS: MMS-TTS für Deutsch
+tts_pipe = pipeline("text-to-speech", model="facebook/mms-tts-deu", device=device)
 
 class OpenSourceHandler(AsyncStreamHandler):
     def __init__(self, expected_layout="mono", output_sample_rate=24000):
@@ -29,24 +30,34 @@ class OpenSourceHandler(AsyncStreamHandler):
 
     async def receive(self, frame: tuple[int, np.ndarray]) -> None:
         rate, array = frame
-        # 1. STT: Convert Audio to Text
-        # Whisper expects float32
+        
+        # Zugriff auf die Gradio-Zusatzeingaben (z.B. Sprache oder System-Prompt)
+        # latest_args[0] ist der erste zusätzliche Input nach dem Audio
+        # Hier beispielhaft für zukünftige Erweiterungen:
+        # system_msg = self.latest_args[0] if self.latest_args else "Du bist ein Assistent."
+
+        # 1. STT: Audio -> Text (Whisper erwartet Float32)
         audio_fp32 = array.astype(np.float32) / 32768.0
         text_result = stt_pipe({"sampling_rate": rate, "raw": audio_fp32})["text"]
         
-        if len(text_result.strip()) > 2:
-            # 2. LLM: Generate Response
+        # Rausch-Filter: Nur antworten, wenn wirklich Text erkannt wurde
+        if len(text_result.strip()) > 3:
+            # 2. LLM: Antwort generieren
+            # return_full_text=False verhindert, dass der Prompt mit ausgegeben wird
             prompt = f"<|user|>\n{text_result}<|end|>\n<|assistant|>"
-            response = llm_pipe(prompt, max_new_tokens=50, do_sample=True)[0]["generated_text"]
-            answer = response.split("<|assistant|>")[-1].strip()
+            outputs = llm_pipe(prompt, max_new_tokens=64, do_sample=True, return_full_text=False)
+            answer = outputs[0]["generated_text"].strip()
 
-            # 3. TTS: Convert Text back to Audio
+            # 3. TTS: Text -> Audio
+            # MMS-TTS gibt ein Dict zurück: {'audio': ndarray, 'sampling_rate': int}
             audio_out = tts_pipe(answer)
-            audio_data = np.frombuffer(audio_out["audio"], dtype=np.float32)
+            audio_data = audio_out["audio"] # Das ist bereits ein numpy array
             
-            # Convert to int16 for the output stream
+            # Resampling / Konvertierung zu Int16 für den Stream
             audio_int16 = (audio_data * 32767).astype(np.int16)
-            self.output_queue.put_nowait((self.output_sample_rate, audio_int16))
+            
+            # Wir nutzen await für die Queue, um sauberes Async-Verhalten zu garantieren
+            await self.output_queue.put((self.output_sample_rate, audio_int16))
 
     async def emit(self) -> tuple[int, np.ndarray] | None:
         return await wait_for_item(self.output_queue)
@@ -54,28 +65,23 @@ class OpenSourceHandler(AsyncStreamHandler):
     def copy(self) -> "OpenSourceHandler":
         return OpenSourceHandler(output_sample_rate=self.output_sample_rate)
 
-# 2. Define the Gradio Stream
+# 2. FastAPI & Stream Setup
+app = FastAPI() # Hier definieren wir die App!
+
 stream = Stream(
     modality="audio",
     mode="send-receive",
     handler=OpenSourceHandler(),
     additional_inputs=[
-        gr.Dropdown(
-            label="System Language",
-            choices=["de-DE", "en-US"],
-            value="de-DE",
-        ),
+        gr.Textbox(label="System Message", value="Du bist ein hilfreicher KI-Assistent."),
     ],
 )
 
+# WICHTIG: Mountet den Stream in die FastAPI App
+stream.mount(app)
+
 if __name__ == "__main__":
     import uvicorn
-    import os
-    
-    # Hugging Face Spaces provides the port via an environment variable, 
-    # but defaults to 7860.
+    # Port 7860 für Hugging Face Spaces
     port = int(os.getenv("PORT", 7860))
-    
-    # We bind to 0.0.0.0 to make the app accessible within the HF container
-    #uvicorn.run(app, host="0.0.0.0", port=port)
-    stream.launch(server_name="0.0.0.0", server_port=port)
+    uvicorn.run(app, host="0.0.0.0", port=port)