Add streaming support for Vapi compatibility

app.py

@@ -75,6 +75,11 @@ def chat_completions():
     
     try:
         data = request.get_json()
+        print(f"Full request data: {data}")
+        
+        # Check if streaming is requested
+        stream = data.get("stream", False)
+        
         messages = data.get("messages", [])
         
         # Extract user messages (ignore system messages from Vapi)
@@ -135,9 +140,10 @@ def chat_completions():
         if hasattr(model, 'device'):
             inputs = {k: v.to(model.device) for k, v in inputs.items()}
         
+        print(f"Generating response...")
         outputs = model.generate(
             **inputs,
-            max_new_tokens=256, 
+            max_new_tokens=150,  # Reduced from 256 for faster response
             temperature=0.7,
             do_sample=True,
             top_p=0.9,
@@ -146,6 +152,7 @@ def chat_completions():
             pad_token_id=tokenizer.pad_token_id,
             eos_token_id=tokenizer.eos_token_id
         )
+        print(f"Response generated!")
 
         full_response = tokenizer.decode(outputs[0], skip_special_tokens=False)
         
@@ -153,7 +160,30 @@ def chat_completions():
         response_text = full_response.split("<start_of_turn>model\n")[-1]
         response_text = response_text.replace("<end_of_turn>", "").strip()
         
-        # OpenAI-compatible response format (Vapi için tam uyumlu)
+        # If streaming requested, return SSE format
+        if stream:
+            def generate():
+                # Send the complete response as a single chunk for simplicity
+                chunk = {
+                    "id": "chatcmpl-" + str(hash(prompt))[-10:],
+                    "object": "chat.completion.chunk",
+                    "created": int(__import__('time').time()),
+                    "model": MODEL_NAME,
+                    "choices": [{
+                        "index": 0,
+                        "delta": {
+                            "role": "assistant",
+                            "content": response_text
+                        },
+                        "finish_reason": "stop"
+                    }]
+                }
+                yield f"data: {__import__('json').dumps(chunk)}\n\n"
+                yield "data: [DONE]\n\n"
+            
+            return app.response_class(generate(), mimetype='text/event-stream')
+        
+        # OpenAI-compatible response format (non-streaming)
         vapi_response = {
             "id": "chatcmpl-" + str(hash(prompt))[-10:],
             "object": "chat.completion",