Update app.py

app.py

@@ -153,11 +153,13 @@ async def v1_chat(request: Request, authorization: str = Header(None)):
 async def v2_chat_completions(request: Request, authorization: str = Header(None)):
     """
     v2 – clean OpenAI-compatible streaming.
-    * Only delta.content
-    * No usage chunk
-    * Immediate empty delta for Continue.dev
+    * First chunk includes role=assistant (required by Continue.dev)
+    * Later chunks send only delta.content
+    * No usage events
     """
+
     check_key(authorization)
+
     if not BYTEZ_AUTH:
         raise HTTPException(status_code=500, detail="Server BYTEZ_API_KEY not configured")
 
@@ -168,11 +170,13 @@ async def v2_chat_completions(request: Request, authorization: str = Header(None
         raise HTTPException(status_code=400, detail=f"Invalid JSON: {e}")
 
     stream = payload.get("stream", False)
+
     upstream_headers = {
         "Authorization": BYTEZ_AUTH,
         "Content-Type": "application/json",
     }
 
+    # Normal content chunk (NO ROLE)
     def make_openai_delta(content: str):
         return {
             "id": f"chatcmpl-v2-{int(time.time())}",
@@ -189,23 +193,41 @@ async def v2_chat_completions(request: Request, authorization: str = Header(None
         }
 
     async def clean_stream():
-        # SEND EMPTY DELTA FIRST → wakes up Continue.dev
-        yield f"data: {json.dumps(make_openai_delta(''))}\n\n"
+        # FIRST CHUNK MUST SET THE ROLE → REQUIRED by Continue.dev
+        first_chunk = {
+            "id": f"chatcmpl-v2-{int(time.time())}",
+            "object": "chat.completion.chunk",
+            "created": int(time.time()),
+            "model": payload.get("model", "unknown"),
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": {"role": "assistant", "content": ""},
+                    "finish_reason": None,
+                }
+            ],
+        }
+
+        # Send first role-setting chunk
+        yield f"data: {json.dumps(first_chunk)}\n\n"
 
         async with httpx.AsyncClient(timeout=180) as client:
             try:
                 async with client.stream(
                     "POST", BYTEZ_CHAT_URL, headers=upstream_headers, json=payload
                 ) as upstream:
+
                     async for line in upstream.aiter_lines():
                         line = line.strip()
                         if not line:
                             continue
+
                         json_str = line[6:] if line.startswith("data: ") else line
 
-                        # Skip usage chunk
+                        # Skip usage events
                         if "usage" in json_str.lower():
                             continue
+
                         if json_str == "[DONE]":
                             yield "data: [DONE]\n\n"
                             return
@@ -230,7 +252,6 @@ async def v2_chat_completions(request: Request, authorization: str = Header(None
                         if text:
                             yield f"data: {json.dumps(make_openai_delta(text))}\n\n"
 
-                # Fallback [DONE]
                 yield "data: [DONE]\n\n"
 
             except Exception as e:
@@ -238,7 +259,7 @@ async def v2_chat_completions(request: Request, authorization: str = Header(None
                 yield f"data: {json.dumps(error_chunk)}\n\n"
                 yield "data: [DONE]\n\n"
 
-    # --- Non-streaming ---
+    # Non-streaming mode
     if not stream:
         async with httpx.AsyncClient(timeout=120) as c:
             r = await c.post(BYTEZ_CHAT_URL, headers=upstream_headers, json=payload)
@@ -246,15 +267,23 @@ async def v2_chat_completions(request: Request, authorization: str = Header(None
             data = r.json()
 
         if "choices" not in data:
-            content = data.get("output") or data.get("response") or data.get("message") or str(data)
+            content = (
+                data.get("output")
+                or data.get("response")
+                or data.get("message")
+                or str(data)
+            )
             data = {
                 "id": "chatcmpl-v2",
                 "object": "chat.completion",
-                "choices": [{"index": 0, "message": {"role": "assistant", "content": content}}],
+                "choices": [
+                    {"index": 0, "message": {"role": "assistant", "content": content}}
+                ],
             }
+
         return JSONResponse(data, headers={"Access-Control-Allow-Origin": "*"})
 
-    # --- Streaming ---
+    # Streaming mode
     return StreamingResponse(
         clean_stream(),
         media_type="text/event-stream",
@@ -263,7 +292,7 @@ async def v2_chat_completions(request: Request, authorization: str = Header(None
             "Access-Control-Allow-Headers": "*",
             "Cache-Control": "no-cache",
             "Connection": "keep-alive",
-            "X-Accel-Buffering": "no",  # CRITICAL for HF Spaces
+            "X-Accel-Buffering": "no",
         },
     )