Update api/utils.py

api/utils.py

@@ -141,7 +141,6 @@ async def process_streaming_response(request: ChatRequest):
         "imageGenerationMode": False,  # Added this line
     }
 
-    response_chunks = []
     async with httpx.AsyncClient() as client:
         try:
             async with client.stream(
@@ -152,48 +151,45 @@ async def process_streaming_response(request: ChatRequest):
                 timeout=100,
             ) as response:
                 response.raise_for_status()
+
+                # Start processing the chunks and yield them one by one
+                timestamp = int(datetime.now().timestamp())
+                response_content = ""  # Collect response content
+
                 async for chunk in response.aiter_text():
-                    timestamp = int(datetime.now().timestamp())
                     if chunk:
                         content = chunk
                         if content.startswith("$@$v=undefined-rv1$@$"):
-                            content = content[21:]
-                        # Remove the blocked message if present
+                            content = content[21:]  # Remove unwanted prefix
+
+                        # Remove blocked message if present
                         if BLOCKED_MESSAGE in content:
-                            logger.info(
-                                f"Blocked message detected in response for Request ID {request_id}."
-                            )
+                            logger.info(f"Blocked message detected in response for Request ID {request_id}.")
                             content = content.replace(BLOCKED_MESSAGE, '').strip()
-                            if not content:
-                                continue  # Skip if content is empty after removal
+
+                        if not content:
+                            continue  # Skip if content is empty after removal
+
+                        # Clean up the content
                         cleaned_content = strip_model_prefix(content, model_prefix)
-                        response_chunks.append(create_chat_completion_data(cleaned_content, request.model, timestamp))
-                
-                # At the very end, add the advertisement text once
+
+                        # Yield each chunk as soon as it's ready
+                        yield f"data: {json.dumps(create_chat_completion_data(cleaned_content, request.model, timestamp))}\n\n"
+                        response_content += cleaned_content  # Collect the full response content
+
+                # After all chunks are processed, add the advertisement text at the end
                 if ADVERTISEMENT_TEXT:
-                    # If there are chunks already, update the last one with the advertisement text.
-                    if response_chunks:
-                        last_chunk = response_chunks[-1]
-                        last_chunk["choices"][0]["delta"]["content"] += "\n\n" + ADVERTISEMENT_TEXT
-                    else:
-                        # If no chunks are returned, just create an empty response with the ad.
-                        response_chunks.append(create_chat_completion_data(ADVERTISEMENT_TEXT, request.model, timestamp))
-
-                # Finalize the response
-                response_chunks.append(create_chat_completion_data('', request.model, timestamp, 'stop'))
-                response_chunks.append({"data": "[DONE]\n\n"})
-                
-                # Yield each chunk as part of the stream response
-                for chunk in response_chunks:
-                    yield f"data: {json.dumps(chunk)}\n\n"
+                    response_content += "\n\n" + ADVERTISEMENT_TEXT
+                    yield f"data: {json.dumps(create_chat_completion_data(response_content, request.model, timestamp, 'stop'))}\n\n"
+
+                # Add the final "done" marker
+                yield "data: [DONE]\n\n"
 
         except httpx.HTTPStatusError as e:
             logger.error(f"HTTP error occurred for Request ID {request_id}: {e}")
             raise HTTPException(status_code=e.response.status_code, detail=str(e))
         except httpx.RequestError as e:
-            logger.error(
-                f"Error occurred during request for Request ID {request_id}: {e}"
-            )
+            logger.error(f"Error occurred during request for Request ID {request_id}: {e}")
             raise HTTPException(status_code=500, detail=str(e))
 
 # Process non-streaming response with headers from config.py