Update app.py
Browse files
app.py
CHANGED
|
@@ -150,35 +150,18 @@ async def create_replicate_prediction(session: aiohttp.ClientSession, model: str
|
|
| 150 |
logger.error(f"Error creating prediction: {str(e)}")
|
| 151 |
raise HTTPException(status_code=500, detail=f"Prediction creation error: {str(e)}")
|
| 152 |
|
| 153 |
-
def transform_replicate_to_openai_stream(event_data: str, model: str) -> str:
|
| 154 |
"""将Replicate流式响应转换为OpenAI格式"""
|
| 155 |
-
if not event_data.startswith("data: "):
|
| 156 |
-
return ""
|
| 157 |
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
if data.get("event") == "output":
|
| 162 |
-
# 构建OpenAI格式的响应
|
| 163 |
-
openai_response = {
|
| 164 |
-
"id": f"chatcmpl-{data.get('id', 'unknown')}",
|
| 165 |
-
"object": "chat.completion.chunk",
|
| 166 |
-
"created": int(asyncio.get_event_loop().time()),
|
| 167 |
-
"model": model,
|
| 168 |
-
"choices": [{
|
| 169 |
-
"index": 0,
|
| 170 |
-
"delta": {
|
| 171 |
-
"content": data.get("data", "")
|
| 172 |
-
},
|
| 173 |
-
"finish_reason": None
|
| 174 |
-
}]
|
| 175 |
-
}
|
| 176 |
-
return f"data: {json.dumps(openai_response)}\n\n"
|
| 177 |
|
| 178 |
-
|
|
|
|
| 179 |
# 发送结束标记
|
| 180 |
openai_response = {
|
| 181 |
-
"id": f"chatcmpl-{
|
| 182 |
"object": "chat.completion.chunk",
|
| 183 |
"created": int(asyncio.get_event_loop().time()),
|
| 184 |
"model": model,
|
|
@@ -190,11 +173,60 @@ def transform_replicate_to_openai_stream(event_data: str, model: str) -> str:
|
|
| 190 |
}
|
| 191 |
return f"data: {json.dumps(openai_response)}\n\ndata: [DONE]\n\n"
|
| 192 |
|
| 193 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
|
| 195 |
-
|
| 196 |
-
logger.warning(f"Failed to parse event data: {event_data}, error: {e}")
|
| 197 |
-
return ""
|
| 198 |
|
| 199 |
@app.get("/")
|
| 200 |
async def root():
|
|
@@ -267,7 +299,7 @@ async def chat_completions(request: Request):
|
|
| 267 |
replicate_data, model = transform_openai_to_replicate(body)
|
| 268 |
|
| 269 |
if body.get("stream", False):
|
| 270 |
-
# 流式响应
|
| 271 |
async def generate_stream():
|
| 272 |
async with aiohttp.ClientSession() as session:
|
| 273 |
try:
|
|
@@ -296,6 +328,8 @@ async def chat_completions(request: Request):
|
|
| 296 |
"Cache-Control": "no-store"
|
| 297 |
}
|
| 298 |
|
|
|
|
|
|
|
| 299 |
async with session.get(stream_url, headers=headers, timeout=300) as response:
|
| 300 |
if response.status != 200:
|
| 301 |
error_text = await response.text()
|
|
@@ -312,9 +346,29 @@ async def chat_completions(request: Request):
|
|
| 312 |
async for line in response.content:
|
| 313 |
line = line.decode('utf-8').strip()
|
| 314 |
if line:
|
| 315 |
-
|
|
|
|
| 316 |
if openai_event:
|
| 317 |
yield openai_event
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
|
| 319 |
except Exception as e:
|
| 320 |
logger.error(f"Stream generation error: {e}")
|
|
|
|
| 150 |
logger.error(f"Error creating prediction: {str(e)}")
|
| 151 |
raise HTTPException(status_code=500, detail=f"Prediction creation error: {str(e)}")
|
| 152 |
|
| 153 |
+
def transform_replicate_to_openai_stream(event_data: str, model: str, prediction_id: str) -> str:
|
| 154 |
"""将Replicate流式响应转换为OpenAI格式"""
|
|
|
|
|
|
|
| 155 |
|
| 156 |
+
# 处理不同的事件格式
|
| 157 |
+
if event_data.startswith("data: "):
|
| 158 |
+
content = event_data[6:] # 移除 "data: " 前缀
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
|
| 160 |
+
# 检查是否是结束标记
|
| 161 |
+
if content.strip() in ["[DONE]", ""]:
|
| 162 |
# 发送结束标记
|
| 163 |
openai_response = {
|
| 164 |
+
"id": f"chatcmpl-{prediction_id}",
|
| 165 |
"object": "chat.completion.chunk",
|
| 166 |
"created": int(asyncio.get_event_loop().time()),
|
| 167 |
"model": model,
|
|
|
|
| 173 |
}
|
| 174 |
return f"data: {json.dumps(openai_response)}\n\ndata: [DONE]\n\n"
|
| 175 |
|
| 176 |
+
# 尝试解析为JSON(用于其他事件类型)
|
| 177 |
+
try:
|
| 178 |
+
data = json.loads(content)
|
| 179 |
+
if data.get("event") == "output":
|
| 180 |
+
openai_response = {
|
| 181 |
+
"id": f"chatcmpl-{prediction_id}",
|
| 182 |
+
"object": "chat.completion.chunk",
|
| 183 |
+
"created": int(asyncio.get_event_loop().time()),
|
| 184 |
+
"model": model,
|
| 185 |
+
"choices": [{
|
| 186 |
+
"index": 0,
|
| 187 |
+
"delta": {
|
| 188 |
+
"content": data.get("data", "")
|
| 189 |
+
},
|
| 190 |
+
"finish_reason": None
|
| 191 |
+
}]
|
| 192 |
+
}
|
| 193 |
+
return f"data: {json.dumps(openai_response)}\n\n"
|
| 194 |
+
elif data.get("event") == "done":
|
| 195 |
+
openai_response = {
|
| 196 |
+
"id": f"chatcmpl-{prediction_id}",
|
| 197 |
+
"object": "chat.completion.chunk",
|
| 198 |
+
"created": int(asyncio.get_event_loop().time()),
|
| 199 |
+
"model": model,
|
| 200 |
+
"choices": [{
|
| 201 |
+
"index": 0,
|
| 202 |
+
"delta": {},
|
| 203 |
+
"finish_reason": "stop"
|
| 204 |
+
}]
|
| 205 |
+
}
|
| 206 |
+
return f"data: {json.dumps(openai_response)}\n\ndata: [DONE]\n\n"
|
| 207 |
+
except json.JSONDecodeError:
|
| 208 |
+
# 不是JSON,作为直接文本内容处理
|
| 209 |
+
if content.strip(): # 只有非空内容才发送
|
| 210 |
+
openai_response = {
|
| 211 |
+
"id": f"chatcmpl-{prediction_id}",
|
| 212 |
+
"object": "chat.completion.chunk",
|
| 213 |
+
"created": int(asyncio.get_event_loop().time()),
|
| 214 |
+
"model": model,
|
| 215 |
+
"choices": [{
|
| 216 |
+
"index": 0,
|
| 217 |
+
"delta": {
|
| 218 |
+
"content": content
|
| 219 |
+
},
|
| 220 |
+
"finish_reason": None
|
| 221 |
+
}]
|
| 222 |
+
}
|
| 223 |
+
return f"data: {json.dumps(openai_response)}\n\n"
|
| 224 |
+
|
| 225 |
+
# 处理其他类型的事件(如event: output等)
|
| 226 |
+
elif event_data.startswith("event: "):
|
| 227 |
+
return "" # 忽略事件类型行
|
| 228 |
|
| 229 |
+
return ""
|
|
|
|
|
|
|
| 230 |
|
| 231 |
@app.get("/")
|
| 232 |
async def root():
|
|
|
|
| 299 |
replicate_data, model = transform_openai_to_replicate(body)
|
| 300 |
|
| 301 |
if body.get("stream", False):
|
| 302 |
+
# 流式响应
|
| 303 |
async def generate_stream():
|
| 304 |
async with aiohttp.ClientSession() as session:
|
| 305 |
try:
|
|
|
|
| 328 |
"Cache-Control": "no-store"
|
| 329 |
}
|
| 330 |
|
| 331 |
+
stream_finished = False
|
| 332 |
+
|
| 333 |
async with session.get(stream_url, headers=headers, timeout=300) as response:
|
| 334 |
if response.status != 200:
|
| 335 |
error_text = await response.text()
|
|
|
|
| 346 |
async for line in response.content:
|
| 347 |
line = line.decode('utf-8').strip()
|
| 348 |
if line:
|
| 349 |
+
logger.info(f"Received stream line: {line}")
|
| 350 |
+
openai_event = transform_replicate_to_openai_stream(line, model, prediction_id)
|
| 351 |
if openai_event:
|
| 352 |
yield openai_event
|
| 353 |
+
# 检查是否是结束事件
|
| 354 |
+
if "[DONE]" in openai_event:
|
| 355 |
+
stream_finished = True
|
| 356 |
+
break
|
| 357 |
+
|
| 358 |
+
# 如果流没有正常结束,发送结束标记
|
| 359 |
+
if not stream_finished:
|
| 360 |
+
final_response = {
|
| 361 |
+
"id": f"chatcmpl-{prediction_id}",
|
| 362 |
+
"object": "chat.completion.chunk",
|
| 363 |
+
"created": int(asyncio.get_event_loop().time()),
|
| 364 |
+
"model": model,
|
| 365 |
+
"choices": [{
|
| 366 |
+
"index": 0,
|
| 367 |
+
"delta": {},
|
| 368 |
+
"finish_reason": "stop"
|
| 369 |
+
}]
|
| 370 |
+
}
|
| 371 |
+
yield f"data: {json.dumps(final_response)}\n\ndata: [DONE]\n\n"
|
| 372 |
|
| 373 |
except Exception as e:
|
| 374 |
logger.error(f"Stream generation error: {e}")
|