Update app.py

app.py

@@ -1,74 +1,99 @@
-import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch
-import json
+import os
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import StreamingResponse, JSONResponse
+from pydantic import BaseModel, Field
+from typing import List, Optional, Union, Dict, Any
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+import time
 
-# 全局变量，避免重复加载
-model = None
-tokenizer = None
+app = FastAPI(title="OpenAI Compatible API for OpenClaw")
 
-def load_model():
-    """加载模型和分词器"""
-    global model, tokenizer
-    model_name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
-    try:
-        tokenizer = AutoTokenizer.from_pretrained(model_name)
-        model = AutoModelForCausalLM.from_pretrained(
-            model_name,
-            torch_dtype=torch.float16,  # 使用半精度减少内存占用
-            device_map="auto",         # 自动分配至CPU
-            low_cpu_mem_usage=True     # 优化CPU内存使用
-        )
-        print("模型加载成功！")
-    except Exception as e:
-        print(f"模型加载失败: {e}")
+# 配置模型：使用 Qwen2.5 1.5B 4位量化版，CPU 运行极快且仅占约 1.5G 内存
+REPO_ID = "Qwen/Qwen2.5-1.5B-Instruct-GGUF"
+FILENAME = "qwen2.5-1.5b-instruct-q4_k_m.gguf"
+MODEL_NAME = "qwen2.5-1.5b"
 
-def openai_compatible_api(message, history):
-    """处理OpenAI格式的请求"""
-    if model is None:
-        load_model()
+print("正在从 Hugging Face 下载 GGUF 模型文件...")
+model_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME)
+print(os.path.abspath(model_path))
+print("模型下载完成，正在加载至内存...")
 
-    # 将历史记录和当前消息组合成对话格式
-    # 这里需要根据DeepSeek模型要求的模板格式化工序
-    # 以下是一个简化示例，您需要根据模型的具体要求调整
-    prompt = f"\n\nHuman: {message}\n\nAssistant:"
+# 初始化 Llama-cpp (配置上下文长度为 2048 以节省内存)
+llm = Llama(
+    model_path=model_path,
+    n_ctx=2048,
+    n_threads=2, # 免费容器为 2核 CPU
+    chat_format="chatml"
+)
+print("模型加载成功！服务已就绪。")
 
-    inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=1024)
+# --- OpenAI 协议数据结构定义 ---
+class ChatMessage(BaseModel):
+    role: str
+    content: str
 
-    with torch.no_grad():
-        outputs = model.generate(
-            **inputs,
-            max_new_tokens=512,
-            temperature=0.7,
-            top_p=0.95,
-            do_sample=True,
-            pad_token_id=tokenizer.eos_token_id
-        )
+class ChatCompletionRequest(BaseModel):
+    model: str
+    messages: List[ChatMessage]
+    temperature: Optional[float] = 0.7
+    top_p: Optional[float] = 0.9
+    max_tokens: Optional[int] = 512
+    stream: Optional[bool] = False
 
-    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    # 提取模型生成的部分
-    generated_text = response.split("Assistant:")[-1].strip()
+@app.get("/")
+def index():
+    return {"status": "running", "compatible_with": "OpenClaw / OpenAI API"}
 
-    # 返回OpenAI兼容格式
-    return {
-        "choices": [{
-            "message": {
-                "role": "assistant",
-                "content": generated_text
+# 1. 适配 OpenClaw 获取模型列表的接口
+@app.get("/v1/models")
+async def list_models():
+    return JSONResponse({
+        "object": "list",
+        "data": [
+            {
+                "id": MODEL_NAME,
+                "object": "model",
+                "created": int(time.time()),
+                "owned_by": "huggingface"
             }
-        }]
-    }
-
-# 在Gradio界面启动前加载模型（可选，或等待第一个请求时加载）
-load_model()
+        ]
+    })
 
-# 创建Gradio界面，但隐藏默认的Web UI，专注于API
-demo = gr.ChatInterface(
-    fn=openai_compatible_api,
-    title="DeepSeek API Service",
-    description="OpenAI-compatible API for DeepSeek-R1"
-)
-
-# 禁用自动创建公共链接，仅以API模式运行
-if __name__ == "__main__":
-    demo.launch(show_api=True, server_name="0.0.0.0", server_port=7860)
+# 2. 适配 OpenClaw 聊天对话接口 (支持流式和非流式)
+@app.post("/v1/chat/completions")
+async def chat_completions(request: ChatCompletionRequest):
+    # 将标准 OpenAI 消息格式转换为 llama-cpp 接收的格式
+    llama_messages = [{"role": msg.role, "content": msg.content} for msg in request.messages]
+    
+    try:
+        if request.stream:
+            # 流式传输实现
+            chunks = llm.create_chat_completion(
+                messages=llama_messages,
+                temperature=request.temperature,
+                top_p=request.top_p,
+                max_tokens=request.max_tokens,
+                stream=True
+            )
+            def stream_generator():
+                for chunk in chunks:
+                    import json
+                    yield f"data: {json.dumps(chunk)}\n\n"
+                yield "data: [DONE]\n\n"
+            return StreamingResponse(stream_generator(), media_type="text/event-stream")
+        else:
+            # 非流式传输实现
+            response = llm.create_chat_completion(
+                messages=llama_messages,
+                temperature=request.temperature,
+                top_p=request.top_p,
+                max_tokens=request.max_tokens,
+                stream=False
+            )
+            # 强制覆盖返回的模型名称，确保与 OpenClaw 请求的一致
+            response["model"] = MODEL_NAME
+            return JSONResponse(response)
+            
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))