File size: 1,728 Bytes
bd0f00b 7ea6195 d28579a bd0f00b 7ea6195 d28579a bd0f00b 7ea6195 bd0f00b d28579a 7ea6195 d28579a bd0f00b 7ea6195 bd0f00b 0dd7ad6 bd0f00b 7ea6195 bd0f00b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | import os
from llama_cpp import Llama
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
import uvicorn
app = FastAPI()
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
print("Loading Qwen 2.5 Coder 7B (4 GB)...", flush=True)
llm = Llama(
model_path="/app/models/qwen2.5-coder-7b-instruct-q4_k_m.gguf",
n_ctx=2048,
n_threads=4,
n_gpu_layers=0,
verbose=False,
)
print("Qwen 7B loaded! Ready for inference.", flush=True)
class ChatRequest(BaseModel):
messages: list
max_tokens: int = 200
temperature: float = 0.7
@app.post("/v1/chat/completions")
async def chat(req: ChatRequest):
prompt = ""
for msg in req.messages:
role = msg.get("role", "user")
content = msg.get("content", "")
if role == "user":
prompt += f"<|im_start|>user\n{content}<|im_end|>\n"
elif role == "system":
prompt += f"<|im_start|>system\n{content}<|im_end|>\n"
elif role == "assistant":
prompt += f"<|im_start|>assistant\n{content}<|im_end|>\n"
prompt += "<|im_start|>assistant\n"
output = llm(prompt, max_tokens=req.max_tokens, temperature=req.temperature, stop=["<|im_end|>"])
text = output["choices"][0]["text"]
return {
"choices": [{"message": {"role": "assistant", "content": text}}],
"model": "qwen2.5-coder-7b",
}
@app.get("/")
async def root():
return {"status": "ready", "model": "Qwen 2.5 Coder 7B", "humaneval": "65%"}
@app.get("/health")
async def health():
return {"status": "healthy"}
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=7860)
|