File size: 905 Bytes
7fe1098 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | from huggingface_hub import hf_hub_download
from llama_cpp import Llama
from fastapi import FastAPI
from pydantic import BaseModel
import uvicorn
MODEL_PATH = hf_hub_download(
repo_id="bartowski/Qwen2.5-Coder-3B-Instruct-abliterated-GGUF",
filename="Qwen2.5-Coder-3B-Instruct-abliterated-Q4_K_M.gguf"
)
llm = Llama(model_path=MODEL_PATH, n_ctx=2048, n_threads=2, chat_format="chatml")
app = FastAPI()
class ChatRequest(BaseModel):
model: str = "local"
messages: list
temperature: float = 0.7
max_tokens: int = 1024
@app.get("/health")
def health():
return {"status": "ok"}
@app.post("/v1/chat/completions")
def chat(req: ChatRequest):
out = llm.create_chat_completion(
messages=req.messages,
temperature=req.temperature,
max_tokens=req.max_tokens
)
return out
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=7860) |