app.py

from huggingface_hub import hf_hub_download
from llama_cpp import Llama
from fastapi import FastAPI
from pydantic import BaseModel
import uvicorn

MODEL_PATH = hf_hub_download(
    repo_id="bartowski/Qwen2.5-Coder-3B-Instruct-abliterated-GGUF",
    filename="Qwen2.5-Coder-3B-Instruct-abliterated-Q4_K_M.gguf"
)

llm = Llama(model_path=MODEL_PATH, n_ctx=2048, n_threads=2, chat_format="chatml")

app = FastAPI()

class ChatRequest(BaseModel):
    model: str = "local"
    messages: list
    temperature: float = 0.7
    max_tokens: int = 1024

@app.get("/health")
def health():
    return {"status": "ok"}

@app.post("/v1/chat/completions")
def chat(req: ChatRequest):
    out = llm.create_chat_completion(
        messages=req.messages,
        temperature=req.temperature,
        max_tokens=req.max_tokens
    )
    return out

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=7860)