Create app.py

app.py

@@ -0,0 +1,36 @@
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+from fastapi import FastAPI
+from pydantic import BaseModel
+import uvicorn
+
+MODEL_PATH = hf_hub_download(
+    repo_id="bartowski/Qwen2.5-Coder-3B-Instruct-abliterated-GGUF",
+    filename="Qwen2.5-Coder-3B-Instruct-abliterated-Q4_K_M.gguf"
+)
+
+llm = Llama(model_path=MODEL_PATH, n_ctx=2048, n_threads=2, chat_format="chatml")
+
+app = FastAPI()
+
+class ChatRequest(BaseModel):
+    model: str = "local"
+    messages: list
+    temperature: float = 0.7
+    max_tokens: int = 1024
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
+@app.post("/v1/chat/completions")
+def chat(req: ChatRequest):
+    out = llm.create_chat_completion(
+        messages=req.messages,
+        temperature=req.temperature,
+        max_tokens=req.max_tokens
+    )
+    return out
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=7860)