Spaces:
Runtime error
Runtime error
File size: 2,072 Bytes
b491696 a3fefda b491696 a3fefda b491696 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | from fastapi import FastAPI, Request, HTTPException
import uvicorn
from llama_cpp import Llama
app = FastAPI()
print("Loading Llama model...")
llm = Llama(
model_path="model.gguf",
n_ctx=4096,
n_gpu_layers=0,
)
print("Model loaded successfully!")
@app.post("/v1/chat/completions")
async def chat_completions(request: Request):
try:
data = await request.json()
except:
raise HTTPException(status_code=400, detail="Invalid JSON body")
messages = data.get("messages", [])
max_tokens = data.get("max_tokens", 1024)
temperature = data.get("temperature", 0.7)
# Construct Dicta-LM Prompt
prompt = "<s>"
for m in messages:
if m["role"] == "user":
prompt += f"[INST] {m['content']} [/INST]\n"
elif m["role"] == "assistant":
prompt += f"{m['content']}</s> "
import asyncio
try:
response = await asyncio.to_thread(
llm,
prompt=prompt,
max_tokens=max_tokens,
temperature=temperature,
stop=["</s>", "[INST]", "[/INST]"],
echo=False
)
return {
"id": response["id"],
"object": "chat.completion",
"created": response["created"],
"model": "dictalm2.0-instruct",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": response["choices"][0]["text"].strip()
},
"finish_reason": response["choices"][0]["finish_reason"]
}
],
"usage": response["usage"]
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.get("/v1/models")
def get_models():
return {
"data": [
{"id": "dictalm2.0-instruct", "object": "model", "owned_by": "dicta"}
]
}
@app.get("/")
def health_check():
return {"status": "running"}
|