Update app.py

app.py

@@ -2,7 +2,7 @@ import spaces
 import torch
 import gradio as gr
 
-from fastapi import FastAPI
+from fastapi import FastAPI, Request
 from gradio.routes import mount_gradio_app
 from pydantic import BaseModel
 from transformers import AutoTokenizer, AutoModelForCausalLM
@@ -11,6 +11,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 MODEL = "Qwen/Qwen2.5-Coder-7B-Instruct"
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
+
 model = None
 
 
@@ -19,7 +20,7 @@ def gerar(prompt):
     global model
 
     if model is None:
-        print("Carregando Qwen...")
+        print("Carregando modelo...")
         model = AutoModelForCausalLM.from_pretrained(
             MODEL,
             torch_dtype=torch.float16,
@@ -35,21 +36,28 @@ def gerar(prompt):
     with torch.no_grad():
         saida = model.generate(
             **entrada,
-            max_new_tokens=1024
+            max_new_tokens=1024,
+            temperature=0.2
         )
 
-    return tokenizer.decode(
+    texto = tokenizer.decode(
         saida[0],
         skip_special_tokens=True
     )
 
+    return texto
+
 
-class Chat(BaseModel):
+class ChatRequest(BaseModel):
     model: str
     messages: list
+    temperature: float | None = 0.2
+    max_tokens: int | None = 1024
 
 
-api = FastAPI()
+api = FastAPI(
+    title="Qwen OpenAI Compatible API"
+)
 
 
 @api.get("/status")
@@ -60,15 +68,38 @@ def status():
     }
 
 
+@api.get("/v1/models")
+def models():
+
+    return {
+        "object": "list",
+        "data": [
+            {
+                "id": MODEL,
+                "object": "model",
+                "owned_by": "local"
+            }
+        ]
+    }
+
+
 @api.post("/v1/chat/completions")
-def completions(req: Chat):
+def chat(req: ChatRequest):
+
+    prompt = ""
 
-    prompt = req.messages[-1]["content"]
+    for msg in req.messages:
+        prompt += (
+            msg["role"]
+            + ": "
+            + msg["content"]
+            + "\n"
+        )
 
     resposta = gerar(prompt)
 
     return {
-        "id": "qwen",
+        "id": "chatcmpl-qwen",
         "object": "chat.completion",
         "model": MODEL,
         "choices": [
@@ -84,13 +115,13 @@ def completions(req: Chat):
     }
 
 
-def chat_interface(msg, history):
+def interface(msg, history):
     return gerar(msg)
 
 
 demo = gr.ChatInterface(
-    fn=chat_interface,
-    title="Qwen2.5 Coder ZeroGPU"
+    fn=interface,
+    title="Qwen2.5 Coder Remote AI"
 )
 
 
@@ -98,13 +129,4 @@ app = mount_gradio_app(
     api,
     demo,
     path="/"
-)
-
-
-if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run(
-        app,
-        host="0.0.0.0",
-        port=7860
-    )
+)