Create app.py

app.py

@@ -0,0 +1,73 @@
+import os
+from fastapi import FastAPI
+from pydantic import BaseModel
+from typing import List
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
+app = FastAPI(title="Qwen 1.5B API")
+
+model_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+# Initialize model directly onto CPU
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype=torch.float32,
+    device_map="cpu"
+)
+pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+class ChatPayload(BaseModel):
+    messages: List[Message]
+    temperature: float = 0.7
+    max_tokens: int = 512
+
+@app.post("/v1/chat/completions")
+async def chat_completion(payload: ChatPayload):
+    try:
+        # Convert incoming array to ChatML format
+        formatted_messages = [{"role": msg.role, "content": msg.content} for msg in payload.messages]
+        
+        formatted_prompt = tokenizer.apply_chat_template(
+            formatted_messages, 
+            tokenize=False, 
+            add_generation_prompt=True
+        )
+        
+        outputs = pipe(
+            formatted_prompt,
+            max_new_tokens=payload.max_tokens,
+            do_sample=True,
+            temperature=payload.temperature,
+            pad_token_id=tokenizer.eos_token_id
+        )
+        
+        generated_text = outputs[0]["generated_text"]
+        response_content = generated_text[len(formatted_prompt):].strip()
+        
+        # Mirror the clean OpenAI JSON standard response
+        return {
+            "choices": [
+                {
+                    "message": {
+                        "role": "assistant",
+                        "content": response_content
+                    }
+                }
+            ]
+        }
+    except Exception as e:
+        return {"error": str(e)}
+
+@app.get("/")
+async def health():
+    return {"status": "online"}
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)