Create app.py

app.py

@@ -0,0 +1,74 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from llama_cpp import Llama
+import os
+
+# Initialize FastAPI app
+app = FastAPI()
+
+# Load the phi3 model using llama-cpp-python
+model_path = "./Phi-3-mini-4k-instruct-q4.gguf"  # Ensure this path is correct
+llama_model = Llama(
+    model_path=model_path,
+    n_ctx=4096,
+    n_threads=8,
+    n_gpu_layers=35
+)
+
+# Request model based on OpenAI API structure
+class RequestModel(BaseModel):
+    model: str
+    prompt: str
+    max_tokens: int = 256
+    temperature: float = 0.7
+    top_p: float = 1.0
+    n: int = 1
+    stream: bool = False
+    logprobs: int = None
+    stop: list = None
+
+# Response model
+class ResponseModel(BaseModel):
+    id: str
+    object: str
+    created: int
+    model: str
+    choices: list
+
+@app.post("/v1/completions", response_model=ResponseModel)
+def create_completion(request: RequestModel):
+    try:
+        # Generate text using the LLaMA model
+        response = llama_model(
+            prompt=request.prompt,
+            max_tokens=request.max_tokens,
+            temperature=request.temperature,
+            stop=request.stop if request.stop else None,
+            echo=True,
+        )
+        
+        # Extract the generated text
+        generated_text = response["choices"][0]["text"]
+
+        # Build the response in OpenAI's format
+        return ResponseModel(
+            id="cmpl-xxxx",  # Replace with unique ID generator if needed
+            object="text_completion",
+            created=int(os.time()),
+            model=request.model,
+            choices=[
+                {
+                    "text": generated_text,
+                    "index": 0,
+                    "logprobs": None,
+                    "finish_reason": "stop"
+                }
+            ]
+        )
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)