update

app.py

@@ -1,5 +1,6 @@
 from fastapi import FastAPI
 from fastapi.responses import StreamingResponse
+from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
 import asyncio
 from fastapi.middleware.cors import CORSMiddleware
@@ -14,13 +15,14 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-# Load model from local file
-model_path = "./unsloth.Q4_K_M.gguf"
-llm = Llama(
-    model_path=model_path,
-    n_ctx=2048,
-    n_batch=512,
-    verbose=False
+# Download the GGUF file
+model_id = "muhammadnoman76/cortex_q4"
+gguf_filename = "unsloth.Q4_K_M.gguf"  # Replace with the correct filename
+model_path = hf_hub_download(
+    repo_id=model_id,
+    filename=gguf_filename,
+    local_dir=".", 
+    local_dir_use_symlinks=False  
 )
 
 alpaca_prompt = """
@@ -49,6 +51,14 @@ Important notes:
 ### Response:
 """
 
+# Load model from local file in the copied folder
+llm = Llama(
+    model_path= r'.//unsloth.Q4_K_M.gguf',
+    n_ctx=2048,
+    n_batch=512,
+    verbose=False
+)
+
 async def stream_llm_response(task_description: str):
     prompt = alpaca_prompt.format(task_description)
     stream = llm(
@@ -67,4 +77,4 @@ async def stream_response(task: str = "make an agent which send mail by searchin
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=7860)
+    uvicorn.run(app, host="0.0.0.0", port=8000)