Update app/app.py

app/app.py

@@ -25,17 +25,16 @@ else:
     print("[INFO] Vector DB ready.")
 
 # -----------------------------
-# ✅ Load GGUF Model with llama-cpp-python
+# ✅ Load GGUF Model with llama-cpp-python (model is pre-downloaded in Dockerfile)
 # -----------------------------
 MODEL_PATH = "/app/tinyllama_dop_q4_k_m.gguf"
 
-
 print(f"[INFO] Loading GGUF model from: {MODEL_PATH}")
 llm = Llama(
     model_path=MODEL_PATH,
-    n_ctx=2048,
-    n_threads=4,  # You can tune this based on CPU cores
-    n_batch=64,
+    n_ctx=1024,
+    n_threads=2,
+    n_batch=32,
     use_mlock=False,
     verbose=False
 )
@@ -58,7 +57,7 @@ async def chat(query: Query):
 
     prompt = f"""### Context:\n{context}\n\n### Question: {question}\n### Answer:"""
 
-    response = llm(prompt, max_tokens=200, stop=["###"])
+    response = llm(prompt, max_tokens=150, stop=["###"])
     answer = response["choices"][0]["text"].strip()
 
     return {"answer": answer}