Upload app/rag/utils.py with huggingface_hub

app/rag/utils.py

@@ -38,7 +38,7 @@ def get_llm():
             logger.info(f"[LLM] Loading {MODEL_PATH} ...")
             _llm = Llama(
                 model_path=MODEL_PATH,
-                n_ctx=2048,          # ✅ FIX: was 1024 — too small, prompt was being cut
+                n_ctx=4096,          
                 n_threads=os.cpu_count() or 4,
                 n_gpu_layers=0,
                 verbose=False,
@@ -72,7 +72,7 @@ SYSTEM_PROMPT = (
 )
 
 
-def build_context(hits: List[Dict[str, Any]], max_chars: int = 2000) -> str:
+def build_context(hits: List[Dict[str, Any]], max_chars: int = 3000) -> str:
     parts = []
     total = 0
     for i, hit in enumerate(hits, 1):
@@ -105,8 +105,8 @@ def generate_answer(
     question: str,
     hits: List[Dict[str, Any]],
     max_tokens: int = 400,
-    temperature: float = 0.2,
-    top_p: float = 0.9,
+    temperature: float = 0.1,
+    top_p: float = 0.95,
     presence_penalty: float = 0.0,
     frequency_penalty: float = 0.0,
     repeat_penalty: float = 1.1,
@@ -147,11 +147,11 @@ def generate_answer(
 def answer_query(
     searcher,
     query: str,
-    top_k: int = 3,
+    top_k: int = 4,
     rerank: bool = True,
     max_tokens: int = 400,
-    temperature: float = 0.2,
-    top_p: float = 0.9,
+    temperature: float = 0.1,
+    top_p: float = 0.95,
     presence_penalty: float = 0.0,
     frequency_penalty: float = 0.0,
 ) -> Dict[str, Any]:
@@ -188,11 +188,11 @@ def answer_query(
 async def answer_query_async(
     searcher,
     query: str,
-    top_k: int = 3,
+    top_k: int = 4,
     rerank: bool = True,
     max_tokens: int = 400,
-    temperature: float = 0.2,
-    top_p: float = 0.9,
+    temperature: float = 0.1,
+    top_p: float = 0.95,
     presence_penalty: float = 0.0,
     frequency_penalty: float = 0.0,
 ) -> Dict[str, Any]:
@@ -209,7 +209,6 @@ async def answer_query_async(
         print(hit["text"])
         print("-" * 80)
 
-    # ✅ FIX — lambda inapeleka ALL parameters vizuri
     # Kabla top_p na repeat_penalty hazikupelekwa — ndio sababu jibu lilibadilika
     answer = await loop.run_in_executor(
         None,