feat: Switch from TinyLlama to Phi-3-mini for larger context window

- Replace TinyLlama 1.1B (2K context) with Phi-3-mini (4K context)
- Allows MAX_CONTEXT_TOKENS up to 10,000 (was limited to 800)
- Phi-3-mini: 2.4GB, faster, better quality
- Use Phi-3 chat format with <|user|> and <|assistant|> tags
- Increase max_tokens from 1000 to 2000 for better answers
- All within HF Spaces constraints (16GB RAM, 2CPU)"

api/generate_response.py

@@ -12,20 +12,21 @@ llm = None
 def load_model_at_startup():
     global llm
     try:
-        logger.info("Loading model into RAM...")
+        logger.info("Loading Phi-3-mini model into RAM...")
 
         llm = Llama.from_pretrained(
-            repo_id="TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF",
-            filename="tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
+            repo_id="microsoft/Phi-3-mini-4k-instruct-gguf",
+            filename="Phi-3-mini-4k-instruct-Q4_K_M.gguf",
             verbose=False,
             n_gpu_layers=0,  # CPU only (safe for HF Spaces)
-            n_ctx=2048,
+            n_ctx=4096,
         )
-        logger.info("Model loaded into RAM successfully.")
+        logger.info("Phi-3-mini model loaded into RAM successfully.")
 
     except Exception as e:
         logger.error("Failed to load model: %s", e)
         llm = None
+
 def generate_response(query: str, context: str) -> str:
 
     if llm is None:
@@ -37,12 +38,18 @@ def generate_response(query: str, context: str) -> str:
     try:
         answer = llm(
             f"[SYSTEM]{SYSTEM_PROMPT}[/SYSTEM]\n{prompt}",
-            max_tokens=7000,
+            max_tokens=2000,
             temperature=1.0,
             top_p=1.0,
-            stop=["Question:", "Context:"]
+            stop=["<|end|>", "Question:", "<|user|>"],
+            echo=False
         )
         answer = answer["choices"][0]["text"].strip()
+
+        if not answer:
+            logger.warning("Failed to generate response. Returning empty response.")
+            return "I couldn't generate response. Please try again."
+
         logging.info('Answer Generation Succeeded.')
         return answer
 

config.py

@@ -15,7 +15,7 @@ RETRIEVED_TRANSCRIPTS_FILE = Path("outputs/retrieved_transcripts.txt")
 RESPONSE_FILE = Path("outputs/generated_response.txt")
 COOKIES_FILE = Path("utils/youtube_cookies.txt")
 
-MAX_CONTEXT_TOKENS = 7000
+MAX_CONTEXT_TOKENS = 10000
 
 SYSTEM_PROMPT = """
 You are speaking as Acharya Prashant.