Changes

app.py

@@ -3,7 +3,7 @@ from typing import List, Union, Dict, Tuple
 from transformers import pipeline 
 from os import getenv
 from huggingface_hub import login 
-import pymupdf  # PyMuPDF
+import pymupdf   
 
 from history import get_history, update_history
 
@@ -41,13 +41,14 @@ def extract_text_from_pdf(pdf_path):
 
 
 #Does the actual inference and streams (yield) the response
-def chat(history:list[dict[str, str]],temperature,top_p,max_tokens):
+def chat(history:list[dict[str, str]],temperature,top_p,max_tokens,top_k):
     for msg in generator(
         history, #message list
         max_new_tokens=max_tokens,
         return_full_text=False,
         temperature=temperature,
         top_p=top_p,
+        top_k=top_k
         #max_tokens=max_tokens
     ): 
         yield msg['generated_text']
@@ -92,7 +93,13 @@ def respond(
         mainMessage=[{"role": "system", "content": system_message}, *messages] 
 
         #calls the inference function and streams the response
-        for msg in chat(mainMessage,temperature=temperature,top_p=top_p,max_tokens=max_tokens):
+        for msg in chat(
+            mainMessage,
+            temperature=temperature,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            top_k=12
+            ):
             token = msg
 
             # This is a stream. Meaning response comes in bits of string.