Update app.py

app.py

@@ -217,14 +217,14 @@ def get_model(temperature, top_p, repetition_penalty):
             "temperature": temperature,
             "top_p": top_p,
             "repetition_penalty": repetition_penalty,
-            "max_length": 1000
+            "max_length": 800
         },
         huggingfacehub_api_token=huggingface_token
     )
 
-MAX_PROMPT_CHARS = 24000  # Adjust based on your model's limitations
+MAX_PROMPT_CHARS = 20000  # Adjust based on your model's limitations
 
-def chunk_text(text: str, max_chunk_size: int = 1000) -> List[str]:
+def chunk_text(text: str, max_chunk_size: int = 800) -> List[str]:
     chunks = []
     current_chunk = ""
     for sentence in re.split(r'(?<=[.!?])\s+', text):
@@ -244,7 +244,7 @@ def get_most_relevant_chunks(question: str, chunks: List[str], top_k: int = 3) -
     top_indices = np.argsort(similarities)[-top_k:]
     return [chunks[i] for i in top_indices]
 
-def generate_chunked_response(model, prompt, max_tokens=1000, max_chunks=5):
+def generate_chunked_response(model, prompt, max_tokens=800, max_chunks=5):
     full_response = ""
     for i in range(max_chunks):
         try:
@@ -395,8 +395,8 @@ def ask_question(question, temperature, top_p, repetition_penalty, web_search, c
         database = None
 
     max_attempts = 3
-    max_input_tokens = 31000  # Leave room for the model's response
-    max_output_tokens = 1000
+    max_input_tokens = 20000  # Leave room for the model's response
+    max_output_tokens = 800
 
     if web_search:
         contextualized_question, topics, entity_tracker, _ = chatbot.process_question(question)