Fix response truncation: disable early stopping, increase token limits to 4096, add debugging logs

gradio_app.py

@@ -100,7 +100,7 @@ def chat_with_model(message, history, temperature):
 """
             
             # Generate response using the model directly
-            inputs = model_manager.tokenizer(prompt, return_tensors="pt", truncation=True, max_length=2048)
+            inputs = model_manager.tokenizer(prompt, return_tensors="pt", truncation=True, max_length=4096)
             
             # Force all inputs to the same device as the model
             if model_manager.device == "cuda:0":
@@ -110,13 +110,15 @@ def chat_with_model(message, history, temperature):
             with torch.no_grad():
                 outputs = model_manager.model.generate(
                     **inputs,
-                    max_new_tokens=2048,
+                    max_new_tokens=4096,
                     temperature=temperature,
                     top_p=0.95,
                     do_sample=True,
                     num_beams=1,
                     pad_token_id=model_manager.tokenizer.eos_token_id,
-                    early_stopping=True
+                    eos_token_id=model_manager.tokenizer.eos_token_id,
+                    early_stopping=False,  # Disable early stopping to prevent premature truncation
+                    repetition_penalty=1.1  # Add slight repetition penalty to improve quality
                 )
             
             # Decode the generated text and remove the input prompt
@@ -128,6 +130,7 @@ def chat_with_model(message, history, temperature):
                 # Find the position after the assistant header
                 response_start = full_text.find(assistant_start) + len(assistant_start)
                 response = full_text[response_start:].strip()
+                logger.info(f"Extracted response length: {len(response)}")
             else:
                 # Fallback: try to remove the original prompt
                 try:
@@ -135,6 +138,10 @@ def chat_with_model(message, history, temperature):
                 except:
                     response = full_text.strip()
             
+            # Check if response ends abruptly (might indicate truncation)
+            if response and not response.endswith(('.', '!', '?', ':', ';')):
+                logger.warning(f"Response may be truncated - ends with: '{response[-20:]}'")
+            
             if not response:
                 response = "I couldn't generate a response. Please try a different prompt."