Fix CoT truncation: increase min_new_tokens to 1000, add generation logging, improve truncated JSON handling

gradio_app.py

@@ -106,8 +106,8 @@ def generate_response(prompt, temperature=0.8):
         
         # Set minimum tokens based on request type
         if is_cot_request:
-            min_tokens = 500  # Higher minimum for CoT to ensure complete responses
-            logger.info("Detected Chain of Thinking request - using min_new_tokens=500")
+            min_tokens = 1000  # Much higher minimum for CoT to ensure complete responses
+            logger.info("Detected Chain of Thinking request - using min_new_tokens=1000")
         else:
             min_tokens = 200  # Standard minimum
 
@@ -147,6 +147,15 @@ def generate_response(prompt, temperature=0.8):
         # Decode the response
         generated_text = model_manager.tokenizer.decode(outputs[0], skip_special_tokens=True)
         
+        # Log generation details for debugging
+        input_length = inputs['input_ids'].shape[1]
+        output_length = outputs[0].shape[0]
+        generated_length = output_length - input_length
+        logger.info(f"Generation stats - Input: {input_length} tokens, Generated: {generated_length} tokens, Min required: {min_tokens}")
+        
+        if generated_length < min_tokens:
+            logger.warning(f"Generated {generated_length} tokens but minimum was {min_tokens} - response may be truncated")
+        
         # Post-decode guard: if a top-level JSON array closes, trim to the first full array
         # This helps prevent trailing prose like 'assistant' or 'Message'.
         try:
@@ -194,6 +203,11 @@ def generate_response(prompt, temperature=0.8):
                 json_text = generated_text[start_idx:end_idx+1]
                 logger.info(f"Extracted complete JSON array of length {len(json_text)}")
                 generated_text = json_text
+            elif start_idx is not None:
+                # Found start but no end - response was truncated
+                logger.warning("JSON array started but never closed - response truncated")
+                # Try to extract what we have and let the client handle it
+                generated_text = generated_text[start_idx:]
         except Exception as e:
             logger.warning(f"Error in JSON extraction: {e}")
             pass