max token arttılırdı

handler.py

@@ -351,9 +351,11 @@ def generate_response(message_text, image_input, max_output_tokens=4096, repetit
         try:
             if hasattr(our_chatbot, 'conv_mode') and our_chatbot.conv_mode and LLAVA_AVAILABLE:
                 our_chatbot.conversation = conv_templates[our_chatbot.conv_mode].copy()
+                print(f"[DEBUG] Reset conversation using conv_mode: {our_chatbot.conv_mode}")
             else:
                 # Use default conversation template
                 our_chatbot.conversation = our_chatbot.conversation.__class__()
+                print(f"[DEBUG] Reset conversation using default template")
         except Exception as e:
             print(f"[DEBUG] Failed to reset conversation: {e}")
             # Continue with existing conversation
@@ -363,6 +365,10 @@ def generate_response(message_text, image_input, max_output_tokens=4096, repetit
         our_chatbot.conversation.append_message(our_chatbot.conversation.roles[1], None)
         prompt = our_chatbot.conversation.get_prompt()
         
+        print(f"[DEBUG] Conversation template: {type(our_chatbot.conversation).__name__}")
+        print(f"[DEBUG] Conversation roles: {our_chatbot.conversation.roles if hasattr(our_chatbot.conversation, 'roles') else 'No roles'}")
+        print(f"[DEBUG] Final prompt length: {len(prompt)} characters")
+        
         # Tokenize input
         input_ids = tokenizer_image_token(
             prompt, our_chatbot.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt"
@@ -370,6 +376,8 @@ def generate_response(message_text, image_input, max_output_tokens=4096, repetit
         
         # No stopping criteria - let model generate freely up to max_new_tokens
         print(f"[DEBUG] No stopping criteria - free generation up to {max_output_tokens} tokens")
+        print(f"[DEBUG] Input prompt length: {len(prompt)} characters")
+        print(f"[DEBUG] Input tokens: {input_ids.shape[1]} tokens")
         stopping_criteria = None
         
         # Set seed for deterministic generation
@@ -381,6 +389,10 @@ def generate_response(message_text, image_input, max_output_tokens=4096, repetit
         
         # Generate response using deterministic greedy decoding
         # This eliminates randomness and ensures consistent responses
+        print(f"[DEBUG] About to generate with max_new_tokens: {max_output_tokens}")
+        print(f"[DEBUG] Model device: {our_chatbot.model.device}")
+        print(f"[DEBUG] Image tensor device: {image_tensor.device}")
+        
         with torch.no_grad():
             outputs = our_chatbot.model.generate(
                 inputs=input_ids,
@@ -392,6 +404,7 @@ def generate_response(message_text, image_input, max_output_tokens=4096, repetit
                 pad_token_id=our_chatbot.tokenizer.eos_token_id,
                 eos_token_id=our_chatbot.tokenizer.eos_token_id,
                 length_penalty=1.0,  # Don't penalize longer sequences
+                early_stopping=False,  # Ensure no early stopping
             )
         
         # Decode response
@@ -399,6 +412,8 @@ def generate_response(message_text, image_input, max_output_tokens=4096, repetit
             print(f"[DEBUG] Outputs shape: {outputs.shape if hasattr(outputs, 'shape') else 'No shape attr'}")
             print(f"[DEBUG] Outputs length: {len(outputs) if hasattr(outputs, '__len__') else 'No length'}")
             print(f"[DEBUG] Input IDs shape: {input_ids.shape}")
+            print(f"[DEBUG] Generated tokens: {outputs.shape[1] - input_ids.shape[1] if hasattr(outputs, 'shape') else 'Unknown'}")
+            print(f"[DEBUG] Expected max tokens: {max_output_tokens}")
             
             if len(outputs) == 0:
                 return {"error": "Model generated empty output"}
@@ -509,7 +524,7 @@ def initialize_model():
                 self.model_base = None
                 self.num_gpus = 1
                 self.conv_mode = None
-                self.max_new_tokens = 1024
+                self.max_new_tokens = 4096
                 self.num_frames = 16
                 self.load_8bit = False
                 self.load_4bit = False
@@ -589,6 +604,16 @@ def query(payload):
         repetition_penalty = float(payload.get("repetition_penalty", 1.0))
         conv_mode_override = payload.get("conv_mode", None)
         
+        # Debug: Log all generation parameters
+        print(f"[DEBUG] Generation parameters:")
+        print(f"[DEBUG]   max_output_tokens: {max_output_tokens}")
+        print(f"[DEBUG]   repetition_penalty: {repetition_penalty}")
+        print(f"[DEBUG]   Original payload max_output_tokens: {payload.get('max_output_tokens')}")
+        print(f"[DEBUG]   Original payload max_new_tokens: {payload.get('max_new_tokens')}")
+        print(f"[DEBUG]   Original payload max_tokens: {payload.get('max_tokens')}")
+        print(f"[DEBUG] Full payload keys: {list(payload.keys())}")
+        print(f"[DEBUG] Payload values: {dict(payload)}")
+        
         if not message_text or not message_text.strip():
             return {"error": "Missing prompt text. Use 'message', 'query', 'prompt', or 'istem' key"}