update handler

handler.py

@@ -3,30 +3,35 @@ import json
 import torch
 from transformers import pipeline, AutoTokenizer, AutoConfig
 from typing import Dict, List, Any, Optional, Union
+import functools
 
 class EndpointHandler:
     def __init__(self, path=""):
         # Initialize model and tokenizer
         self.model_path = path if path else os.environ.get("MODEL_PATH", "")
         
-        # Fix RoPE scaling configuration
+        # Monkey patch the RoPE scaling validation to bypass the length check
         try:
-            config = AutoConfig.from_pretrained(self.model_path)
+            from transformers.models.phi3.configuration_phi3 import Phi3Config
+            original_validation = Phi3Config._rope_scaling_validation
             
-            # Check if config has rope_scaling attribute and fix the short_factor length
-            if hasattr(config, "rope_scaling") and "short_factor" in config.rope_scaling:
-                short_factor = config.rope_scaling["short_factor"]
-                if len(short_factor) == 48:  # If we have the problematic length
-                    print("Fixing rope_scaling short_factor length from 48 to 64")
-                    # Pad to length 64
-                    padded_short_factor = list(short_factor) + [0.0] * (64 - len(short_factor))
-                    config.rope_scaling["short_factor"] = padded_short_factor
-                    
-                    # Save the fixed config
-                    config.save_pretrained(self.model_path)
-                    print("Fixed config saved")
+            # Create a patched version that doesn't validate length
+            @functools.wraps(original_validation)
+            def patched_validation(self_config):
+                # Skip validation if short_factor length is 48
+                if (hasattr(self_config, "rope_scaling") and 
+                    "short_factor" in self_config.rope_scaling and 
+                    len(self_config.rope_scaling["short_factor"]) == 48):
+                    print("Bypassing RoPE scaling validation for short_factor of length 48")
+                    return
+                # Otherwise call the original validation
+                return original_validation(self_config)
+            
+            # Apply the monkey patch
+            Phi3Config._rope_scaling_validation = patched_validation
+            print("Successfully patched RoPE scaling validation")
         except Exception as e:
-            print(f"Warning: Could not fix RoPE scaling configuration: {str(e)}")
+            print(f"Warning: Could not patch RoPE scaling validation: {str(e)}")
         
         # Load tokenizer
         self.tokenizer = AutoTokenizer.from_pretrained(self.model_path)