Port to newer version of transformers package

configuration_slimmoe.py

@@ -111,7 +111,7 @@ class PhiMoEConfig(PretrainedConfig):
     >>> configuration = model.config
     ```"""
     
-    model_type = "phimoe"
+    model_type = "phimoe_slim"  # renamed from "phimoe" to bypass transformers >=4.46 conversion_mapping
     keys_to_ignore_at_inference = ["past_key_values"]
 
     def __init__(

modeling_slimmoe.py

@@ -46,7 +46,6 @@ from transformers.utils import (
     logging,
     replace_return_docstrings,
 )
-from transformers.utils.import_utils import is_torch_fx_available
 from .configuration_slimmoe import PhiMoEConfig
 
 from einops import rearrange
@@ -61,11 +60,10 @@ if is_flash_attn_2_available():
 
 # This makes `_prepare_4d_causal_attention_mask` a leaf function in the FX graph.
 # It means that the function will not be traced through and simply appear as a node in the graph.
-if is_torch_fx_available():
-    if not is_torch_greater_or_equal_than_1_13:
-        import torch.fx
+if not is_torch_greater_or_equal_than_1_13:
+    import torch.fx
 
-    _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_mask)
+_prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_mask)
 
 
 logger = logging.get_logger(__name__)
@@ -332,9 +330,17 @@ class PhiMoEAttention(nn.Module):
                 base=self.rope_theta,
             )
         else:
-            scaling_type = self.config.rope_scaling["type"]
+            # "type" key was renamed to "rope_type" in transformers >=4.46; handle both
+            scaling_type = self.config.rope_scaling.get("type") or self.config.rope_scaling.get("rope_type", "")
             if scaling_type == "longrope":
                 self.rotary_emb = Phi3LongRoPEScaledRotaryEmbedding(self.head_dim, self.config)
+            elif not scaling_type or scaling_type == "default":
+                # newer transformers injects {"rope_type": "default"} when rope_scaling is absent
+                self.rotary_emb = PhiMoERotaryEmbedding(
+                    self.head_dim,
+                    max_position_embeddings=self.max_position_embeddings,
+                    base=self.rope_theta,
+                )
             else:
                 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")