Rotary embedding correction

modeling_phi.py

@@ -414,10 +414,12 @@ class MHA(nn.Module):
         super().__init__()
 
         # Rotary embedding
-        self.rotary_emb = RotaryEmbedding(
-            d_rotary=math.ceil((rotary_dim // n_head) / 2),  # d_rotary is half of d_head
-            initial_cos_sin_cache_len=config.n_positions,
-        )
+        self.rotary_dim = rotary_dim if rotary_dim is not None else getattr(config, "rotary_dim", 0)
+        if self.rotary_dim > 0:
+            self.rotary_emb = RotaryEmbedding(
+                d_rotary=math.ceil((rotary_dim // n_head) / 2),  # d_rotary is half of d_head
+                initial_cos_sin_cache_len=config.n_positions,
+            )
 
         # MLP
         self.n_head, self.n_head_kv, self.head_dim = _find_mha_dims(