Upload folder using huggingface_hub

config.json

@@ -1,40 +1,40 @@
 {
   "architectures": [
-    "SarvamMoEForCausalLM"
+    "SarvamMLAForCausalLM"
   ],
   "attention_dropout": 0.0,
   "attn_implementation": null,
   "auto_map": {
-    "AutoConfig": "configuration_sarvam_moe.SarvamMoEConfig",
-    "AutoModel": "modeling_sarvam_moe.SarvamMoEModel",
-    "AutoModelForCausalLM": "modeling_sarvam_moe.SarvamMoEForCausalLM"
+    "AutoConfig": "configuration_sarvam_moe.SarvamMLAConfig",
+    "AutoModel": "modeling_sarvam_moe.SarvamMLAModel",
+    "AutoModelForCausalLM": "modeling_sarvam_moe.SarvamMLAForCausalLM"
   },
+  "default_theta": 10000.0,
   "dtype": "float32",
   "embedding_dropout": 0.0,
   "eos_token_id": 1,
   "first_k_dense_replace": 1,
-  "head_dim": 64,
+  "head_dim": 576,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.006,
-  "intermediate_size": 8192,
+  "intermediate_size": 16384,
+  "kv_lora_rank": 512,
   "max_position_embeddings": 131072,
-  "max_window_layers": 19,
-  "model_type": "sarvam_moe",
-  "moe_intermediate_size": 1024,
+  "model_type": "sarvam_mla",
+  "moe_intermediate_size": 2048,
   "moe_router_enable_expert_bias": true,
-  "moe_shared_expert_intermediate_size": 1024,
-  "n_group": 1,
-  "norm_topk_prob": true,
   "num_attention_heads": 64,
   "num_experts": 128,
-  "num_experts_per_tok": 6,
-  "num_hidden_layers": 19,
-  "num_key_value_heads": 4,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 32,
   "num_shared_experts": 1,
   "output_dropout": 0.0,
   "output_router_logits": false,
   "pad_token_id": 0,
+  "q_head_dim": 192,
+  "qk_nope_head_dim": 128,
+  "qk_rope_head_dim": 64,
   "quantization_config": {
     "config_groups": {
       "group_0": {
@@ -86,18 +86,21 @@
     "version": "0.14.0"
   },
   "rms_norm_eps": 1e-06,
-  "rope_scaling": null,
-  "rope_theta": 8000000,
+  "rope_scaling": {
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "factor": 40,
+    "mscale": 1.0,
+    "mscale_all_dim": 1.0,
+    "original_max_position_embeddings": 4096,
+    "type": "deepseek_yarn"
+  },
+  "rope_theta": 10000.0,
   "routed_scaling_factor": 2.5,
-  "router_dtype": "fp32",
-  "score_function": "sigmoid",
   "tie_word_embeddings": false,
-  "topk_group": 1,
   "transformers_version": "4.56.2",
-  "use_bias": false,
   "use_cache": true,
   "use_qk_norm": true,
-  "use_qkv_bias": false,
-  "use_rmsnorm": true,
+  "v_head_dim": 128,
   "vocab_size": 262144
 }

configuration_sarvam_moe.py

@@ -1,99 +1,120 @@
 from transformers.configuration_utils import PretrainedConfig
 
 
-class SarvamMoEConfig(PretrainedConfig):
-    model_type = "sarvam_moe"
+class SarvamMLAConfig(PretrainedConfig):
+    model_type = "sarvam_mla"
+
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.kv_b_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+    }
+
     def __init__(
         self,
-        vocab_size=262144,
-        hidden_size=4096,
-        intermediate_size=8192,
-        num_hidden_layers=19,
-        num_attention_heads=16,
-        num_key_value_heads=4,
-        hidden_act="silu",
-        use_qkv_bias=False,
-        use_bias=False,
-        rms_norm_eps=1e-06,
-        tie_word_embeddings=False,
-        embedding_dropout=0.0,
-        attention_dropout=0.0,
-        output_dropout=0.0,
-        initializer_range=0.006,
-        max_position_embeddings=4096,
-        rope_theta=10000.0,
-        use_cache=True,
-        max_window_layers=19,
-        rope_scaling=None,
-        pad_token_id=0,
-        eos_token_id=1,
-        num_experts=128,
-        num_shared_experts=1,
-        num_experts_per_tok=6,
-        n_group=1,
-        topk_group=1,
-        moe_intermediate_size=1024,
-        first_k_dense_replace=1,
-        head_dim=256,
-        output_router_logits=False,
-        use_qk_norm=True,
-        moe_router_enable_expert_bias=True,
-        routed_scaling_factor=2.5,
+        vocab_size: int = 262144,
+        hidden_size: int = 4096,
+        num_hidden_layers: int = 32,
+        intermediate_size: int = 16384,
+        moe_intermediate_size: int = 2048,
+        num_experts: int = 128,
+        num_experts_per_tok: int = 8,
+        num_shared_experts: int = 1,
+        first_k_dense_replace: int = 1,
+        num_attention_heads: int = 64,
+        qk_rope_head_dim: int = 64,
+        qk_nope_head_dim: int = 128,
+        kv_lora_rank: int = 512,
+        v_head_dim: int = 128,
+        max_position_embeddings: int = 4096,
+        rope_theta: float = 10000.0,
+        rope_scaling: dict = None,
+        attention_dropout: float = 0.0,
+        output_dropout: float = 0.0,
+        rms_norm_eps: float = 1e-6,
+        hidden_act: str = "silu",
+        use_cache: bool = True,
+        use_qk_norm: bool = True,
+        moe_router_enable_expert_bias: bool = True,
+        routed_scaling_factor: float = 2.5,
+        output_router_logits: bool = False,
+        tie_word_embeddings: bool = False,
+        pad_token_id: int = 0,
+        eos_token_id: int = 1,
+        embedding_dropout: float = 0.0,
+        initializer_range: float = 0.006,
         attn_implementation: str = "eager",
         **kwargs,
     ):
-        self.num_hidden_layers = num_hidden_layers
+        # core geometry
         self.vocab_size = vocab_size
         self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
         self.intermediate_size = intermediate_size
         self.num_attention_heads = num_attention_heads
-        self.num_key_value_heads = num_key_value_heads
-        self.hidden_act = hidden_act
-        self.use_qkv_bias = use_qkv_bias
-        self.use_bias = use_bias
-        self.rms_norm_eps = rms_norm_eps
-        self.embedding_dropout = embedding_dropout
+        self.max_position_embeddings = max_position_embeddings
+
+        # MLA geometry
+        self.qk_rope_head_dim = qk_rope_head_dim
+        self.qk_nope_head_dim = qk_nope_head_dim
+        self.kv_lora_rank = kv_lora_rank
+        self.v_head_dim = v_head_dim
+        # convenient derived dim
+        self.q_head_dim = qk_rope_head_dim + qk_nope_head_dim
+        # vLLM MLA expects "head size" = Lkv + R, not hidden_size/num_heads.
+        self.head_dim = int(self.kv_lora_rank + self.qk_rope_head_dim)
+
+        # MoE
+        self.moe_intermediate_size = moe_intermediate_size
+        self.num_experts = num_experts
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_shared_experts = num_shared_experts
+        self.first_k_dense_replace = first_k_dense_replace
+
+        # Router
+        self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
+        self.routed_scaling_factor = routed_scaling_factor
+        self.output_router_logits = output_router_logits
+
+        # dropouts / norms / init
         self.attention_dropout = attention_dropout
         self.output_dropout = output_dropout
+        self.embedding_dropout = embedding_dropout
+        self.rms_norm_eps = rms_norm_eps
         self.initializer_range = initializer_range
-        self.max_position_embeddings = max_position_embeddings
+        self.hidden_act = hidden_act
+
+        # rope / cache
         self.rope_theta = rope_theta
         self.use_cache = use_cache
-        self.max_window_layers = max_window_layers
-        self.head_dim = head_dim or hidden_size // num_attention_heads
-        self.rope_scaling = rope_scaling
         self.use_qk_norm = use_qk_norm
-        self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
-        self.routed_scaling_factor = routed_scaling_factor
-        self.num_experts = num_experts
-        self.num_shared_experts = num_shared_experts
-        self.num_experts_per_tok = num_experts_per_tok
-        self.n_group = n_group
-        self.topk_group = topk_group
-        self.moe_intermediate_size = moe_intermediate_size
-        self.first_k_dense_replace = first_k_dense_replace
-        self.output_router_logits = output_router_logits
+        self.rope_scaling = rope_scaling
+        self.default_theta = 10000.0
+        
+        if self.rope_scaling is None:
+            self.rope_scaling = {
+                'beta_fast': 32,
+                'beta_slow': 1,
+                'factor': 40,
+                'mscale': 1.0,
+                'mscale_all_dim': 1.0,
+                'original_max_position_embeddings': 4096,
+                'rope_type': 'deepseek_yarn',
+            }
+
         self.attn_implementation = attn_implementation
         self._attn_implementation = attn_implementation
-
-        self.base_model_tp_plan = {
-            "layers.*.attention.query_key_value": "colwise",
-            "layers.*.attention.dense": "rowwise",
-            "layers.*.mlp.gate_proj": "colwise",
-            "layers.*.mlp.up_proj": "colwise",
-            "layers.*.mlp.down_proj": "rowwise",
-            "layers.*.mlp.experts.*.gate_proj": "colwise",
-            "layers.*.mlp.experts.*.up_proj": "colwise",
-            "layers.*.mlp.experts.*.down_proj": "rowwise",
-            "layers.*.mlp.shared_experts.gate_proj": "colwise",
-            "layers.*.mlp.shared_experts.up_proj": "colwise",
-            "layers.*.mlp.shared_experts.down_proj": "rowwise",
-        }
-        self.base_model_pp_plan = {
-            "word_embeddings": (["input_ids"], ["inputs_embeds"]),
-            "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
-            "norm": (["hidden_states"], ["hidden_states"]),
-        }
+        
+        if "_attn_implementation" in kwargs:
+            self._attn_implementation = kwargs.pop("_attn_implementation")
+            if hasattr(self, "attn_implementation"):
+                self.attn_implementation = self._attn_implementation
 
         super().__init__(
             pad_token_id=pad_token_id,
@@ -101,3 +122,19 @@ class SarvamMoEConfig(PretrainedConfig):
             tie_word_embeddings=tie_word_embeddings,
             **kwargs,
         )
+
+    def convert_rope_params_to_dict(self, ignore_keys_at_rope_validation: set | None = None, **kwargs):
+        rope_scaling = kwargs.pop("rope_scaling", None)
+        self.rope_parameters = rope_scaling or self.rope_parameters
+        self.rope_parameters = self.rope_parameters if self.rope_parameters is not None else {}
+
+        # Standardize and validate the correctness of rotary position embeddings parameters
+        self.rope_parameters.setdefault("rope_theta", kwargs.pop("rope_theta", self.default_theta))
+        self.standardize_rope_params()
+        self.validate_rope(ignore_keys=ignore_keys_at_rope_validation)
+
+        # Convert to float because RoPE fn expect a float. Models on the hub were saved as int
+        for key in ["beta_fast", "beta_slow", "factor"]:
+            if key in self.rope_parameters:
+                self.rope_parameters[key] = float(self.rope_parameters[key])
+        return kwargs

model-00001-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d5a5940fb4d1cfe41f246105efc710912897bcd78424dd21c400e36230913c2
+size 4996069592

model-00002-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f24c183584df371c4080debbaf653142471db515c82f5945acfa36a808d1867
+size 4999722272

model-00003-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3313fcccd81f53976b4c70851b981c6b02b79205cf9a2e878f03cb08e539e7dc
+size 4998461512

model-00004-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36938b4142c0eba90c686ad776dd3cbf25ca1f898d913e174f5449a1717ca832
+size 4998460680

model-00005-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be232a8c28fa38a08cb7d6660eb0bd261c067113213e6d12de8bb6d438107f03
+size 4999724400

model-00006-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aa5193a7030b93c939e44a62de91f83826c8e8961af095bdd165082f7949384
+size 4998465624

model-00007-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14968bf6934fba39cc56c480cbc280469cb0f8acaaa3ebeca81eb0ba3501c911
+size 4998464744

model-00008-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:807c891b1668e2d5fdd5ce0349a2718e9750d212908ddb514e1d21b5627ba5d5
+size 4999726168

model-00009-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa6a9d7b17bc3b602a5ca78a1ab4d551007ad81f2e0f5b718bfaa38f317a5590
+size 4998465576

model-00010-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e45d0afaa845f305af9194ff6897a7c78419c6d7abc3778d58096412883f13e
+size 4998464648

model-00011-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d37af3afb6b58c169d5539c1ba6136aef876d7220f892a482bfc1d15875b57c3
+size 4999726024

model-00012-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0e9730cab05a3b4103a9d64c74855024b8b00bee33297be56e01273f98a1a0b
+size 4998465528

model-00013-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27728722f5ab17e2d9a5cc0fe3b1c8e954fa951b58d89eff60728cce7e7559bb
+size 2808751984

model-00014-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b705792f86d54e5c8f859fa7de49f88fffa1391373e24a93110fb1f13ab6987
+size 4294967424

modeling_sarvam_moe.py

@@ -1,11 +1,30 @@
-"""PyTorch Sarvam MoE model."""
+# Copyright 2026 Sarvam AI team. All rights reserved.
+#
+# This code is based on Llama and Deepseek MoE implementations
+# in this library. It has been modified from its original forms to
+# accommodate Sarvam's MLA (multi-latent attention) MoE architecture.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 import math
+import warnings
 from typing import List, Optional, Tuple, Union
 
 import torch
 import torch.nn.functional as F
+import torch.utils.checkpoint
 from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
 
 from transformers.activations import ACT2FN
 from transformers.cache_utils import Cache, DynamicCache
@@ -13,48 +32,39 @@ from transformers.modeling_attn_mask_utils import (
     AttentionMaskConverter,
     _prepare_4d_attention_mask,
     _prepare_4d_causal_attention_mask,
-    _prepare_4d_causal_attention_mask_for_sdpa,
 )
-from transformers.modeling_outputs import MoeModelOutputWithPast
-from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
-from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
-from transformers.modeling_utils import PreTrainedModel
-from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+)
+from transformers.modeling_utils import PreTrainedModel, ALL_ATTENTION_FUNCTIONS
+from transformers.pytorch_utils import (
+    ALL_LAYERNORM_LAYERS,
+    is_torch_greater_or_equal_than_1_13,
+)
 from transformers.utils import (
-    is_flash_attn_2_available,
-    is_flash_attn_greater_or_equal_2_10,
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
     logging,
+    replace_return_docstrings,
 )
-from transformers.generation.utils import GenerationMixin
-from dataclasses import dataclass
-from transformers.utils import ModelOutput
-
+from transformers.utils.import_utils import is_torch_fx_available
 
-if is_flash_attn_2_available():
-    from flash_attn import flash_attn_func, flash_attn_varlen_func
-    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input
+import torch.distributed as dist
+import numpy as np
 
-from .configuration_sarvam_moe import SarvamMoEConfig
-
-logger = logging.get_logger(__name__)
+from .configuration_sarvam_moe import SarvamMLAConfig
 
-_CONFIG_FOR_DOC = "SarvamMoEConfig"
+if is_torch_fx_available():
+    if not is_torch_greater_or_equal_than_1_13:
+        import torch.fx
 
+    _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_mask)
 
-@dataclass
-class SarvamMoECausalLMOutputWithPast(ModelOutput):
-    loss: Optional[torch.FloatTensor] = None
-    logits: Optional[torch.FloatTensor] = None
-    past_key_values: Optional[Cache] = None
-    hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
-    attentions: Optional[tuple[torch.FloatTensor, ...]] = None
-    z_loss: Optional[torch.FloatTensor] = None
-    aux_loss: Optional[torch.FloatTensor] = None
-    router_logits: Optional[tuple[torch.FloatTensor]] = None
 
+logger = logging.get_logger(__name__)
 
-class SarvamMoEModelOutputWithPast(MoeModelOutputWithPast):
-    pass
+_CONFIG_FOR_DOC = "SarvamMLAConfig"
 
 
 def _get_unpad_data(attention_mask):
@@ -62,23 +72,27 @@ def _get_unpad_data(attention_mask):
     indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
     max_seqlen_in_batch = seqlens_in_batch.max().item()
     cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
-    return indices, cu_seqlens, max_seqlen_in_batch
-
-
-def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
-    return _prepare_4d_attention_mask(mask=mask, dtype=dtype, tgt_len=tgt_len)
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
 
 
-def _make_causal_mask(
-    input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
-):
-    return AttentionMaskConverter._make_causal_mask(
-        input_ids_shape=input_ids_shape, dtype=dtype, device=device, past_key_values_length=past_key_values_length
-    )
+def _get_usable_past_kv_length(cache: Cache, new_seq_length: int, layer_idx: int = 0) -> int:
+    previous_length = cache.get_seq_length(layer_idx)
+    # Dynamic layers return -1, static layers return an int
+    max_length = cache.get_max_cache_shape(layer_idx)
+    if max_length is not None and max_length != -1 and previous_length + new_seq_length > max_length:
+        return max_length - new_seq_length
+    return previous_length
 
 
-class SarvamMoERMSNorm(nn.Module):
+class SarvamMLARMSNorm(nn.Module):
     def __init__(self, hidden_size, eps=1e-6):
+        """
+        SarvamMLARMSNorm is equivalent to T5LayerNorm
+        """
         super().__init__()
         self.weight = nn.Parameter(torch.ones(hidden_size))
         self.variance_epsilon = eps
@@ -91,275 +105,357 @@ class SarvamMoERMSNorm(nn.Module):
         return self.weight * hidden_states.to(input_dtype)
 
 
-ALL_LAYERNORM_LAYERS.append(SarvamMoERMSNorm)
+ALL_LAYERNORM_LAYERS.append(SarvamMLARMSNorm)
 
 
-class SarvamMoERotaryEmbedding(nn.Module):
-    def __init__(self, config: SarvamMoEConfig, device=None):
+class SarvamMLARotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
         super().__init__()
-        self.config = config
-        self.max_seq_len_cached = config.max_position_embeddings
-        self.original_max_seq_len = config.max_position_embeddings
-        rope_scaling = getattr(config, "rope_scaling", None)
-        if rope_scaling is None:
-            self.rope_type = "default"
-            inv_freq, self.attention_scaling = self.compute_default_rope_parameters(
-                config, device
-            )
-        else:
-            self.rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", "default"))
-            if self.rope_type == "default":
-                inv_freq, self.attention_scaling = self.compute_default_rope_parameters(
-                    config, device
-                )
-            else:
-                rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
-                inv_freq, self.attention_scaling = rope_init_fn(config, device)
+
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
         self.register_buffer("inv_freq", inv_freq, persistent=False)
-        self.original_inv_freq = self.inv_freq
 
-    @staticmethod
-    def compute_default_rope_parameters(
-        config: SarvamMoEConfig,
-        device: Optional[torch.device] = None,
-        seq_len: Optional[int] = None,
-    ) -> Tuple[torch.Tensor, float]:
-        """
-        Default RoPE parameters (classic rotary embedding).
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings,
+            device=self.inv_freq.device,
+            dtype=torch.get_default_dtype(),
+        )
+        self.max_seq_len_cached = None
 
-        Mirrors HF's default implementation: use `rope_theta`, head_dim and
-        return (inv_freq, attention_scaling).
-        """
-        base = config.rope_theta
-        dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
-        inv_freq = 1.0 / (
-            base
-            ** (
-                torch.arange(0, dim, 2, dtype=torch.int64, device=device)
-                .to(dtype=torch.float32)
-                / dim
-            )
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+
+        freqs = torch.outer(t, self.inv_freq.to(t.device))
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+    def forward(self, x, seq_len=None):
+        if self.max_seq_len_cached is None or seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+
+        return (
+            self.cos_cached[:seq_len].to(dtype=x.dtype),
+            self.sin_cached[:seq_len].to(dtype=x.dtype),
         )
-        attention_factor = 1.0
-        return inv_freq, attention_factor
 
-    @torch.no_grad()
-    @dynamic_rope_update
-    def forward(self, x, position_ids):
-        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
-        position_ids_expanded = position_ids[:, None, :].float()
 
-        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
-        with torch.autocast(device_type=device_type, enabled=False):
-            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
-            emb = torch.cat((freqs, freqs), dim=-1)
-            cos = emb.cos() * self.attention_scaling
-            sin = emb.sin() * self.attention_scaling
+def yarn_find_correction_dim(num_rotations, dim, base=10000, max_position_embeddings=2048):
+    return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (2 * math.log(base))
+
+
+def yarn_find_correction_range(low_rot, high_rot, dim, base=10000, max_position_embeddings=2048):
+    low = math.floor(yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings))
+    high = math.ceil(yarn_find_correction_dim(high_rot, dim, base, max_position_embeddings))
+    return max(low, 0), min(high, dim - 1)
+
+
+def yarn_get_mscale(scale=1, mscale=1):
+    if scale <= 1:
+        return 1.0
+    return 0.1 * mscale * math.log(scale) + 1.0
 
-        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
 
+def yarn_linear_ramp_mask(min_val, max_val, dim):
+    if min_val == max_val:
+        max_val += 0.001
+    linear_func = (torch.arange(dim, dtype=torch.float32) - min_val) / (max_val - min_val)
+    return torch.clamp(linear_func, 0, 1)
 
+
+class SarvamMLAYarnRotaryEmbedding(SarvamMLARotaryEmbedding):
+    def __init__(
+        self,
+        dim,
+        max_position_embeddings=2048,
+        base=10000,
+        device=None,
+        scaling_factor=40.0,
+        original_max_position_embeddings=4096,
+        beta_fast=32,
+        beta_slow=1,
+        mscale=1.0,
+        mscale_all_dim=1.0,
+    ):
+        self.scaling_factor = float(scaling_factor)
+        self.original_max_position_embeddings = int(original_max_position_embeddings)
+        self.beta_fast = float(beta_fast)
+        self.beta_slow = float(beta_slow)
+        self.mscale = float(mscale)
+        self.mscale_all_dim = float(mscale_all_dim)
+        super().__init__(dim, max_position_embeddings, base, device)
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        dim = self.dim
+
+        freq_extra = 1.0 / (self.base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim))
+        freq_inter = 1.0 / (
+            self.scaling_factor * self.base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)
+        )
+
+        low, high = yarn_find_correction_range(
+            self.beta_fast,
+            self.beta_slow,
+            dim,
+            self.base,
+            self.original_max_position_embeddings,
+        )
+
+        inv_freq_mask = 1.0 - yarn_linear_ramp_mask(low, high, dim // 2).to(device=device, dtype=torch.float32)
+        inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        t = torch.arange(seq_len, device=device, dtype=torch.float32)
+        freqs = torch.outer(t, inv_freq)
+
+        _mscale = float(
+            yarn_get_mscale(self.scaling_factor, self.mscale)
+            / yarn_get_mscale(self.scaling_factor, self.mscale_all_dim)
+        )
+
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", (emb.cos() * _mscale).to(dtype), persistent=False)
+        self.register_buffer("sin_cached", (emb.sin() * _mscale).to(dtype), persistent=False)
+
+
+# Copied from transformers.models.llama.modeling_llama.rotate_half
 def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
     x1 = x[..., : x.shape[-1] // 2]
     x2 = x[..., x.shape[-1] // 2 :]
     return torch.cat((-x2, x1), dim=-1)
 
 
-def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
-    cos = cos.unsqueeze(unsqueeze_dim)
-    sin = sin.unsqueeze(unsqueeze_dim)
-    rotary_dim = cos.shape[-1]
-    q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
-    k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
-    q_embed = (q_rot * cos) + (rotate_half(q_rot) * sin)
-    k_embed = (k_rot * cos) + (rotate_half(k_rot) * sin)
-    q_embed = torch.cat([q_embed, q_pass], dim=-1)
-    k_embed = torch.cat([k_embed, k_pass], dim=-1)
+# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
+    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
+    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
+
+    b, h, s, d = q.shape
+    q = q.view(b, h, s, d // 2, 2).transpose(4, 3).reshape(b, h, s, d)
+
+    b, h, s, d = k.shape
+    k = k.view(b, h, s, d // 2, 2).transpose(4, 3).reshape(b, h, s, d)
+
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
     return q_embed, k_embed
 
 
-class SarvamMoEMLP(nn.Module):
-    def __init__(self, config: SarvamMoEConfig, intermediate_size: int):
+class SarvamMLAMLP(nn.Module):
+    def __init__(self, config, hidden_size=None, intermediate_size=None):
         super().__init__()
         self.config = config
-        self.hidden_size = config.hidden_size
-        self.intermediate_size = intermediate_size
+        self.hidden_size = config.hidden_size if hidden_size is None else hidden_size
+        self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size
+
         self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
         self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
         self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
         self.act_fn = ACT2FN[config.hidden_act]
 
     def forward(self, x):
-        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+        return down_proj
 
 
-class SarvamMoEGate(nn.Module):
+class MoEGate(nn.Module):
     def __init__(self, config):
         super().__init__()
         self.config = config
         self.top_k = config.num_experts_per_tok
-        self.num_experts = config.num_experts
-        self.n_group = config.n_group
-        self.topk_group = config.topk_group
-        self.gating_dim = config.hidden_size
-        self.weight = nn.Parameter(torch.empty((self.num_experts, self.gating_dim)))
+        self.n_routed_experts = config.num_experts
         self.routed_scaling_factor = config.routed_scaling_factor
-        self.score_function = config.score_function
-        # Ideally, we should register the expert_bias as a buffer, but vllm complains about it.
-        # self.register_buffer("expert_bias", torch.zeros((self.num_experts)))
-        self.expert_bias = nn.Parameter(
-            torch.zeros((self.num_experts)),
-            requires_grad=False,
-        )
+        self.scoring_func = "sigmoid"
+        self.topk_method = "noaux_tc"
+        self.n_group = getattr(config, "n_group", self.n_routed_experts // 8)
+        self.topk_group = getattr(config, "topk_group", 2)
+
+        self.norm_topk_prob = True
+        self.gating_dim = config.hidden_size
+        self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim)))
+        if self.topk_method == "noaux_tc":
+            self.e_score_correction_bias = nn.Parameter(torch.empty((self.n_routed_experts)))
         self.reset_parameters()
 
     def reset_parameters(self) -> None:
         import torch.nn.init as init
 
         init.kaiming_uniform_(self.weight, a=math.sqrt(5))
-
-    def group_limited_topk(self, scores: torch.Tensor):
-        num_tokens, _ = scores.size()
-        group_scores = scores.view(num_tokens, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
-        group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
-        group_mask = torch.zeros_like(group_scores)
-        group_mask.scatter_(1, group_idx, 1)
-        score_mask = (
-            group_mask.unsqueeze(-1)
-            .expand(num_tokens, self.n_group, self.num_experts // self.n_group)
-            .reshape(num_tokens, -1)
-        )
-        masked_scores = scores.masked_fill(~score_mask.bool(), float("-inf"))
-        probs, top_indices = torch.topk(masked_scores, k=self.top_k, dim=-1)
-        return probs, top_indices
+        if hasattr(self, "e_score_correction_bias"):
+            init.zeros_(self.e_score_correction_bias)
 
     def forward(self, hidden_states):
-        hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
-        logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32))
-        scores = torch.sigmoid(logits.float()).type_as(logits)
-        scores_for_routing = scores + self.expert_bias
-        _, topk_idx = self.group_limited_topk(scores_for_routing)
-        scores = torch.gather(scores, dim=1, index=topk_idx).type_as(logits)
-        topk_weight = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if self.top_k > 1 else scores
-        topk_weight = topk_weight * self.routed_scaling_factor
-        return topk_idx, topk_weight, logits
-
-
-class SarvamMoEExperts(nn.ModuleList):
-    def __init__(self, config: SarvamMoEConfig):
-        # one MLP per expert
-        experts = [
-            SarvamMoEMLP(config=config, intermediate_size=config.moe_intermediate_size)
-            for _ in range(config.num_experts)
-        ]
-        super().__init__(experts)
+        bsz, seq_len, h = hidden_states.shape
+        hidden_states = hidden_states.view(-1, h)
+        logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32), None)
+        if self.scoring_func == "sigmoid":
+            scores = logits.sigmoid()
+        else:
+            raise NotImplementedError(f"insupportable scoring function for MoE gating: {self.scoring_func}")
+
+        if self.topk_method == "noaux_tc":
+            assert not self.training
+            scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)
+            group_scores = (
+                scores_for_choice.view(bsz * seq_len, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
+            )  # [n, n_group]
+            group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]  # [n, top_k_group]
+            group_mask = torch.zeros_like(group_scores)  # [n, n_group]
+            group_mask.scatter_(1, group_idx, 1)  # [n, n_group]
+            score_mask = (
+                group_mask.unsqueeze(-1)
+                .expand(bsz * seq_len, self.n_group, self.n_routed_experts // self.n_group)
+                .reshape(bsz * seq_len, -1)
+            )  # [n, e]
+            tmp_scores = scores_for_choice.masked_fill(~score_mask.bool(), float("-inf"))  # [n, e]
+            _, topk_idx = torch.topk(tmp_scores, k=self.top_k, dim=-1, sorted=False)
+            topk_weight = scores.gather(1, topk_idx)
+        else:
+            raise NotImplementedError(f"insupportable TopK function for MoE gating: {self.topk_method}")
+
+        ### norm gate to sum 1
+        if self.top_k > 1 and self.norm_topk_prob:
+            denominator = topk_weight.sum(dim=-1, keepdim=True) + 1e-20
+            topk_weight = topk_weight / denominator
+        topk_weight = topk_weight * self.routed_scaling_factor  # must multiply the scaling factor
+
+        return topk_idx, topk_weight
+
+
+class SarvamMLAMoE(nn.Module):
+    def __init__(self, config):
+        super().__init__()
         self.config = config
         self.num_experts_per_tok = config.num_experts_per_tok
 
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        top_k_index: torch.LongTensor,
-        top_k_weights: torch.Tensor,
-    ) -> torch.Tensor:
-        """
-        hidden_states: (tokens, hidden_size) or (batch * seq, hidden_size)
-        top_k_index:   (tokens, top_k)
-        top_k_weights: (tokens, top_k)
-        """
-        tokens, hidden_dim = hidden_states.shape
-        flat_topk_idx = top_k_index.view(-1)
-
-        if self.training:
-            # training path: same as your previous logic
-            x = hidden_states.repeat_interleave(self.num_experts_per_tok, dim=0)
-            y = torch.empty_like(x)
-            for i, expert in enumerate(self):
-                mask = flat_topk_idx == i
-                if mask.any():
-                    y[mask] = expert(x[mask])
-            y = (y.view(*top_k_weights.shape, -1) * top_k_weights.unsqueeze(-1)).sum(dim=1)
-            return y.to(hidden_states.dtype)
-
-        # inference path: previous moe_infer logic
-        num_experts = len(self)
-        cnts = top_k_index.new_zeros((tokens, num_experts))
-        cnts.scatter_(1, top_k_index, 1)
-        tokens_per_expert = cnts.sum(dim=0)
+        if hasattr(config, "ep_size") and config.ep_size > 1:
+            assert config.ep_size == dist.get_world_size()
+            self.ep_size = config.ep_size
+            self.experts_per_rank = config.num_experts // config.ep_size
+            self.ep_rank = dist.get_rank()
+            self.experts = nn.ModuleList(
+                [
+                    (
+                        SarvamMLAMLP(config, intermediate_size=config.moe_intermediate_size)
+                        if i >= self.ep_rank * self.experts_per_rank and i < (self.ep_rank + 1) * self.experts_per_rank
+                        else None
+                    )
+                    for i in range(config.num_experts)
+                ]
+            )
+        else:
+            self.ep_size = 1
+            self.experts_per_rank = config.num_experts
+            self.ep_rank = 0
+            self.experts = nn.ModuleList(
+                [
+                    SarvamMLAMLP(config, intermediate_size=config.moe_intermediate_size)
+                    for i in range(config.num_experts)
+                ]
+            )
+        self.gate = MoEGate(config)
+        if (
+            hasattr(config, "num_shared_experts")
+            and config.num_shared_experts is not None
+            and config.num_shared_experts > 0
+        ):
+            intermediate_size = config.moe_intermediate_size * config.num_shared_experts
+            self.shared_experts = SarvamMLAMLP(config=config, intermediate_size=intermediate_size)
+        else:
+            self.shared_experts = None
+
+    def forward(self, hidden_states):
+        identity = hidden_states
+        orig_shape = hidden_states.shape
+        topk_idx, topk_weight = self.gate(hidden_states)
+        hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+        flat_topk_idx = topk_idx.view(-1)
+        if not self.training:
+            y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
+        else:
+            # Training mode - simple implementation
+            # In practice, you'd want a more sophisticated training implementation
+            y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
+        if self.shared_experts is not None:
+            y = y + self.shared_experts(identity)
+        return y
 
-        idxs = top_k_index.view(-1).argsort()
-        sorted_tokens = hidden_states[idxs // top_k_index.shape[1]]
+    @torch.no_grad()
+    def moe_infer(self, x, topk_ids, topk_weight):
+        cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts)))
+        cnts.scatter_(1, topk_ids, 1)
+        tokens_per_expert = cnts.sum(dim=0)
+        idxs = topk_ids.view(-1).argsort()
+        sorted_tokens = x[idxs // topk_ids.shape[1]]
+        sorted_tokens_shape = sorted_tokens.shape
+        if self.ep_size > 1:
+            tokens_per_ep_rank = tokens_per_expert.view(self.ep_size, -1).sum(dim=1)
+            tokens_per_expert_group = tokens_per_expert.new_empty(tokens_per_expert.shape[0])
+            dist.all_to_all_single(tokens_per_expert_group, tokens_per_expert)
+            output_splits = tokens_per_expert_group.view(self.ep_size, -1).sum(1).cpu().numpy().tolist()
+            gathered_tokens = sorted_tokens.new_empty(
+                tokens_per_expert_group.sum(dim=0).cpu().item(), sorted_tokens.shape[1]
+            )
+            input_split_sizes = tokens_per_ep_rank.cpu().numpy().tolist()
+            dist.all_to_all(
+                list(gathered_tokens.split(output_splits)),
+                list(sorted_tokens.split(input_split_sizes)),
+            )
+            tokens_per_expert_post_gather = tokens_per_expert_group.view(self.ep_size, self.experts_per_rank).sum(dim=0)
+            gatherd_idxs = np.zeros(shape=(gathered_tokens.shape[0],), dtype=np.int32)
+            s = 0
+            for i, k in enumerate(tokens_per_expert_group.cpu().numpy()):
+                gatherd_idxs[s : s + k] = i % self.experts_per_rank
+                s += k
+            gatherd_idxs = gatherd_idxs.argsort()
+            sorted_tokens = gathered_tokens[gatherd_idxs]
+            tokens_per_expert = tokens_per_expert_post_gather
+        tokens_per_expert = tokens_per_expert.cpu().numpy()
 
-        tokens_per_expert = tokens_per_expert.cpu().numpy().tolist()
         outputs = []
         start_idx = 0
         for i, num_tokens in enumerate(tokens_per_expert):
             end_idx = start_idx + num_tokens
             if num_tokens == 0:
                 continue
-            expert = self[i]
-            tokens_for_expert = sorted_tokens[start_idx:end_idx]
-            expert_out = expert(tokens_for_expert)
-            outputs.append(expert_out.to(hidden_states.device))
+            expert = self.experts[i + self.ep_rank * self.experts_per_rank]
+            if expert is None:
+                continue
+            tokens_for_this_expert = sorted_tokens[start_idx:end_idx]
+            expert_out = expert(tokens_for_this_expert)
+            outputs.append(expert_out)
             start_idx = end_idx
 
         outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
+        if self.ep_size > 1:
+            new_x = torch.empty_like(outs)
+            new_x[gatherd_idxs] = outs
+            gathered_tokens = new_x.new_empty(*sorted_tokens_shape)
+            dist.all_to_all(
+                list(gathered_tokens.split(input_split_sizes)),
+                list(new_x.split(output_splits)),
+            )
+            outs = gathered_tokens
+
         new_x = torch.empty_like(outs)
         new_x[idxs] = outs
-
         final_out = (
-            new_x.view(*top_k_index.shape, -1)
-            .type(top_k_weights.dtype)
-            .mul_(top_k_weights.unsqueeze(dim=-1))
+            new_x.view(*topk_ids.shape, -1)
+            .type(topk_weight.dtype)
+            .mul_(topk_weight.unsqueeze(dim=-1))
             .sum(dim=1)
             .type(new_x.dtype)
         )
         return final_out
 
 
-class SarvamMoESparseMoeBlock(nn.Module):
-    def __init__(self, config: SarvamMoEConfig):
-        super().__init__()
-        self.config = config
-        self.num_experts_per_tok = config.num_experts_per_tok
-
-        # use the new experts container
-        self.experts = SarvamMoEExperts(config)
-        self.gate = SarvamMoEGate(config)
-
-        if config.num_shared_experts is not None:
-            self.shared_experts = SarvamMoEMLP(
-                config=config,
-                intermediate_size=config.moe_intermediate_size * config.num_shared_experts,
-            )
-
-    # _setup_experts no longer needed
-
-    def forward(self, hidden_states):
-        identity = hidden_states
-        bsz, seq_len, h = hidden_states.shape
-
-        topk_idx, topk_weight, router_logits = self.gate(hidden_states)
-
-        # flatten batch+seq for experts
-        flat_hidden = hidden_states.view(-1, h)
-        flat_topk_idx = topk_idx.view(-1, topk_idx.shape[-1])
-        flat_topk_weight = topk_weight.view(-1, topk_weight.shape[-1])
-
-        y = self.experts(flat_hidden, flat_topk_idx, flat_topk_weight)
-        y = y.view(bsz, seq_len, h)
-
-        if self.config.num_shared_experts is not None:
-            y = y + self.shared_experts(identity)
-
-        # router logits shape: (bsz, seq_len, num_experts)
-        router_info = (
-            router_logits.view(bsz, seq_len, -1),
-            topk_idx.view(bsz, seq_len, -1),
-        )
-        return y, router_info
-
-
+# Copied from transformers.models.llama.modeling_llama.repeat_kv
 def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
     batch, num_key_value_heads, slen, head_dim = hidden_states.shape
     if n_rep == 1:
@@ -368,9 +464,9 @@ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
     return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
 
 
-class SarvamMoEAttention(nn.Module):
-    is_causal = True    # vLLM / Transformers backend critical flag
-    def __init__(self, config: SarvamMoEConfig, layer_idx: Optional[int] = None):
+class SarvamMLAAttention(nn.Module):
+    is_causal = True
+    def __init__(self, config: SarvamMLAConfig, layer_idx: Optional[int] = None):
         super().__init__()
         self.config = config
         self.layer_idx = layer_idx
@@ -380,29 +476,84 @@ class SarvamMoEAttention(nn.Module):
                 "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
                 "when creating this class."
             )
+
         self.attention_dropout = config.attention_dropout
         self.hidden_size = config.hidden_size
         self.num_heads = config.num_attention_heads
-        self.head_dim = config.head_dim or self.hidden_size // self.num_heads
-        partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
-        self.rope_dim = int(self.head_dim * partial_rotary_factor)
-        self.num_key_value_heads = config.num_key_value_heads
-        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+
         self.max_position_embeddings = config.max_position_embeddings
         self.rope_theta = config.rope_theta
-        self.query_key_value = nn.Linear(
+        self.q_lora_rank = getattr(config, "q_lora_rank", None)
+        self.qk_rope_head_dim = config.qk_rope_head_dim
+        self.kv_lora_rank = config.kv_lora_rank
+        self.v_head_dim = config.v_head_dim
+        self.qk_nope_head_dim = config.qk_nope_head_dim
+        self.q_head_dim = config.q_head_dim
+
+        if self.q_lora_rank is None:
+            self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.q_head_dim, bias=False)
+        else:
+            self.q_a_proj = nn.Linear(
+                self.hidden_size, config.q_lora_rank, bias=getattr(config, "attention_bias", False)
+            )
+            self.q_a_layernorm = SarvamMLARMSNorm(config.q_lora_rank)
+            self.q_b_proj = nn.Linear(config.q_lora_rank, self.num_heads * self.q_head_dim, bias=False)
+
+        self.kv_a_proj_with_mqa = nn.Linear(
             self.hidden_size,
-            (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim,
-            bias=config.use_qkv_bias,
+            config.kv_lora_rank + config.qk_rope_head_dim,
+            bias=getattr(config, "attention_bias", False),
+        )
+        self.kv_a_layernorm = SarvamMLARMSNorm(config.kv_lora_rank)
+        self.kv_b_proj = nn.Linear(
+            config.kv_lora_rank,
+            self.num_heads * (self.q_head_dim - self.qk_rope_head_dim + self.v_head_dim),
+            bias=False,
         )
-        if self.config.use_qk_norm:
-            self.query_layernorm = SarvamMoERMSNorm(self.head_dim, eps=config.rms_norm_eps)
-            self.key_layernorm = SarvamMoERMSNorm(self.head_dim, eps=config.rms_norm_eps)
-        self.dense = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.use_bias)
-        self.scaling = self.head_dim**-0.5
+
+        self.o_proj = nn.Linear(
+            self.num_heads * self.v_head_dim,
+            self.hidden_size,
+            bias=getattr(config, "attention_bias", False),
+        )
+        self._init_rope()
+
+        self.softmax_scale = self.q_head_dim ** (-0.5)
+        if self.config.rope_scaling is not None:
+            mscale_all_dim = self.config.rope_scaling.get("mscale_all_dim", 0)
+            scaling_factor = self.config.rope_scaling["factor"]
+            if mscale_all_dim:
+                mscale = yarn_get_mscale(scaling_factor, mscale_all_dim)
+                self.softmax_scale = self.softmax_scale * mscale * mscale
+
+    def _init_rope(self):
+        rope_scaling = getattr(self.config, "rope_scaling", None)
+        if rope_scaling is None or rope_scaling.get("type", None) in (None, "default"):
+            self.rotary_emb = SarvamMLARotaryEmbedding(
+                self.qk_rope_head_dim,
+                max_position_embeddings=self.max_position_embeddings,
+                base=self.rope_theta,
+            )
+            return
+
+        rope_type = rope_scaling.get("type")
+        if rope_type == "deepseek_yarn":
+            self.rotary_emb = SarvamMLAYarnRotaryEmbedding(
+                self.qk_rope_head_dim,
+                max_position_embeddings=self.max_position_embeddings,
+                base=self.rope_theta,
+                scaling_factor=rope_scaling.get("factor", 40.0),
+                original_max_position_embeddings=rope_scaling.get("original_max_position_embeddings", 4096),
+                beta_fast=rope_scaling.get("beta_fast", 32),
+                beta_slow=rope_scaling.get("beta_slow", 1),
+                mscale=rope_scaling.get("mscale", 1.0),
+                mscale_all_dim=rope_scaling.get("mscale_all_dim", 1.0),
+            )
+            return
+        raise ValueError(f"Unknown rope_scaling type: {rope_type}")
 
     def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
-        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
+        return tensor.view(bsz, seq_len, self.num_heads, self.v_head_dim).transpose(1, 2).contiguous()
 
     def forward(
         self,
@@ -412,321 +563,101 @@ class SarvamMoEAttention(nn.Module):
         past_key_value: Optional[Cache] = None,
         output_attentions: bool = False,
         use_cache: bool = False,
-        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
         **kwargs,
-    ):
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
         bsz, q_len, _ = hidden_states.size()
-        qkv = self.query_key_value(hidden_states)
-        qkv = qkv.view(
-            bsz, q_len, self.num_heads + 2 * self.num_key_value_heads, self.head_dim
-        )
-        query_states, key_states, value_states = qkv.split(
-            [self.num_heads, self.num_key_value_heads, self.num_key_value_heads],
-            dim=-2,
-        )
-        query_states = query_states.transpose(1, 2).contiguous()
-        key_states = key_states.transpose(1, 2).contiguous()
-        value_states = value_states.transpose(1, 2).contiguous()
-        if self.config.use_qk_norm:
-            query_states = self.query_layernorm(query_states)
-            key_states = self.key_layernorm(key_states)
-        cos, sin = position_embeddings
-        query_states, key_states = apply_rotary_pos_emb(
-            query_states, key_states, cos, sin
+
+        if self.q_lora_rank is None:
+            q = self.q_proj(hidden_states)
+        else:
+            q = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(hidden_states)))
+        q = q.view(bsz, q_len, self.num_heads, self.q_head_dim).transpose(1, 2)
+        q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
+
+        compressed_kv = self.kv_a_proj_with_mqa(hidden_states)
+        compressed_kv, k_pe = torch.split(compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
+        k_pe = k_pe.view(bsz, q_len, 1, self.qk_rope_head_dim).transpose(1, 2)
+        kv = (
+            self.kv_b_proj(self.kv_a_layernorm(compressed_kv))
+            .view(bsz, q_len, self.num_heads, self.qk_nope_head_dim + self.v_head_dim)
+            .transpose(1, 2)
         )
+
+        k_nope, value_states = torch.split(kv, [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
+        kv_seq_len = value_states.shape[-2]
         if past_key_value is not None:
             if self.layer_idx is None:
                 raise ValueError(
-                    "When using cache, SarvamMoEAttention must be initialized with layer_idx."
+                    f"The cache structure has changed in a previous version. If you are using {self.__class__.__name__} "
+                    "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+                    "with a layer index."
                 )
-            cache_kwargs = {"sin": sin, "cos": cos}
-            key_states, value_states = past_key_value.update(
-                key_states, value_states, self.layer_idx, cache_kwargs
-            )
-        # NOTE: vLLM will set config._attn_implementation = "vllm"
-        if self.config._attn_implementation == "vllm":
-            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
-            attn_output, attn_weights = attention_interface(
-                self,
-                query_states,
-                key_states,
-                value_states,
-                attention_mask,
-                dropout=0.0 if not self.training else self.attention_dropout,
-                scaling=self.scaling,
-                **kwargs,
-            )
-            # vLLM backend may return [B, L, hidden] or [B*L, hidden]
-            if attn_output.dim() == 4:
-                # [B, H, L, Dh] -> [B, L, hidden]
-                attn_output = attn_output.transpose(1, 2).contiguous()
-                attn_output = attn_output.view(bsz, q_len, -1)
-            elif attn_output.dim() == 3:
-                if attn_output.shape[0] != bsz or attn_output.shape[1] != q_len:
-                    raise ValueError(
-                        f"Unexpected vLLM attention output shape {attn_output.shape}, "
-                        f"expected (bsz={bsz}, q_len={q_len}, hidden=*)"
-                    )
-            elif attn_output.dim() == 2:
-                attn_output = attn_output.view(bsz, q_len, -1)
-            else:
-                raise ValueError(
-                    f"Unsupported vLLM attention output rank {attn_output.dim()} "
-                    f"with shape {attn_output.shape}"
-                )
-            attn_output = self.dense(attn_output)
-            if not output_attentions:
-                attn_weights = None
-            return attn_output, attn_weights, past_key_value
-
-        key_states = repeat_kv(key_states, self.num_key_value_groups)
-        value_states = repeat_kv(value_states, self.num_key_value_groups)
-        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
-        kv_seq_len = key_states.shape[-2]
+            kv_seq_len += _get_usable_past_kv_length(past_key_value, kv_seq_len, self.layer_idx)
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+
+        q_pe, k_pe = apply_rotary_pos_emb(q_pe, k_pe, cos, sin, position_ids)
+
+        query_states = k_pe.new_empty(bsz, self.num_heads, q_len, self.q_head_dim)
+        query_states[:, :, :, : self.qk_nope_head_dim] = q_nope
+        query_states[:, :, :, self.qk_nope_head_dim :] = q_pe
+
+        key_states = k_pe.new_empty(bsz, self.num_heads, q_len, self.q_head_dim)
+        key_states[:, :, :, : self.qk_nope_head_dim] = k_nope
+        key_states[:, :, :, self.qk_nope_head_dim :] = k_pe
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.softmax_scale
+
         if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
             raise ValueError(
                 f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
                 f" {attn_weights.size()}"
             )
+        assert attention_mask is not None
         if attention_mask is not None:
             if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
                 raise ValueError(
                     f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
                 )
             attn_weights = attn_weights + attention_mask
+
         attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
         attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
         attn_output = torch.matmul(attn_weights, value_states)
-        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.v_head_dim):
             raise ValueError(
-                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.v_head_dim)}, but is"
                 f" {attn_output.size()}"
             )
         attn_output = attn_output.transpose(1, 2).contiguous()
-        attn_output = attn_output.reshape(bsz, q_len, -1)
-        attn_output = self.dense(attn_output)
-        if not output_attentions:
-            attn_weights = None
-        return attn_output, attn_weights, past_key_value
-
+        attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.v_head_dim)
+        attn_output = self.o_proj(attn_output)
 
-class SarvamMoEFlashAttention2(SarvamMoEAttention):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
-
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        attention_mask: Optional[torch.LongTensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
-        past_key_value: Optional[Cache] = None,
-        output_attentions: bool = False,
-        use_cache: bool = False,
-        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
-        **kwargs,
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
-        output_attentions = False
-        bsz, q_len, _ = hidden_states.size()
-        qkv = self.query_key_value(hidden_states)
-        qkv = qkv.view(bsz, q_len, self.num_heads + 2 * self.num_key_value_heads, self.head_dim)
-        query_states, key_states, value_states = qkv.split(
-            [self.num_heads, self.num_key_value_heads, self.num_key_value_heads], dim=-2
-        )
-        query_states = query_states.transpose(1, 2)
-        key_states = key_states.transpose(1, 2)
-        value_states = value_states.transpose(1, 2)
-        if self.config.use_qk_norm:
-            query_states = self.query_layernorm(query_states)
-            key_states = self.key_layernorm(key_states)
-        cos, sin = position_embeddings
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
-        if past_key_value is not None:
-            cache_kwargs = {"sin": sin, "cos": cos}
-            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
-        query_states = query_states.transpose(1, 2)
-        key_states = key_states.transpose(1, 2)
-        value_states = value_states.transpose(1, 2)
-        dropout_rate = self.attention_dropout if self.training else 0.0
-        input_dtype = query_states.dtype
-        if input_dtype == torch.float32:
-            if hasattr(self.config, "_pre_quantization_dtype"):
-                target_dtype = self.config._pre_quantization_dtype
-            elif torch.is_autocast_enabled():
-                target_dtype = torch.get_autocast_gpu_dtype()
-            else:
-                target_dtype = self.query_key_value.weight.dtype
-            logger.warning_once(
-                f"The input hidden states seems to be silently casted in float32, this might be related to"
-                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
-                f" {target_dtype}."
-            )
-            query_states = query_states.to(target_dtype)
-            key_states = key_states.to(target_dtype)
-            value_states = value_states.to(target_dtype)
-        attn_output = self._flash_attention_forward(
-            query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
-        )
-        attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
-        attn_output = self.dense(attn_output)
         if not output_attentions:
             attn_weights = None
-        return attn_output, attn_weights, past_key_value
-
-    def _flash_attention_forward(
-        self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
-    ):
-        if not self._flash_attn_uses_top_left_mask:
-            causal = self.is_causal
-        else:
-            causal = self.is_causal and query_length != 1
-        if attention_mask is not None:
-            batch_size = query_states.shape[0]
-            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
-                query_states, key_states, value_states, attention_mask, query_length
-            )
-            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
-            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
-            attn_output_unpad = flash_attn_varlen_func(
-                query_states,
-                key_states,
-                value_states,
-                cu_seqlens_q=cu_seqlens_q,
-                cu_seqlens_k=cu_seqlens_k,
-                max_seqlen_q=max_seqlen_in_batch_q,
-                max_seqlen_k=max_seqlen_in_batch_k,
-                dropout_p=dropout,
-                softmax_scale=softmax_scale,
-                causal=causal,
-            )
-            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
-        else:
-            attn_output = flash_attn_func(
-                query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
-            )
-        return attn_output
-
-    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
-        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
-        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
-        key_layer = index_first_axis(
-            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
-        )
-        value_layer = index_first_axis(
-            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
-        )
-        if query_length == kv_seq_len:
-            query_layer = index_first_axis(
-                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
-            )
-            cu_seqlens_q = cu_seqlens_k
-            max_seqlen_in_batch_q = max_seqlen_in_batch_k
-            indices_q = indices_k
-        elif query_length == 1:
-            max_seqlen_in_batch_q = 1
-            cu_seqlens_q = torch.arange(
-                batch_size + 1, dtype=torch.int32, device=query_layer.device
-            )
-            indices_q = cu_seqlens_q[:-1]
-            query_layer = query_layer.squeeze(1)
-        else:
-            attention_mask = attention_mask[:, -query_length:]
-            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
-        return (
-            query_layer,
-            key_layer,
-            value_layer,
-            indices_q,
-            (cu_seqlens_q, cu_seqlens_k),
-            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
-        )
-
-
-class SarvamMoESdpaAttention(SarvamMoEAttention):
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        attention_mask: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
-        past_key_value: Optional[Cache] = None,
-        output_attentions: Optional[bool] = False,
-        use_cache: Optional[bool] = False,
-        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
-        **kwargs,
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
-        if output_attentions:
-            return super().forward(
-                hidden_states=hidden_states,
-                attention_mask=attention_mask,
-                position_ids=position_ids,
-                past_key_value=past_key_value,
-                output_attentions=output_attentions,
-                use_cache=use_cache,
-                **kwargs,
-            )
-        bsz, q_len, _ = hidden_states.size()
-        qkv = self.query_key_value(hidden_states)
-        qkv = qkv.view(bsz, q_len, self.num_heads + 2 * self.num_key_value_heads, self.head_dim)
-        query_states, key_states, value_states = qkv.split(
-            [self.num_heads, self.num_key_value_heads, self.num_key_value_heads], dim=-2
-        )
-        query_states = query_states.transpose(1, 2)
-        key_states = key_states.transpose(1, 2)
-        value_states = value_states.transpose(1, 2)
-        if self.config.use_qk_norm:
-            query_states = self.query_layernorm(query_states)
-            key_states = self.key_layernorm(key_states)
-        cos, sin = position_embeddings
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
-        if past_key_value is not None:
-            cache_kwargs = {"sin": sin, "cos": cos}
-            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
-        key_states = repeat_kv(key_states, self.num_key_value_groups)
-        value_states = repeat_kv(value_states, self.num_key_value_groups)
-        if attention_mask is not None:
-            kv_seq_len = key_states.shape[-2]
-            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
-                raise ValueError(
-                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
-                )
-        if query_states.device.type == "cuda" and attention_mask is not None:
-            query_states = query_states.contiguous()
-            key_states = key_states.contiguous()
-            value_states = value_states.contiguous()
-        attn_output = torch.nn.functional.scaled_dot_product_attention(
-            query_states,
-            key_states,
-            value_states,
-            attn_mask=attention_mask,
-            dropout_p=self.attention_dropout if self.training else 0.0,
-            is_causal=self.is_causal and attention_mask is None and q_len > 1,
-        )
-        attn_output = attn_output.transpose(1, 2).contiguous()
-        attn_output = attn_output.reshape(bsz, q_len, -1)
-        attn_output = self.dense(attn_output)
-        return attn_output, None, past_key_value
 
-
-ATTENTION_CLASSES = {
-    "eager": SarvamMoEAttention,
-    "flash_attention_2": SarvamMoEFlashAttention2,
-    "sdpa": SarvamMoESdpaAttention,
-    "vllm": SarvamMoEAttention,
-}
+        return attn_output, attn_weights, past_key_value
 
 
-class SarvamMoEDecoderLayer(nn.Module):
-    def __init__(self, config: SarvamMoEConfig, layer_idx: int):
+class SarvamMLADecoderLayer(nn.Module):
+    def __init__(self, config: SarvamMLAConfig, layer_idx: int):
         super().__init__()
         self.hidden_size = config.hidden_size
-        self.attention = ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx)
-        self.mlp = (
-            SarvamMoESparseMoeBlock(config)
-            if (config.num_experts is not None and layer_idx >= config.first_k_dense_replace)
-            else SarvamMoEMLP(config=config, intermediate_size=config.intermediate_size)
+        self.self_attn = SarvamMLAAttention(config=config, layer_idx=layer_idx)
+
+        use_moe = (
+            hasattr(config, "num_experts")
+            and config.num_experts is not None
+            and layer_idx >= getattr(config, "first_k_dense_replace", 0)
+            and layer_idx % getattr(config, "moe_layer_freq", 1) == 0
         )
-        self.input_layernorm = SarvamMoERMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.post_attention_layernorm = SarvamMoERMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.mlp = SarvamMLAMoE(config) if use_moe else SarvamMLAMLP(config)
+        self.input_layernorm = SarvamMLARMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = SarvamMLARMSNorm(config.hidden_size, eps=config.rms_norm_eps)
 
     def forward(
         self,
@@ -735,49 +666,44 @@ class SarvamMoEDecoderLayer(nn.Module):
         position_ids: Optional[torch.LongTensor] = None,
         past_key_value: Optional[Tuple[torch.Tensor]] = None,
         output_attentions: Optional[bool] = False,
-        output_router_logits: Optional[bool] = False,
         use_cache: Optional[bool] = False,
-        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
         **kwargs,
     ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
         residual = hidden_states
         hidden_states = self.input_layernorm(hidden_states)
-        hidden_states, self_attn_weights, present_key_value = self.attention(
+
+        hidden_states, self_attn_weights, present_key_value = self.self_attn(
             hidden_states=hidden_states,
             attention_mask=attention_mask,
             position_ids=position_ids,
             past_key_value=past_key_value,
             output_attentions=output_attentions,
-            position_embeddings=position_embeddings,
             use_cache=use_cache,
             **kwargs,
         )
         hidden_states = residual + hidden_states
+
         residual = hidden_states
         hidden_states = self.post_attention_layernorm(hidden_states)
         hidden_states = self.mlp(hidden_states)
-        if isinstance(hidden_states, tuple):
-            hidden_states, router_logits = hidden_states
-        else:
-            router_logits = None
-        hidden_states = residual + hidden_states.to(residual.device)
+        hidden_states = residual + hidden_states
+
         outputs = (hidden_states,)
+
         if output_attentions:
             outputs += (self_attn_weights,)
         if use_cache:
             outputs += (present_key_value,)
-        if output_router_logits:
-            outputs += (router_logits,)
         return outputs
 
-class SarvamMoEPreTrainedModel(PreTrainedModel):
-    config_class = SarvamMoEConfig
+
+class SarvamMLAPreTrainedModel(PreTrainedModel):
+    config_class = SarvamMLAConfig
     base_model_prefix = "model"
     supports_gradient_checkpointing = True
-    _no_split_modules = ["SarvamMoEDecoderLayer"]
+    _no_split_modules = ["SarvamMLADecoderLayer"]
     _skip_keys_device_placement = "past_key_values"
-    _supports_flash_attn_2 = True
-    _supports_sdpa = True
+    _supports_flash_attn_2 = False  # Not implemented yet
     _supports_cache_class = True
 
     def _init_weights(self, module):
@@ -792,30 +718,28 @@ class SarvamMoEPreTrainedModel(PreTrainedModel):
                 module.weight.data[module.padding_idx].zero_()
 
 
-
-class SarvamMoEModel(SarvamMoEPreTrainedModel):
-    _supports_attention_backend = True
-    def __init__(self, config: SarvamMoEConfig):
+class SarvamMLAModel(SarvamMLAPreTrainedModel):
+    def __init__(self, config: SarvamMLAConfig):
         super().__init__(config)
         self.padding_idx = config.pad_token_id
         self.vocab_size = config.vocab_size
-        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
-        self.layers = []
-        for layer_idx in range(config.num_hidden_layers):
-            self.layers.append(SarvamMoEDecoderLayer(config, layer_idx))
-        self.layers = nn.ModuleList(self.layers)
-        self._use_sdpa = config._attn_implementation == "sdpa"
-        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
-        self.norm = SarvamMoERMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.rotary_emb = SarvamMoERotaryEmbedding(config=config)
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [SarvamMLADecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self._use_flash_attention_2 = False  # Not implemented yet
+        self.norm = SarvamMLARMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
         self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
         self.post_init()
 
     def get_input_embeddings(self):
-        return self.word_embeddings
+        return self.embed_tokens
 
     def set_input_embeddings(self, value):
-        self.word_embeddings = value
+        self.embed_tokens = value
 
     def forward(
         self,
@@ -827,19 +751,17 @@ class SarvamMoEModel(SarvamMoEPreTrainedModel):
         use_cache: Optional[bool] = None,
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
-        output_router_logits: Optional[bool] = None,
         return_dict: Optional[bool] = None,
-        **kwargs,
-    ) -> Union[Tuple, SarvamMoEModelOutputWithPast]:
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
         )
-        output_router_logits = (
-            output_router_logits if output_router_logits is not None else self.config.output_router_logits
-        )
         use_cache = use_cache if use_cache is not None else self.config.use_cache
+
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # retrieve input_ids and inputs_embeds
         if input_ids is not None and inputs_embeds is not None:
             raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
         elif input_ids is not None:
@@ -848,111 +770,91 @@ class SarvamMoEModel(SarvamMoEPreTrainedModel):
             batch_size, seq_length = inputs_embeds.shape[:2]
         else:
             raise ValueError("You have to specify either input_ids or inputs_embeds")
-        if self.gradient_checkpointing and self.training:
-            if use_cache:
-                logger.warning_once(
-                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`transformers."
-                )
-                use_cache = False
-        if use_cache and past_key_values is None:
-            past_key_values = DynamicCache()
-        if inputs_embeds is None:
-            inputs_embeds = self.word_embeddings(input_ids)
-        past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+
+        past_key_values_length = 0
+        if use_cache:
+            use_legacy_cache = not isinstance(past_key_values, Cache)
+            if use_legacy_cache:
+                past_key_values = DynamicCache.from_legacy_cache(past_key_values)
+            past_key_values_length = _get_usable_past_kv_length(past_key_values, seq_length)
+
         if position_ids is None:
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
             position_ids = torch.arange(
-                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+                past_key_values_length,
+                seq_length + past_key_values_length,
+                dtype=torch.long,
+                device=device,
             )
             position_ids = position_ids.unsqueeze(0)
-        if self._use_flash_attention_2:
-            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
-        elif self._use_sdpa and not output_attentions:
-            attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
-                attention_mask,
-                (batch_size, seq_length),
-                inputs_embeds,
-                past_seen_tokens,
-            )
-        else:
-            attention_mask = _prepare_4d_causal_attention_mask(
-                attention_mask, (batch_size, seq_length), inputs_embeds, past_seen_tokens
-            )
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        attention_mask = _prepare_4d_causal_attention_mask(
+            attention_mask,
+            (batch_size, seq_length),
+            inputs_embeds,
+            past_key_values_length,
+        )
+
         hidden_states = inputs_embeds
-        position_embeddings = self.rotary_emb(hidden_states, position_ids)
         all_hidden_states = () if output_hidden_states else None
         all_self_attns = () if output_attentions else None
-        all_router_logits = () if output_router_logits else None
         next_decoder_cache = None
-        layers = self.layers
-        for decoder_layer in layers:
+
+        for decoder_layer in self.layers:
             if output_hidden_states:
                 all_hidden_states += (hidden_states,)
-            if self.gradient_checkpointing and self.training:
-                layer_outputs = self._gradient_checkpointing_func(
-                    decoder_layer.__call__,
-                    hidden_states,
-                    attention_mask,
-                    position_ids,
-                    past_key_values,
-                    output_attentions,
-                    output_router_logits,
-                    use_cache,
-                    position_embeddings,
-                    **kwargs,
-                )
-            else:
-                layer_outputs = decoder_layer(
-                    hidden_states,
-                    attention_mask=attention_mask,
-                    position_ids=position_ids,
-                    past_key_value=past_key_values,
-                    output_attentions=output_attentions,
-                    output_router_logits=output_router_logits,
-                    use_cache=use_cache,
-                    position_embeddings=position_embeddings,
-                    **kwargs,
-                )
+
+            layer_outputs = decoder_layer(
+                hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_values,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+            )
+
             hidden_states = layer_outputs[0]
             if use_cache:
                 next_decoder_cache = layer_outputs[2 if output_attentions else 1]
             if output_attentions:
                 all_self_attns += (layer_outputs[1],)
-            if output_router_logits and layer_outputs[-1] is not None:
-                all_router_logits += (layer_outputs[-1],)
+
         hidden_states = self.norm(hidden_states)
+
         if output_hidden_states:
             all_hidden_states += (hidden_states,)
+
         next_cache = None
         if use_cache:
-            next_cache = next_decoder_cache
+            next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache
         if not return_dict:
-            return tuple(
-                v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_router_logits] if v is not None
-            )
-        return SarvamMoEModelOutputWithPast(
+            return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
+        return BaseModelOutputWithPast(
             last_hidden_state=hidden_states,
             past_key_values=next_cache,
             hidden_states=all_hidden_states,
             attentions=all_self_attns,
-            router_logits=all_router_logits,
         )
 
 
-class SarvamMoEForCausalLM(SarvamMoEPreTrainedModel, GenerationMixin):
+class SarvamMLAForCausalLM(SarvamMLAPreTrainedModel):
     _tied_weights_keys = ["lm_head.weight"]
 
-    def __init__(self, config: SarvamMoEConfig):
+    def __init__(self, config):
         super().__init__(config)
-        self.model = SarvamMoEModel(config)
+        self.model = SarvamMLAModel(config)
         self.vocab_size = config.vocab_size
         self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
         self.post_init()
 
     def get_input_embeddings(self):
-        return self.model.word_embeddings
+        return self.model.embed_tokens
 
     def set_input_embeddings(self, value):
-        self.model.word_embeddings = value
+        self.model.embed_tokens = value
 
     def get_output_embeddings(self):
         return self.lm_head
@@ -977,18 +879,14 @@ class SarvamMoEForCausalLM(SarvamMoEPreTrainedModel, GenerationMixin):
         use_cache: Optional[bool] = None,
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
-        output_router_logits: Optional[bool] = None,
         return_dict: Optional[bool] = None,
-        **kwargs,
-    ) -> Union[Tuple, SarvamMoEModelOutputWithPast]:
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
         )
-        output_router_logits = (
-            output_router_logits if output_router_logits is not None else self.config.output_router_logits
-        )
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
         outputs = self.model(
             input_ids=input_ids,
             attention_mask=attention_mask,
@@ -998,28 +896,98 @@ class SarvamMoEForCausalLM(SarvamMoEPreTrainedModel, GenerationMixin):
             use_cache=use_cache,
             output_attentions=output_attentions,
             output_hidden_states=output_hidden_states,
-            output_router_logits=output_router_logits,
             return_dict=return_dict,
-            **kwargs,
         )
-        loss = None
-        aux_loss = None
+
         hidden_states = outputs[0]
         logits = self.lm_head(hidden_states)
         logits = logits.float()
+
+        loss = None
         if labels is not None:
-            loss = self.loss_function(logits, labels, self.config.vocab_size, **kwargs)
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
         if not return_dict:
             output = (logits,) + outputs[1:]
-            if output_router_logits:
-                output = (aux_loss,) + output
             return (loss,) + output if loss is not None else output
-        return SarvamMoECausalLMOutputWithPast(
+
+        return CausalLMOutputWithPast(
             loss=loss,
             logits=logits,
             past_key_values=outputs.past_key_values,
             hidden_states=outputs.hidden_states,
             attentions=outputs.attentions,
-            aux_loss=aux_loss,
-            router_logits=outputs.router_logits,
-        )
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        **kwargs,
+    ):
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.get_seq_length() if past_key_values is not None else 0
+                if hasattr(past_key_values, "get_max_length"):
+                    max_cache_length = past_key_values.get_max_length()
+                else:
+                    max_cache_length = None
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+                max_cache_length = None
+
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+
+            if (
+                max_cache_length is not None
+                and attention_mask is not None
+                and cache_length + input_ids.shape[1] > max_cache_length
+            ):
+                attention_mask = attention_mask[:, -max_cache_length:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+            }
+        )
+        return model_inputs
+
+    @staticmethod
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (
+                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
+            )
+        return reordered_past

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29af880c3290d504b4ee80d998f4c48de7ba5972656e745226c47a8723238412
-size 33627350
+oid sha256:a574ceaaff7c7a8f091179c53fd17ae33567089c099d4ff37d4cb3bc1a87e80e
+size 33627251