Update configuration_neollm.py

configuration_neollm.py

@@ -719,7 +719,7 @@ class NeoLLMConfig(PretrainedConfig):
         head_dim=64,
         use_momentum_attention=True,
         momentum_gamma=0.10,
-        use_mea_attention=True,
+        use_mea_attention=False,
         mea_component_key_value_heads=None,
         mea_groupnorm_eps=1e-6,
         use_lucid_attention=True,
@@ -775,39 +775,26 @@ class NeoLLMConfig(PretrainedConfig):
         # ── DCA (Heddes et al., 2025) ─────────────────────────────────────
         use_dca=False,
         dca_k=1,
-        dca_use_final_grn=True,
+        dca_use_final_grn=False,
         dca_grn_eps=1e-6,
         # ── MUDD connections (Xiao et al., 2025) ─────────────────────────
         use_mudd=False,
         mudd_dense_type="qkvr",
-        mudd_dynamic_dense=True,
+        mudd_dynamic_dense=False,
         mudd_round64=True,
         mudd_expand_last=True,
         mudd_sepln=False,
         # ── StackTrans (Zhang et al., NeurIPS 2025) ───────────────────────
         use_stacktrans=False,
         stacktrans_num_heads=4,
-        stacktrans_stack_slots=24,
-        stacktrans_stack_d_model=64,
+        stacktrans_stack_slots=16,
+        stacktrans_stack_d_model=32,
         stacktrans_forward_bs=1,
         # ── LAuReL (Menghani, Kumar & Kumar, ICML 2025) ───────────────────
         use_laurel=False,
-        use_laurel_rw=True,
+        use_laurel_rw=False,
         use_laurel_lr=True,
         laurel_lr_rank=32,
-        # ── GatedDeltaNet linear attention (Yang et al., 2024) ───────────
-        # Replaces full attention every `linear_attention_every_n` layers
-        # (0-indexed: layers 2, 5, 8, ... for every_n=3).
-        # REPO applies to linear attention layers when both
-        # use_repo=True and use_repo_in_linear_attn=True.
-        use_linear_attention=False,
-        linear_attention_every_n=3,
-        use_repo_in_linear_attn=False,
-        linear_conv_kernel_dim=4,
-        linear_key_head_dim=32,
-        linear_value_head_dim=32,
-        linear_num_key_heads=8,
-        linear_num_value_heads=16,
         **kwargs,
     ):
         # ── Generator / tying consistency ─────────────────────────────────
@@ -1032,16 +1019,6 @@ class NeoLLMConfig(PretrainedConfig):
         self.use_laurel_lr                 = use_laurel_lr
         self.laurel_lr_rank                = laurel_lr_rank
 
-        # ── GatedDeltaNet linear attention ────────────────────────────────
-        self.use_linear_attention     = use_linear_attention
-        self.linear_attention_every_n = linear_attention_every_n
-        self.use_repo_in_linear_attn  = use_repo_in_linear_attn
-        self.linear_conv_kernel_dim   = linear_conv_kernel_dim
-        self.linear_key_head_dim      = linear_key_head_dim
-        self.linear_value_head_dim    = linear_value_head_dim
-        self.linear_num_key_heads     = linear_num_key_heads
-        self.linear_num_value_heads   = linear_num_value_heads
-
         # ── VersatileFFN (Nie et al., 2026) ───────────────────────────────
         self.use_versatile_ffn             = use_versatile_ffn
         self.versatile_total_experts       = versatile_total_experts