Update configuration_neollm.py

configuration_neollm.py

@@ -82,7 +82,10 @@ class NeoLLMConfig(PretrainedConfig):
         meap_mask_ratio (:obj:`float`, *optional*, defaults to ``0.15``):
             Fraction of eligible input positions replaced in each sequence.
         meap_mask_token_id (:obj:`int`, *optional*):
-            Token ID used as the MEAP replacement. Required when MEAP is enabled.
+            Dedicated reserved token ID used only as the MEAP replacement.
+            Required when MEAP is enabled; it must be inside the vocabulary and
+            distinct from PAD/BOS/EOS. Its final trainable representation is
+            independent from dense embeddings, Leviathan, and Spelling Bee.
         meap_seed (:obj:`int`, *optional*, defaults to ``0``):
             Base 32-bit seed. The Trainer derives a distinct deterministic seed
             for each microstep and distributed process.
@@ -699,6 +702,9 @@ class NeoLLMConfig(PretrainedConfig):
         mile_loss_gamma=1.0,
         use_mu_loss=True,
         mu_loss_lambda=1e-4,
+        # Must remain disabled in the no-argument configuration. Transformers
+        # constructs ``NeoLLMConfig()`` while saving to determine class defaults;
+        # training enables MEAP explicitly together with its mask token ID.
         use_meap=False,
         meap_mask_ratio=0.15,
         meap_mask_token_id=None,
@@ -773,7 +779,7 @@ class NeoLLMConfig(PretrainedConfig):
         # ── PolyNorm exclusivity ──────────────────────────────────────────
         polynorm_exclusive=False,
         # ── Spelling Bee Embeddings (Rabe et al., 2026) ───────────────────
-        use_spelling_bee_embeddings=True,
+        use_spelling_bee_embeddings=False,
         # ── Context Re-Positioning (Li et al., 2026) ──────────────────────
         use_repo=True,
         repo_start_layer=None,
@@ -822,7 +828,7 @@ class NeoLLMConfig(PretrainedConfig):
         nextlat_kl_weight=1.0,
         nextlat_dynamics_proj_factor=1.0,
         # ── Interleaved Head Attention (Duvvuri et al., 2026) ─────────────
-        use_iha=True,
+        use_iha=False,
         iha_num_pseudo_heads=2,  # P=2 → 2×2=4 patrones por head
         iha_local_global_pattern="LLLLG",  # 4 locales + 1 global (paper §5.1)
         iha_sliding_window=None,  # auto = context // (2*P^2), independent of padding
@@ -863,8 +869,26 @@ class NeoLLMConfig(PretrainedConfig):
             )
         if use_meap and meap_mask_token_id is None:
             raise ValueError("`meap_mask_token_id` is required when `use_meap=True`.")
-        if meap_mask_token_id is not None and int(meap_mask_token_id) < 0:
-            raise ValueError("`meap_mask_token_id` must be a non-negative integer.")
+        if meap_mask_token_id is not None:
+            meap_mask_token_id = int(meap_mask_token_id)
+            if not 0 <= meap_mask_token_id < int(vocab_size):
+                raise ValueError(
+                    "`meap_mask_token_id` must be inside the configured vocabulary."
+                )
+            if use_meap:
+                reserved_for_model = {
+                    token_id
+                    for token_id in (
+                        kwargs.get("pad_token_id"),
+                        kwargs.get("bos_token_id"),
+                        kwargs.get("eos_token_id"),
+                    )
+                    if token_id is not None
+                }
+                if meap_mask_token_id in reserved_for_model:
+                    raise ValueError(
+                        "`meap_mask_token_id` must differ from PAD, BOS, and EOS."
+                    )
         if not isinstance(meap_seed, int) or not 0 <= meap_seed <= 0xFFFFFFFF:
             raise ValueError("`meap_seed` must be an integer in [0, 2**32 - 1].")
         meap_implementation = str(meap_implementation).strip().lower()