KitsuVp commited on
Commit
e2d023d
Β·
verified Β·
1 Parent(s): f54e3ef

Update configuration_neollm.py

Browse files
Files changed (1) hide show
  1. configuration_neollm.py +3 -3
configuration_neollm.py CHANGED
@@ -322,7 +322,7 @@ class NeoLLMConfig(PretrainedConfig):
322
  use_mea_attention=True,
323
  mea_component_key_value_heads=None,
324
  mea_groupnorm_eps=1e-6,
325
- use_lucid_attention=True,
326
  lucid_attention_eps=1e-6,
327
  use_affine_scaled_attention=True,
328
  affine_momentum=0.9,
@@ -333,8 +333,8 @@ class NeoLLMConfig(PretrainedConfig):
333
  directional_routing_k=4,
334
  directional_routing_temp=5.0,
335
  # ── Attention Residuals (Kimi Team, 2026) ─────────────────────────
336
- use_attn_res=True,
337
- attn_res_num_blocks=4,
338
  fan_ratio=0.125,
339
  fan_ratio_ffn=0.0625,
340
  dropout_rate=0.1,
 
322
  use_mea_attention=True,
323
  mea_component_key_value_heads=None,
324
  mea_groupnorm_eps=1e-6,
325
+ use_lucid_attention=False,
326
  lucid_attention_eps=1e-6,
327
  use_affine_scaled_attention=True,
328
  affine_momentum=0.9,
 
333
  directional_routing_k=4,
334
  directional_routing_temp=5.0,
335
  # ── Attention Residuals (Kimi Team, 2026) ─────────────────────────
336
+ use_attn_res=False,
337
+ attn_res_num_blocks=0,
338
  fan_ratio=0.125,
339
  fan_ratio_ffn=0.0625,
340
  dropout_rate=0.1,