Update configuration_neollm.py
Browse files- configuration_neollm.py +3 -3
configuration_neollm.py
CHANGED
|
@@ -322,7 +322,7 @@ class NeoLLMConfig(PretrainedConfig):
|
|
| 322 |
use_mea_attention=True,
|
| 323 |
mea_component_key_value_heads=None,
|
| 324 |
mea_groupnorm_eps=1e-6,
|
| 325 |
-
use_lucid_attention=
|
| 326 |
lucid_attention_eps=1e-6,
|
| 327 |
use_affine_scaled_attention=True,
|
| 328 |
affine_momentum=0.9,
|
|
@@ -333,8 +333,8 @@ class NeoLLMConfig(PretrainedConfig):
|
|
| 333 |
directional_routing_k=4,
|
| 334 |
directional_routing_temp=5.0,
|
| 335 |
# ββ Attention Residuals (Kimi Team, 2026) βββββββββββββββββββββββββ
|
| 336 |
-
use_attn_res=
|
| 337 |
-
attn_res_num_blocks=
|
| 338 |
fan_ratio=0.125,
|
| 339 |
fan_ratio_ffn=0.0625,
|
| 340 |
dropout_rate=0.1,
|
|
|
|
| 322 |
use_mea_attention=True,
|
| 323 |
mea_component_key_value_heads=None,
|
| 324 |
mea_groupnorm_eps=1e-6,
|
| 325 |
+
use_lucid_attention=False,
|
| 326 |
lucid_attention_eps=1e-6,
|
| 327 |
use_affine_scaled_attention=True,
|
| 328 |
affine_momentum=0.9,
|
|
|
|
| 333 |
directional_routing_k=4,
|
| 334 |
directional_routing_temp=5.0,
|
| 335 |
# ββ Attention Residuals (Kimi Team, 2026) βββββββββββββββββββββββββ
|
| 336 |
+
use_attn_res=False,
|
| 337 |
+
attn_res_num_blocks=0,
|
| 338 |
fan_ratio=0.125,
|
| 339 |
fan_ratio_ffn=0.0625,
|
| 340 |
dropout_rate=0.1,
|