Update model weights after training (epoch 3, loss 2.9528)

cross_attention.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:deec8e7a08caa092ae8f2831f90c3a76bd49297d6cc2c0fd8daf80bf163b2128
+oid sha256:90cfc123da137f4fbc6084c56bc6480dd6985f95f25d66efa7e141d55dcac62d
 size 174191400

llm.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22e14456b6c0badb864ac7491d545f18369dd99ad87a247bb0ad716911b28fea
-size 1506836434
+oid sha256:65b2d1e302245fba130eb2853e9daf227d270d0639d9a1adadc24cee75ef0740
+size 3381777564

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 7309258640,
+    "total_size": 9184163778,
     "format": "components"
   },
   "weight_map": {
@@ -37,8 +37,10 @@
     "llm.model.layers.1.input_layernorm.weight": "llm.safetensors",
     "llm.model.layers.1.post_attention_layernorm.weight": "llm.safetensors",
     "llm.model.layers.1.mlp.router.expert_bias": "llm.safetensors",
+    "llm.model.layers.1.mlp.router.deep_expert_bias": "llm.safetensors",
     "llm.model.layers.1.mlp.router.input_norm.weight": "llm.safetensors",
     "llm.model.layers.1.mlp.router.gate.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.router.deep_gate.weight": "llm.safetensors",
     "llm.model.layers.1.mlp.experts.0.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.1.mlp.experts.0.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.1.mlp.experts.0.gate_proj.linear.weight": "llm.safetensors",
@@ -111,6 +113,48 @@
     "llm.model.layers.1.mlp.experts.7.down_proj.lora_A": "llm.safetensors",
     "llm.model.layers.1.mlp.experts.7.down_proj.lora_B": "llm.safetensors",
     "llm.model.layers.1.mlp.experts.7.down_proj.linear.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.0.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.0.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.0.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.0.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.0.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.0.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.1.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.2.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.gate_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.up_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.1.mlp.deep_experts.3.down_projs.4.weight": "llm.safetensors",
     "llm.model.layers.1.mlp.shared_expert.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.1.mlp.shared_expert.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.1.mlp.shared_expert.gate_proj.linear.weight": "llm.safetensors",
@@ -152,8 +196,10 @@
     "llm.model.layers.3.input_layernorm.weight": "llm.safetensors",
     "llm.model.layers.3.post_attention_layernorm.weight": "llm.safetensors",
     "llm.model.layers.3.mlp.router.expert_bias": "llm.safetensors",
+    "llm.model.layers.3.mlp.router.deep_expert_bias": "llm.safetensors",
     "llm.model.layers.3.mlp.router.input_norm.weight": "llm.safetensors",
     "llm.model.layers.3.mlp.router.gate.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.router.deep_gate.weight": "llm.safetensors",
     "llm.model.layers.3.mlp.experts.0.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.3.mlp.experts.0.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.3.mlp.experts.0.gate_proj.linear.weight": "llm.safetensors",
@@ -226,6 +272,48 @@
     "llm.model.layers.3.mlp.experts.7.down_proj.lora_A": "llm.safetensors",
     "llm.model.layers.3.mlp.experts.7.down_proj.lora_B": "llm.safetensors",
     "llm.model.layers.3.mlp.experts.7.down_proj.linear.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.0.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.0.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.0.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.0.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.0.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.0.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.1.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.2.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.gate_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.up_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.3.mlp.deep_experts.3.down_projs.4.weight": "llm.safetensors",
     "llm.model.layers.3.mlp.shared_expert.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.3.mlp.shared_expert.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.3.mlp.shared_expert.gate_proj.linear.weight": "llm.safetensors",
@@ -267,8 +355,10 @@
     "llm.model.layers.5.input_layernorm.weight": "llm.safetensors",
     "llm.model.layers.5.post_attention_layernorm.weight": "llm.safetensors",
     "llm.model.layers.5.mlp.router.expert_bias": "llm.safetensors",
+    "llm.model.layers.5.mlp.router.deep_expert_bias": "llm.safetensors",
     "llm.model.layers.5.mlp.router.input_norm.weight": "llm.safetensors",
     "llm.model.layers.5.mlp.router.gate.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.router.deep_gate.weight": "llm.safetensors",
     "llm.model.layers.5.mlp.experts.0.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.5.mlp.experts.0.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.5.mlp.experts.0.gate_proj.linear.weight": "llm.safetensors",
@@ -341,6 +431,48 @@
     "llm.model.layers.5.mlp.experts.7.down_proj.lora_A": "llm.safetensors",
     "llm.model.layers.5.mlp.experts.7.down_proj.lora_B": "llm.safetensors",
     "llm.model.layers.5.mlp.experts.7.down_proj.linear.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.0.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.0.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.0.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.0.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.0.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.0.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.1.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.2.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.gate_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.up_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.5.mlp.deep_experts.3.down_projs.4.weight": "llm.safetensors",
     "llm.model.layers.5.mlp.shared_expert.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.5.mlp.shared_expert.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.5.mlp.shared_expert.gate_proj.linear.weight": "llm.safetensors",
@@ -382,8 +514,10 @@
     "llm.model.layers.7.input_layernorm.weight": "llm.safetensors",
     "llm.model.layers.7.post_attention_layernorm.weight": "llm.safetensors",
     "llm.model.layers.7.mlp.router.expert_bias": "llm.safetensors",
+    "llm.model.layers.7.mlp.router.deep_expert_bias": "llm.safetensors",
     "llm.model.layers.7.mlp.router.input_norm.weight": "llm.safetensors",
     "llm.model.layers.7.mlp.router.gate.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.router.deep_gate.weight": "llm.safetensors",
     "llm.model.layers.7.mlp.experts.0.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.7.mlp.experts.0.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.7.mlp.experts.0.gate_proj.linear.weight": "llm.safetensors",
@@ -456,6 +590,48 @@
     "llm.model.layers.7.mlp.experts.7.down_proj.lora_A": "llm.safetensors",
     "llm.model.layers.7.mlp.experts.7.down_proj.lora_B": "llm.safetensors",
     "llm.model.layers.7.mlp.experts.7.down_proj.linear.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.0.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.0.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.0.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.0.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.0.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.0.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.1.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.2.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.gate_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.up_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.7.mlp.deep_experts.3.down_projs.4.weight": "llm.safetensors",
     "llm.model.layers.7.mlp.shared_expert.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.7.mlp.shared_expert.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.7.mlp.shared_expert.gate_proj.linear.weight": "llm.safetensors",
@@ -497,8 +673,10 @@
     "llm.model.layers.9.input_layernorm.weight": "llm.safetensors",
     "llm.model.layers.9.post_attention_layernorm.weight": "llm.safetensors",
     "llm.model.layers.9.mlp.router.expert_bias": "llm.safetensors",
+    "llm.model.layers.9.mlp.router.deep_expert_bias": "llm.safetensors",
     "llm.model.layers.9.mlp.router.input_norm.weight": "llm.safetensors",
     "llm.model.layers.9.mlp.router.gate.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.router.deep_gate.weight": "llm.safetensors",
     "llm.model.layers.9.mlp.experts.0.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.9.mlp.experts.0.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.9.mlp.experts.0.gate_proj.linear.weight": "llm.safetensors",
@@ -571,6 +749,48 @@
     "llm.model.layers.9.mlp.experts.7.down_proj.lora_A": "llm.safetensors",
     "llm.model.layers.9.mlp.experts.7.down_proj.lora_B": "llm.safetensors",
     "llm.model.layers.9.mlp.experts.7.down_proj.linear.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.0.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.0.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.0.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.0.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.0.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.0.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.1.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.2.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.gate_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.up_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.9.mlp.deep_experts.3.down_projs.4.weight": "llm.safetensors",
     "llm.model.layers.9.mlp.shared_expert.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.9.mlp.shared_expert.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.9.mlp.shared_expert.gate_proj.linear.weight": "llm.safetensors",
@@ -612,8 +832,10 @@
     "llm.model.layers.11.input_layernorm.weight": "llm.safetensors",
     "llm.model.layers.11.post_attention_layernorm.weight": "llm.safetensors",
     "llm.model.layers.11.mlp.router.expert_bias": "llm.safetensors",
+    "llm.model.layers.11.mlp.router.deep_expert_bias": "llm.safetensors",
     "llm.model.layers.11.mlp.router.input_norm.weight": "llm.safetensors",
     "llm.model.layers.11.mlp.router.gate.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.router.deep_gate.weight": "llm.safetensors",
     "llm.model.layers.11.mlp.experts.0.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.11.mlp.experts.0.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.11.mlp.experts.0.gate_proj.linear.weight": "llm.safetensors",
@@ -686,6 +908,48 @@
     "llm.model.layers.11.mlp.experts.7.down_proj.lora_A": "llm.safetensors",
     "llm.model.layers.11.mlp.experts.7.down_proj.lora_B": "llm.safetensors",
     "llm.model.layers.11.mlp.experts.7.down_proj.linear.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.0.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.0.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.0.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.0.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.0.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.0.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.1.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.2.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.gate_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.gate_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.up_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.up_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.up_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.up_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.up_projs.4.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.down_projs.0.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.down_projs.1.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.down_projs.2.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.down_projs.3.weight": "llm.safetensors",
+    "llm.model.layers.11.mlp.deep_experts.3.down_projs.4.weight": "llm.safetensors",
     "llm.model.layers.11.mlp.shared_expert.gate_proj.lora_A": "llm.safetensors",
     "llm.model.layers.11.mlp.shared_expert.gate_proj.lora_B": "llm.safetensors",
     "llm.model.layers.11.mlp.shared_expert.gate_proj.linear.weight": "llm.safetensors",
@@ -699,6 +963,17 @@
     "llm.model.thought_gate.weight": "llm.safetensors",
     "llm.model.thought_gate.bias": "llm.safetensors",
     "llm.model.thought_layernorm.weight": "llm.safetensors",
+    "llm.model.thought_halt_head.weight": "llm.safetensors",
+    "llm.model.thought_halt_head.bias": "llm.safetensors",
+    "llm.model.fast_ponder_block.gate_projs.0.weight": "llm.safetensors",
+    "llm.model.fast_ponder_block.gate_projs.1.weight": "llm.safetensors",
+    "llm.model.fast_ponder_block.gate_projs.2.weight": "llm.safetensors",
+    "llm.model.fast_ponder_block.up_projs.0.weight": "llm.safetensors",
+    "llm.model.fast_ponder_block.up_projs.1.weight": "llm.safetensors",
+    "llm.model.fast_ponder_block.up_projs.2.weight": "llm.safetensors",
+    "llm.model.fast_ponder_block.down_projs.0.weight": "llm.safetensors",
+    "llm.model.fast_ponder_block.down_projs.1.weight": "llm.safetensors",
+    "llm.model.fast_ponder_block.down_projs.2.weight": "llm.safetensors",
     "llm.lm_head.weight": "llm.safetensors",
     "vision_encoder.vision_model.vision_model.embeddings.patch_embedding.weight": "vision_encoder.safetensors",
     "vision_encoder.vision_model.vision_model.embeddings.patch_embedding.bias": "vision_encoder.safetensors",

modeling_xoron.py

@@ -8851,22 +8851,32 @@ class AuxLosslessMoERouter (nn .Module ):
         self .gate =nn .Linear (hidden_size ,num_experts ,bias =False )
         nn .init .normal_ (self .gate .weight ,mean =0.0 ,std =0.01 )
 
-
-
-
         self .expert_bias =nn .Parameter (torch .zeros (num_experts ))
 
+        # Deep experts gate (4 deep experts)
+        self .num_deep_experts = 4
+        self .deep_gate = nn .Linear (hidden_size , self .num_deep_experts , bias =False )
+        nn .init .normal_ (self .deep_gate .weight , mean =0.0 , std =0.01 )
+        self .deep_expert_bias = nn .Parameter (torch .zeros (self .num_deep_experts ))
+
     def forward (self ,hidden_states :torch .Tensor )->Tuple [torch .Tensor ,torch .Tensor ,torch .Tensor ]:
         batch_size ,seq_len ,hidden_dim =hidden_states .shape 
         hidden_flat =hidden_states .view (-1 ,hidden_dim )
 
         hidden_norm =self .input_norm (hidden_flat )
-        router_logits =self .gate (hidden_norm )
-
+        
+        # Standard experts
+        router_logits_std =self .gate (hidden_norm )
+        biased_logits_std =router_logits_std +self .expert_bias 
+        
+        # Deep experts
+        router_logits_deep = self .deep_gate (hidden_norm )
+        biased_logits_deep = router_logits_deep + self .deep_expert_bias
 
-        biased_logits =router_logits +self .expert_bias 
+        # Concatenate: [batch*seq, num_experts + num_deep_experts]
+        router_logits = torch .cat ([biased_logits_std , biased_logits_deep ], dim =-1 )
 
-        router_probs =F .softmax (biased_logits ,dim =-1 ,dtype =hidden_states .dtype )
+        router_probs =F .softmax (router_logits ,dim =-1 ,dtype =hidden_states .dtype )
 
         top_k_probs ,top_k_indices =torch .topk (router_probs ,self .top_k ,dim =-1 )
 
@@ -8897,6 +8907,37 @@ class MoEExpert (nn .Module ):
         return self .down_proj (self .act_fn (self .gate_proj (x ))*self .up_proj (x ))
 
 
+class DeepMoEExpert (nn .Module ):
+    """Deep MoE Expert with multiple sequential SwiGLU transformations."""
+
+    def __init__ (self ,hidden_size :int ,intermediate_size :int ,depth :int =2 ):
+        super ().__init__ ()
+        self .depth = depth
+        
+        self .gate_projs = nn .ModuleList ([nn .Linear (hidden_size if i == 0 else intermediate_size , intermediate_size , bias =False ) for i in range (depth )])
+        self .up_projs = nn .ModuleList ([nn .Linear (hidden_size if i == 0 else intermediate_size , intermediate_size , bias =False ) for i in range (depth )])
+        self .down_projs = nn .ModuleList ([nn .Linear (intermediate_size , intermediate_size if i < depth - 1 else hidden_size , bias =False ) for i in range (depth )])
+        
+        self .act_fn = nn .SiLU ()
+        self ._init_weights ()
+
+    def _init_weights (self ):
+        std =0.02 
+        for g , u , d in zip (self .gate_projs , self .up_projs , self .down_projs ):
+            nn .init .normal_ (g .weight ,mean =0.0 ,std =std )
+            nn .init .normal_ (u .weight ,mean =0.0 ,std =std )
+            nn .init .normal_ (d .weight ,mean =0.0 ,std =std *0.5 )
+
+    def forward (self ,x :torch .Tensor )->torch .Tensor :
+        for i in range (self .depth ):
+            # Optional residual connection if intermediate sizes match, but standard SwiGLU doesn't usually use them internally unless specified.
+            # We'll stick to sequential application as defined: Input -> SwiGLU -> SwiGLU ... -> DownProj
+            gate = self .act_fn (self .gate_projs [i ](x ))
+            up = self .up_projs [i ](x )
+            x = self .down_projs [i ](gate * up )
+        return x
+
+
 class IsolatedSharedExpert (nn .Module ):
     """
     Isolated Shared Expert that always processes all tokens.
@@ -8946,6 +8987,13 @@ class AuxLosslessMoELayer (nn .Module ):
         MoEExpert (hidden_size ,intermediate_size )
         for _ in range (num_experts )
         ])
+        
+        # Deep Experts: Depths 2, 3, 4, 5
+        self .num_deep_experts = 4
+        self .deep_experts = nn .ModuleList ([
+        DeepMoEExpert (hidden_size , intermediate_size , depth =d )
+        for d in range (2 , 6 )
+        ])
 
         shared_size =shared_expert_intermediate_size or intermediate_size 
         self .shared_expert =IsolatedSharedExpert (hidden_size ,shared_size )
@@ -8964,8 +9012,14 @@ class AuxLosslessMoELayer (nn .Module ):
 
         final_output =torch .zeros_like (hidden_flat )
 
-        for expert_idx in range (self .num_experts ):
-            expert =self .experts [expert_idx ]
+        total_experts = self .num_experts + self .num_deep_experts
+        for expert_idx in range (total_experts ):
+            # Determine which expert list to use
+            if expert_idx < self .num_experts :
+                expert =self .experts [expert_idx ]
+            else :
+                expert =self .deep_experts [expert_idx - self .num_experts ]
+
             for k in range (self .num_experts_per_tok ):
                 mask =(top_k_indices [:,k ]==expert_idx )
                 if mask .any ():
@@ -9002,7 +9056,26 @@ class AuxLosslessMoELayer (nn .Module ):
 
         z_loss =torch .logsumexp (router_logits ,dim =-1 ).square ().mean ()*0.0001 
 
+        # Add penalty for choosing deep experts
+        # Depths are 2, 3, 4, 5 for indices (num_experts) to (num_experts + 3)
+        # Cost is roughly proportional to depth
+        deep_penalty = torch .tensor (0.0 , device =router_logits .device , dtype =router_logits .dtype )
+        
+        # Calculate how often each deep expert was selected
+        # top_k_indices shape: [batch*seq, top_k]
+        for i in range (self .num_deep_experts ):
+            expert_idx = self .num_experts + i
+            depth = i + 2 # depths 2, 3, 4, 5
+            
+            # Count how many times this deep expert was chosen in top-k
+            selection_count = (top_k_indices == expert_idx ).sum ()
+            
+            # Simple penalty: deeper experts cost more
+            # Multiplied by a small scalar to act as a soft deterrent
+            # The model must truly need the depth to offset this loss increase
+            deep_penalty += selection_count .float () * depth * 0.00005
 
+        return z_loss + deep_penalty
 
         expert_mask =F .one_hot (top_k_indices ,self .num_experts ).float ()
         tokens_per_expert =expert_mask .sum (dim =(0 ,1 ))
@@ -9129,6 +9202,14 @@ class MoELlamaModel (nn .Module ):
         nn .init .constant_ (self .thought_gate .bias , -2.0 )  # Initialize gate biased toward original (sigmoid(-2)≈0.12)
         self .thought_layernorm = LlamaRMSNorm (config .hidden_size , eps =config .rms_norm_eps )
 
+        # Halt head: dynamically decides when to stop thinking
+        self .thought_halt_head = nn .Linear (config .hidden_size , 1 , bias =True )
+        nn .init .constant_ (self .thought_halt_head .bias , -2.0 ) # Biased toward continuing to think initially
+
+        # Fast Ponder Block for hyper-efficient 10x faster latent reasoning
+        # Bypasses O(N^2) attention, uses pure deep SwiGLU logic
+        self .fast_ponder_block = DeepMoEExpert (config .hidden_size , config .intermediate_size , depth =3 )
+
         self ._init_weights ()
 
     def _init_weights (self ):
@@ -9226,21 +9307,21 @@ class MoELlamaModel (nn .Module ):
             ).unsqueeze (0 ).expand (batch_size , -1 )
 
             for thought_step in range (thinking_depth ):
-                # Normalize before re-entering the layers
+                # Check if we should halt thinking (only during inference or if forced)
+                # We evaluate the halt head on the *current* hidden state of the last token
+                halt_logits = self .thought_halt_head (hidden_states [:, -1:, :])
+                halt_prob = torch .sigmoid (halt_logits )
+                
+                # If during generation we decide to stop, break early
+                if not self .training and (halt_prob > 0.5 ).all ():
+                    break
+
+                # Normalize before processing
                 hidden_states = self .thought_layernorm (hidden_states )
 
-                # Run through all layers again (no cache — full re-computation)
-                for layer in self .layers :
-                    hidden_states , _ , _ , step_aux = layer (
-                        hidden_states =hidden_states ,
-                        attention_mask =None ,  # Self-attend freely in thought space
-                        position_ids =thought_position_ids ,
-                        past_key_value =None ,
-                        output_attentions =False ,
-                        use_cache =False ,
-                    )
-                    if step_aux is not None :
-                        total_aux_loss = total_aux_loss + step_aux
+                # Run purely through the attention-free fast ponder block
+                # This achieves ~10x speedup by completely bypassing the O(N^2) self-attention stack
+                hidden_states = self .fast_ponder_block (hidden_states )
 
                 # Gated residual: blend thought with original
                 # gate ∈ [0,1], initialized small so early training
@@ -11563,13 +11644,13 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
             return torch .load (state_path ,map_location ='cpu')
         return None 
 
-    def freeze_components (self ,components :List [str ]):
+    def freeze_components (self ,components :List [str ],hard_freeze :bool =True ):
         """
         Freeze specific components of the model.
         
         IMPORTANT RULES:
         1. LLM is NEVER frozen - it's trained from scratch and always needs full weight training
-        2. LoRA parameters are NEVER frozen - they should always be trainable
+        2. LoRA parameters are usually kept trainable, UNLESS hard_freeze=True
         
         Args:
             components: List of component group names to freeze.
@@ -11578,13 +11659,15 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
                        'modality_markers'
                        
                        NOTE: 'llm' is NOT a valid group to freeze - will be ignored!
+            hard_freeze: If True, completely freezes the component including its LoRA adapters.
+                         This prevents inactive components from updating via weight decay/momentum.
         """
 
         if 'llm'in components :
             logger .warning ("Ignoring 'llm' in freeze list - LLM must always train (from scratch)")
             components =[c for c in components if c !='llm']
 
-        logger .info (f"Freezing components: {components }")
+        logger .info (f"Freezing components: {components } (hard_freeze={hard_freeze })")
 
         for group_name in components :
             if group_name not in COMPONENT_GROUPS :
@@ -11601,12 +11684,12 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
                             for name ,param in component .named_parameters ():
 
                                 path_lora ='lora_A'in name or 'lora_B'in name or 'magnitude'in name 
-                                if not path_lora :
+                                if hard_freeze or not path_lora :
                                     param .requires_grad =False 
                         logger .info (f"Frozen: {attr_name }")
 
 
-        if self .lora_applied :
+        if self .lora_applied and not hard_freeze:
             enable_lora_training (self )
             logger .info ("LoRA parameters remain trainable")
 
@@ -11639,7 +11722,7 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
 
         self ._print_stats ()
 
-    def freeze_all_except (self ,components :List [str ]):
+    def freeze_all_except (self ,components :List [str ],hard_freeze :bool =True ):
         """
         Freeze all components except the specified ones.
         
@@ -11654,7 +11737,7 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
 
         all_groups =list (COMPONENT_GROUPS .keys ())
         groups_to_freeze =[g for g in all_groups if g not in components ]
-        self .freeze_components (groups_to_freeze )
+        self .freeze_components (groups_to_freeze ,hard_freeze =hard_freeze )
 
     def get_trainable_component_names (self )->List [str ]:
         """Get list of component groups that have trainable parameters."""

streaming_state.json

@@ -1,5 +1,5 @@
 {
-  "epoch": 158,
+  "epoch": 163,
   "unique_samples": 1500,
   "total_yields": 3000,
   "dataset_positions": {
@@ -10,26 +10,26 @@
     "NewYorker": 386,
     "Football": 6,
     "MagicBrush": 386,
-    "WildChat": 350,
-    "Synth-ShellExecution": 350,
+    "WildChat": 500,
+    "Synth-ShellExecution": 500,
     "Midjourney-Prompts": 200,
     "Synth-KnowledgeCutoff": 550,
     "Synth-GroundedResponse": 550,
     "CodeParrot-Clean": 350,
-    "ShareGPT-Clean": 350,
+    "ShareGPT-Clean": 500,
     "Synth-Issues": 350,
     "Dolly-15k": 800,
     "Conversation-Summarization": 800,
-    "Synth-ShellTimeout": 350,
-    "Synth-Docker": 350,
+    "Synth-ShellTimeout": 500,
+    "Synth-Docker": 500,
     "Synth-Documents": 450,
     "HumanEval-JavaScript": 164,
     "OpenOrca": 800,
-    "Synth-MultiStepExecution": 350,
+    "Synth-MultiStepExecution": 500,
     "Synth-Citation": 550,
     "NoRobots": 800,
-    "Synth-LanguageSetup": 350,
-    "Function-Calling-ChatML": 350,
+    "Synth-LanguageSetup": 500,
+    "Function-Calling-ChatML": 500,
     "Synth-CoT": 900,
     "Python-Code-18k": 350,
     "Code-Feedback": 350,
@@ -43,38 +43,38 @@
     "HumanEval-Go": 164,
     "Synth-SelfCorrection": 550,
     "Synth-FactCheck": 550,
-    "Synth-Downloads": 350,
+    "Synth-Downloads": 500,
     "Synth-RetrievalGrounded": 550,
     "Synth-IDK": 550,
-    "Synth-APIGen": 350,
-    "Synth-PythonScripts": 350,
+    "Synth-APIGen": 500,
+    "Synth-PythonScripts": 500,
     "Synth-Uncertainty": 550,
     "HumanEval-Python": 164,
     "Golang-QA-2k": 350,
-    "Synth-ShellErrors": 350,
-    "Synth-Jupyter": 350,
+    "Synth-ShellErrors": 500,
+    "Synth-Jupyter": 500,
     "Jupyter-Code": 350,
-    "Synth-Execution": 350,
-    "Synth-Monitoring": 350,
-    "Synth-DatabaseSetup": 350,
+    "Synth-Execution": 500,
+    "Synth-Monitoring": 500,
+    "Synth-DatabaseSetup": 500,
     "HumanEval-Java": 164,
-    "Synth-AptInstall": 350,
-    "UltraChat": 350,
-    "Synth-DesktopSetup": 350,
+    "Synth-AptInstall": 500,
+    "UltraChat": 500,
+    "Synth-DesktopSetup": 500,
     "SD-Prompts-2M": 200,
-    "Synth-WebserverSetup": 350,
-    "Pythonic-Function-Calling": 350,
+    "Synth-WebserverSetup": 500,
+    "Pythonic-Function-Calling": 500,
     "Swift-Code-Edit": 10,
-    "Glaive-Code-Assistant": 350,
-    "File-Operations-Medium": 350,
+    "Glaive-Code-Assistant": 500,
+    "File-Operations-Medium": 500,
     "Swift-Code-RLVR": 350,
-    "Synth-SSHSetup": 350,
+    "Synth-SSHSetup": 500,
     "HumanEval-Rust": 164,
     "Synth-Commits": 350,
     "Synth-FIM": 350,
-    "Synth-Debugging": 350,
-    "Tool-Calls-SingleTurn": 350,
-    "Tool-Calls-Multiturn": 350,
+    "Synth-Debugging": 500,
+    "Tool-Calls-SingleTurn": 500,
+    "Tool-Calls-Multiturn": 500,
     "OpenAssistant": 800,
     "T2V-Sora-Preferences-2": 650,
     "T2V-Human-Preferences": 650,
@@ -100,22 +100,22 @@
     "Medical-O1-Reasoning-EN": 650,
     "OpenThoughts-114k": 350,
     "Bespoke-Stratos-17k": 350,
-    "Synth-FileOps": 150,
-    "Synth-EditLines": 150,
-    "Agentic-CoT-Coding": 150
+    "Synth-FileOps": 300,
+    "Synth-EditLines": 300,
+    "Agentic-CoT-Coding": 300
   },
   "modality_positions": {
     "text": {
-      "WildChat": 350,
+      "WildChat": 500,
       "Midjourney-Prompts": 200,
       "CodeParrot-Clean": 350,
-      "ShareGPT-Clean": 350,
+      "ShareGPT-Clean": 500,
       "Dolly-15k": 800,
       "Conversation-Summarization": 800,
       "HumanEval-JavaScript": 164,
       "OpenOrca": 800,
       "NoRobots": 800,
-      "Function-Calling-ChatML": 350,
+      "Function-Calling-ChatML": 500,
       "Python-Code-18k": 350,
       "Code-Feedback": 350,
       "HumanEval-CPP": 164,
@@ -123,20 +123,20 @@
       "SD-Prompts": 200,
       "Golang-Coder": 350,
       "HumanEval-Go": 164,
-      "Synth-APIGen": 350,
+      "Synth-APIGen": 500,
       "HumanEval-Python": 164,
       "Golang-QA-2k": 350,
       "Jupyter-Code": 350,
       "HumanEval-Java": 164,
-      "UltraChat": 350,
+      "UltraChat": 500,
       "SD-Prompts-2M": 200,
-      "Pythonic-Function-Calling": 350,
+      "Pythonic-Function-Calling": 500,
       "Swift-Code-Edit": 10,
-      "Glaive-Code-Assistant": 350,
+      "Glaive-Code-Assistant": 500,
       "Swift-Code-RLVR": 350,
       "HumanEval-Rust": 164,
-      "Tool-Calls-SingleTurn": 350,
-      "Tool-Calls-Multiturn": 350,
+      "Tool-Calls-SingleTurn": 500,
+      "Tool-Calls-Multiturn": 500,
       "OpenAssistant": 800,
       "SmolTalk-OpenHermes": 600,
       "SmolTalk-All": 600,
@@ -168,27 +168,27 @@
       "Synth-Commits": 350,
       "Synth-FIM": 350,
       "Synth-Diffs": 350,
-      "Synth-Monitoring": 350,
-      "Synth-FileOps": 150,
-      "Synth-Debugging": 350,
-      "Synth-Downloads": 350,
-      "Synth-ShellErrors": 350,
-      "Synth-DesktopSetup": 350,
-      "Synth-ShellExecution": 350,
-      "Synth-LanguageSetup": 350,
-      "Synth-DatabaseSetup": 350,
-      "Synth-MultiStepExecution": 350,
-      "Synth-Jupyter": 350,
-      "File-Operations-Medium": 350,
-      "Synth-ShellTimeout": 350,
-      "Synth-Docker": 350,
-      "Synth-SSHSetup": 350,
-      "Synth-EditLines": 150,
-      "Synth-AptInstall": 350,
-      "Synth-Execution": 350,
-      "Synth-PythonScripts": 350,
-      "Synth-WebserverSetup": 350,
-      "Agentic-CoT-Coding": 150
+      "Synth-Monitoring": 500,
+      "Synth-FileOps": 300,
+      "Synth-Debugging": 500,
+      "Synth-Downloads": 500,
+      "Synth-ShellErrors": 500,
+      "Synth-DesktopSetup": 500,
+      "Synth-ShellExecution": 500,
+      "Synth-LanguageSetup": 500,
+      "Synth-DatabaseSetup": 500,
+      "Synth-MultiStepExecution": 500,
+      "Synth-Jupyter": 500,
+      "File-Operations-Medium": 500,
+      "Synth-ShellTimeout": 500,
+      "Synth-Docker": 500,
+      "Synth-SSHSetup": 500,
+      "Synth-EditLines": 300,
+      "Synth-AptInstall": 500,
+      "Synth-Execution": 500,
+      "Synth-PythonScripts": 500,
+      "Synth-WebserverSetup": 500,
+      "Agentic-CoT-Coding": 300
     },
     "image": {
       "WebSight": 386,

trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_model_checkpoint": "/kaggle/working/xoron-final",
-  "best_metric": 3.3970277398874362,
+  "best_metric": 2.9528104483510056,
   "epoch": 3,
   "epochs_completed": 3,
   "global_step": 561,

training_state.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c68c3e0a999f4613a219fa4812a9e409690b7cabef93d848a043cf5c66e2b3b9
-size 1514917181
+oid sha256:0ee9691b252ac12027d0606006126d568cad36fa0777733f9d1069a70306095d
+size 5230529859