diff --git a/cosmos3_nano_stage_one_lora_r256_replicate_only_500k_20261103/attempt01/step_004000/checkpoint_validation.json b/cosmos3_nano_stage_one_lora_r256_replicate_only_500k_20261103/attempt01/step_004000/checkpoint_validation.json
new file mode 100644
index 0000000000000000000000000000000000000000..bd7a041b4d0e49a0ae4d0f139dd1a8bcb0e63322
--- /dev/null
+++ b/cosmos3_nano_stage_one_lora_r256_replicate_only_500k_20261103/attempt01/step_004000/checkpoint_validation.json
@@ -0,0 +1,3723 @@
+{
+  "all_finite": true,
+  "bytes": 2990807184,
+  "dtype": "float32",
+  "metadata": {
+    "base_identity": "nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92",
+    "config_identity": "cosmos3_nano_omni4d_stage_one_lora_r256_native_layer_masks_fp32_rope_full_checkpoint_retained_forward_parameters_global32",
+    "exact_training_resume": "false",
+    "format": "omni4d_trainable_weights_v1",
+    "parameter_names": "[\"language_model.model.layers.0.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.0.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.0.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.0.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.0.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.0.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.0.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.0.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.0.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.0.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.0.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.0.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.0.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.0.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.1.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.1.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.1.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.1.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.1.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.1.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.1.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.1.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.1.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.1.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.1.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.1.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.1.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.1.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.2.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.2.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.2.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.2.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.2.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.2.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.2.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.2.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.2.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.2.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.2.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.2.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.2.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.2.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.3.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.3.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.3.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.3.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.3.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.3.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.3.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.3.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.3.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.3.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.3.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.3.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.3.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.3.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.4.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.4.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.4.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.4.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.4.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.4.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.4.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.4.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.4.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.4.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.4.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.4.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.4.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.4.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.5.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.5.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.5.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.5.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.5.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.5.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.5.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.5.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.5.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.5.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.5.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.5.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.5.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.5.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.6.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.6.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.6.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.6.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.6.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.6.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.6.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.6.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.6.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.6.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.6.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.6.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.6.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.6.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.7.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.7.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.7.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.7.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.7.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.7.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.7.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.7.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.7.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.7.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.7.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.7.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.7.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.7.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.8.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.8.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.8.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.8.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.8.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.8.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.8.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.8.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.8.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.8.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.8.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.8.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.8.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.8.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.9.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.9.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.9.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.9.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.9.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.9.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.9.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.9.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.9.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.9.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.9.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.9.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.9.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.9.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.10.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.10.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.10.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.10.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.10.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.10.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.10.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.10.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.10.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.10.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.10.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.10.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.10.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.10.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.11.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.11.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.11.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.11.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.11.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.11.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.11.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.11.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.11.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.11.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.11.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.11.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.11.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.11.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.12.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.12.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.12.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.12.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.12.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.12.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.12.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.12.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.12.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.12.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.12.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.12.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.12.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.12.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.13.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.13.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.13.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.13.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.13.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.13.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.13.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.13.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.13.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.13.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.13.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.13.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.13.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.13.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.14.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.14.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.14.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.14.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.14.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.14.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.14.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.14.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.14.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.14.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.14.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.14.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.14.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.14.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.15.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.15.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.15.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.15.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.15.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.15.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.15.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.15.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.15.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.15.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.15.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.15.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.15.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.15.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.16.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.16.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.16.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.16.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.16.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.16.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.16.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.16.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.16.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.16.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.16.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.16.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.16.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.16.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.17.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.17.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.17.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.17.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.17.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.17.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.17.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.17.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.17.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.17.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.17.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.17.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.17.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.17.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.18.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.18.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.18.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.18.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.18.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.18.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.18.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.18.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.18.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.18.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.18.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.18.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.18.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.18.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.19.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.19.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.19.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.19.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.19.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.19.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.19.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.19.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.19.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.19.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.19.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.19.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.19.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.19.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.20.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.20.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.20.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.20.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.20.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.20.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.20.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.20.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.20.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.20.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.20.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.20.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.20.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.20.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.21.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.21.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.21.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.21.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.21.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.21.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.21.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.21.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.21.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.21.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.21.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.21.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.21.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.21.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.22.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.22.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.22.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.22.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.22.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.22.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.22.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.22.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.22.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.22.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.22.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.22.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.22.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.22.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.23.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.23.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.23.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.23.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.23.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.23.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.23.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.23.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.23.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.23.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.23.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.23.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.23.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.23.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.24.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.24.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.24.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.24.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.24.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.24.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.24.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.24.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.24.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.24.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.24.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.24.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.24.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.24.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.25.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.25.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.25.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.25.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.25.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.25.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.25.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.25.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.25.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.25.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.25.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.25.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.25.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.25.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.26.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.26.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.26.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.26.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.26.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.26.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.26.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.26.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.26.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.26.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.26.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.26.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.26.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.26.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.27.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.27.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.27.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.27.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.27.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.27.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.27.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.27.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.27.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.27.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.27.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.27.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.27.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.27.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.28.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.28.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.28.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.28.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.28.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.28.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.28.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.28.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.28.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.28.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.28.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.28.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.28.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.28.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.29.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.29.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.29.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.29.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.29.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.29.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.29.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.29.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.29.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.29.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.29.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.29.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.29.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.29.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.30.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.30.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.30.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.30.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.30.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.30.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.30.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.30.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.30.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.30.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.30.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.30.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.30.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.30.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.31.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.31.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.31.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.31.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.31.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.31.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.31.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.31.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.31.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.31.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.31.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.31.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.31.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.31.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.32.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.32.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.32.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.32.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.32.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.32.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.32.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.32.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.32.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.32.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.32.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.32.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.32.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.32.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.33.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.33.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.33.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.33.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.33.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.33.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.33.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.33.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.33.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.33.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.33.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.33.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.33.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.33.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.34.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.34.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.34.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.34.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.34.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.34.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.34.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.34.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.34.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.34.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.34.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.34.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.34.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.34.mlp_moe_gen.down_proj.lora_B.weight\", \"language_model.model.layers.35.self_attn.q_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.35.self_attn.q_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.35.self_attn.k_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.35.self_attn.k_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.35.self_attn.v_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.35.self_attn.v_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.35.self_attn.o_proj_moe_gen.lora_A.weight\", \"language_model.model.layers.35.self_attn.o_proj_moe_gen.lora_B.weight\", \"language_model.model.layers.35.mlp_moe_gen.gate_proj.lora_A.weight\", \"language_model.model.layers.35.mlp_moe_gen.gate_proj.lora_B.weight\", \"language_model.model.layers.35.mlp_moe_gen.up_proj.lora_A.weight\", \"language_model.model.layers.35.mlp_moe_gen.up_proj.lora_B.weight\", \"language_model.model.layers.35.mlp_moe_gen.down_proj.lora_A.weight\", \"language_model.model.layers.35.mlp_moe_gen.down_proj.lora_B.weight\", \"time_embedder.mlp.0.weight\", \"time_embedder.mlp.0.bias\", \"time_embedder.mlp.2.weight\", \"time_embedder.mlp.2.bias\", \"omni4d_stream_projector.input.video.weight\", \"omni4d_stream_projector.input.video.bias\", \"omni4d_stream_projector.input.pointmap.weight\", \"omni4d_stream_projector.input.pointmap.bias\", \"omni4d_stream_projector.input.camera.weight\", \"omni4d_stream_projector.input.camera.bias\", \"omni4d_stream_projector.input.dino.weight\", \"omni4d_stream_projector.input.dino.bias\", \"omni4d_stream_projector.input.track_video.weight\", \"omni4d_stream_projector.input.track_video.bias\", \"omni4d_stream_projector.dino_feature_norm.weight\", \"omni4d_stream_projector.dino_feature_norm.bias\", \"omni4d_stream_projector.output.video.weight\", \"omni4d_stream_projector.output.video.bias\", \"omni4d_stream_projector.output.pointmap.weight\", \"omni4d_stream_projector.output.pointmap.bias\", \"omni4d_stream_projector.output.camera.weight\", \"omni4d_stream_projector.output.camera.bias\", \"omni4d_stream_projector.output.dino.weight\", \"omni4d_stream_projector.output.dino.bias\", \"omni4d_stream_projector.output.track_video.weight\", \"omni4d_stream_projector.output.track_video.bias\"]",
+    "update": "4000"
+  },
+  "parameter_count": 747673344,
+  "sha256": "ca1b11df3fd31480aee3564c3a57782ea29e7e083f6e21922d1a27de532e1497",
+  "source": "/scratch/shared/engs2907/logs/vgg_forecast_4d/train/cosmos3_nano_five_stream_stage_one_lora_r256_replicate_only_500k_20261103/attempt01/trainable_overlays/step_004000.safetensors",
+  "source_stat": {
+    "device": 57,
+    "inode": 18133911060991891096,
+    "mtime_ns": 1793764496514355758,
+    "size": 2990807184
+  },
+  "status": "passed",
+  "tensor_count": 530,
+  "tensor_layout": {
+    "language_model.model.layers.0.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.0.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.0.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.0.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.0.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.0.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.0.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.0.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.0.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.0.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.0.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.0.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.0.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.0.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.1.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.1.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.1.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.1.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.1.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.1.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.1.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.1.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.1.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.1.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.1.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.1.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.1.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.1.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.10.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.10.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.10.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.10.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.10.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.10.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.10.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.10.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.10.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.10.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.10.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.10.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.10.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.10.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.11.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.11.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.11.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.11.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.11.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.11.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.11.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.11.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.11.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.11.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.11.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.11.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.11.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.11.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.12.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.12.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.12.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.12.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.12.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.12.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.12.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.12.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.12.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.12.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.12.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.12.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.12.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.12.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.13.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.13.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.13.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.13.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.13.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.13.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.13.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.13.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.13.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.13.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.13.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.13.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.13.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.13.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.14.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.14.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.14.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.14.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.14.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.14.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.14.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.14.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.14.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.14.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.14.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.14.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.14.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.14.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.15.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.15.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.15.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.15.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.15.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.15.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.15.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.15.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.15.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.15.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.15.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.15.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.15.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.15.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.16.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.16.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.16.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.16.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.16.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.16.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.16.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.16.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.16.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.16.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.16.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.16.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.16.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.16.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.17.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.17.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.17.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.17.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.17.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.17.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.17.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.17.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.17.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.17.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.17.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.17.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.17.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.17.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.18.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.18.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.18.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.18.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.18.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.18.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.18.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.18.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.18.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.18.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.18.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.18.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.18.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.18.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.19.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.19.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.19.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.19.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.19.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.19.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.19.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.19.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.19.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.19.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.19.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.19.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.19.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.19.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.2.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.2.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.2.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.2.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.2.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.2.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.2.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.2.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.2.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.2.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.2.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.2.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.2.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.2.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.20.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.20.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.20.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.20.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.20.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.20.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.20.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.20.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.20.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.20.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.20.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.20.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.20.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.20.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.21.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.21.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.21.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.21.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.21.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.21.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.21.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.21.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.21.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.21.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.21.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.21.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.21.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.21.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.22.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.22.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.22.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.22.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.22.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.22.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.22.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.22.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.22.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.22.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.22.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.22.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.22.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.22.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.23.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.23.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.23.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.23.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.23.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.23.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.23.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.23.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.23.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.23.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.23.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.23.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.23.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.23.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.24.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.24.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.24.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.24.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.24.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.24.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.24.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.24.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.24.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.24.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.24.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.24.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.24.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.24.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.25.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.25.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.25.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.25.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.25.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.25.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.25.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.25.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.25.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.25.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.25.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.25.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.25.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.25.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.26.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.26.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.26.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.26.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.26.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.26.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.26.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.26.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.26.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.26.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.26.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.26.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.26.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.26.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.27.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.27.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.27.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.27.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.27.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.27.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.27.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.27.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.27.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.27.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.27.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.27.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.27.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.27.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.28.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.28.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.28.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.28.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.28.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.28.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.28.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.28.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.28.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.28.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.28.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.28.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.28.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.28.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.29.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.29.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.29.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.29.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.29.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.29.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.29.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.29.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.29.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.29.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.29.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.29.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.29.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.29.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.3.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.3.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.3.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.3.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.3.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.3.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.3.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.3.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.3.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.3.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.3.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.3.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.3.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.3.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.30.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.30.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.30.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.30.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.30.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.30.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.30.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.30.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.30.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.30.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.30.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.30.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.30.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.30.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.31.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.31.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.31.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.31.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.31.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.31.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.31.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.31.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.31.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.31.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.31.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.31.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.31.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.31.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.32.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.32.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.32.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.32.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.32.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.32.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.32.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.32.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.32.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.32.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.32.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.32.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.32.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.32.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.33.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.33.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.33.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.33.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.33.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.33.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.33.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.33.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.33.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.33.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.33.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.33.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.33.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.33.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.34.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.34.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.34.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.34.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.34.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.34.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.34.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.34.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.34.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.34.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.34.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.34.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.34.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.34.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.35.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.35.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.35.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.35.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.35.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.35.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.35.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.35.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.35.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.35.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.35.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.35.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.35.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.35.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.4.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.4.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.4.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.4.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.4.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.4.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.4.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.4.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.4.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.4.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.4.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.4.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.4.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.4.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.5.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.5.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.5.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.5.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.5.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.5.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.5.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.5.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.5.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.5.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.5.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.5.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.5.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.5.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.6.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.6.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.6.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.6.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.6.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.6.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.6.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.6.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.6.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.6.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.6.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.6.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.6.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.6.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.7.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.7.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.7.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.7.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.7.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.7.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.7.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.7.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.7.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.7.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.7.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.7.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.7.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.7.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.8.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.8.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.8.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.8.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.8.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.8.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.8.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.8.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.8.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.8.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.8.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.8.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.8.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.8.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.9.mlp_moe_gen.down_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        12288
+      ]
+    },
+    "language_model.model.layers.9.mlp_moe_gen.down_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.9.mlp_moe_gen.gate_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.9.mlp_moe_gen.gate_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.9.mlp_moe_gen.up_proj.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.9.mlp_moe_gen.up_proj.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        12288,
+        256
+      ]
+    },
+    "language_model.model.layers.9.self_attn.k_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.9.self_attn.k_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "language_model.model.layers.9.self_attn.o_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.9.self_attn.o_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.9.self_attn.q_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.9.self_attn.q_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "language_model.model.layers.9.self_attn.v_proj_moe_gen.lora_A.weight": {
+      "dtype": "F32",
+      "shape": [
+        256,
+        4096
+      ]
+    },
+    "language_model.model.layers.9.self_attn.v_proj_moe_gen.lora_B.weight": {
+      "dtype": "F32",
+      "shape": [
+        1024,
+        256
+      ]
+    },
+    "omni4d_stream_projector.dino_feature_norm.bias": {
+      "dtype": "F32",
+      "shape": [
+        3072
+      ]
+    },
+    "omni4d_stream_projector.dino_feature_norm.weight": {
+      "dtype": "F32",
+      "shape": [
+        3072
+      ]
+    },
+    "omni4d_stream_projector.input.camera.bias": {
+      "dtype": "F32",
+      "shape": [
+        4096
+      ]
+    },
+    "omni4d_stream_projector.input.camera.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        192
+      ]
+    },
+    "omni4d_stream_projector.input.dino.bias": {
+      "dtype": "F32",
+      "shape": [
+        4096
+      ]
+    },
+    "omni4d_stream_projector.input.dino.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        3072
+      ]
+    },
+    "omni4d_stream_projector.input.pointmap.bias": {
+      "dtype": "F32",
+      "shape": [
+        4096
+      ]
+    },
+    "omni4d_stream_projector.input.pointmap.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        192
+      ]
+    },
+    "omni4d_stream_projector.input.track_video.bias": {
+      "dtype": "F32",
+      "shape": [
+        4096
+      ]
+    },
+    "omni4d_stream_projector.input.track_video.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        192
+      ]
+    },
+    "omni4d_stream_projector.input.video.bias": {
+      "dtype": "F32",
+      "shape": [
+        4096
+      ]
+    },
+    "omni4d_stream_projector.input.video.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        192
+      ]
+    },
+    "omni4d_stream_projector.output.camera.bias": {
+      "dtype": "F32",
+      "shape": [
+        192
+      ]
+    },
+    "omni4d_stream_projector.output.camera.weight": {
+      "dtype": "F32",
+      "shape": [
+        192,
+        4096
+      ]
+    },
+    "omni4d_stream_projector.output.dino.bias": {
+      "dtype": "F32",
+      "shape": [
+        3072
+      ]
+    },
+    "omni4d_stream_projector.output.dino.weight": {
+      "dtype": "F32",
+      "shape": [
+        3072,
+        4096
+      ]
+    },
+    "omni4d_stream_projector.output.pointmap.bias": {
+      "dtype": "F32",
+      "shape": [
+        192
+      ]
+    },
+    "omni4d_stream_projector.output.pointmap.weight": {
+      "dtype": "F32",
+      "shape": [
+        192,
+        4096
+      ]
+    },
+    "omni4d_stream_projector.output.track_video.bias": {
+      "dtype": "F32",
+      "shape": [
+        192
+      ]
+    },
+    "omni4d_stream_projector.output.track_video.weight": {
+      "dtype": "F32",
+      "shape": [
+        192,
+        4096
+      ]
+    },
+    "omni4d_stream_projector.output.video.bias": {
+      "dtype": "F32",
+      "shape": [
+        192
+      ]
+    },
+    "omni4d_stream_projector.output.video.weight": {
+      "dtype": "F32",
+      "shape": [
+        192,
+        4096
+      ]
+    },
+    "time_embedder.mlp.0.bias": {
+      "dtype": "F32",
+      "shape": [
+        4096
+      ]
+    },
+    "time_embedder.mlp.0.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        256
+      ]
+    },
+    "time_embedder.mlp.2.bias": {
+      "dtype": "F32",
+      "shape": [
+        4096
+      ]
+    },
+    "time_embedder.mlp.2.weight": {
+      "dtype": "F32",
+      "shape": [
+        4096,
+        4096
+      ]
+    }
+  },
+  "validated_at_utc": "2026-11-04T04:10:29.216377+00:00"
+}