final step=0

final/config.json

@@ -0,0 +1,83 @@
+{
+  "step": 0,
+  "epoch": 0,
+  "total_tokens": 0,
+  "best_val": Infinity,
+  "val_loss": NaN,
+  "mtp_val": NaN,
+  "fp8_mode": "torchao_rowwise",
+  "seq_len": 4096,
+  "config": {
+    "size_name": "7b-moe",
+    "vocab_size": 50304,
+    "d_model": 1536,
+    "n_layers": 30,
+    "n_heads": 12,
+    "n_kv_heads": 2,
+    "head_dim": 128,
+    "ffn_hidden": 6144,
+    "use_moe": true,
+    "n_dense_layers": 1,
+    "n_routed_experts": 64,
+    "n_shared_experts": 2,
+    "moe_top_k": 6,
+    "expert_hidden": 768,
+    "routed_scale": 1.0,
+    "bias_update_rate": 0.001,
+    "aux_loss_weight": 0.001,
+    "router_z_weight": 0.0001,
+    "global_every": 4,
+    "window": 1024,
+    "rope_local": 10000.0,
+    "rope_global": 500000.0,
+    "use_qk_norm": true,
+    "doc_masking": true,
+    "tie_embeddings": false,
+    "z_loss": 0.0001,
+    "n_ce_chunks": 8,
+    "use_mtp": true,
+    "mtp_weight": 0.3,
+    "resid_dropout": 0.0,
+    "attn_dropout": 0.0,
+    "use_fim": true,
+    "fim_rate": 0.6,
+    "fim_spm_rate": 0.5,
+    "eos_id": 50268,
+    "pad_id": 50272,
+    "fim_pre_id": 50269,
+    "fim_suf_id": 50270,
+    "fim_mid_id": 50271,
+    "fim_pad_id": -1,
+    "param_dtype_str": "bf16",
+    "stochastic_rounding": true,
+    "ckpt_attn": true,
+    "ckpt_ffn": true,
+    "muon_lr": 0.0022,
+    "adam_lr": 0.00045,
+    "muon_momentum": 0.95,
+    "muon_ns_steps": 5,
+    "weight_decay": 0.1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "grad_clip": 1.0,
+    "min_lr_ratio": 0.02,
+    "warmup_tokens": 100000000,
+    "decay_frac": 0.2,
+    "target_total_tokens": 10000000000,
+    "max_epochs": 4.0,
+    "seq_len": 4096,
+    "long_ctx_len": 16384,
+    "long_ctx_frac": 0.12,
+    "max_len": 16384,
+    "micro_batch": 16,
+    "grad_accum": 16,
+    "batch_ramp_tokens": 2000000000,
+    "log_every": 10,
+    "eval_every_steps": 500,
+    "eval_batches": 40,
+    "ckpt_every_min": 45.0,
+    "max_session_hours": 11.3,
+    "overfit_patience": 5,
+    "seed": 1337
+  }
+}

final/model-00000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba6c9cb5892b7cc474f44e804c30e0b4ad6e42c35005e5073a349c593a18469b
+size 3556361992

final/model-00001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16f289448c10f6e6c37e771cef1a463b8f2d4d1309abfec2cab7d4b3609c14b0
+size 3524905792

final/model-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2195b5baf835ac64242783713a2051c9e1c8d639333c1426cd412b42732f75a4
+size 3650524024

final/model-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:576a529f0c297ef08b5e7fe5755e0724ff3b1229d0a1481c70508495b03231fd
+size 3517424064

final/model-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9df3d5fd8365bd82c2490fb61ba13b72848d8d2cd1e51485e6ef7f80a2a04f50
+size 77090776

final/model.index.json

@@ -0,0 +1 @@
+{"weight_map": {"embed.weight": "model-00000.safetensors", "layers.0.pre_a.scale": "model-00000.safetensors", "layers.0.post_a.scale": "model-00000.safetensors", "layers.0.pre_f.scale": "model-00000.safetensors", "layers.0.post_f.scale": "model-00000.safetensors", "layers.0.attn.q.weight": "model-00000.safetensors", "layers.0.attn.k.weight": "model-00000.safetensors", "layers.0.attn.v.weight": "model-00000.safetensors", "layers.0.attn.o.weight": "model-00000.safetensors", "layers.0.attn.qn.scale": "model-00000.safetensors", "layers.0.attn.kn.scale": "model-00000.safetensors", "layers.0.ffn.gate.weight": "model-00000.safetensors", "layers.0.ffn.up.weight": "model-00000.safetensors", "layers.0.ffn.down.weight": "model-00000.safetensors", "layers.1.pre_a.scale": "model-00000.safetensors", "layers.1.post_a.scale": "model-00000.safetensors", "layers.1.pre_f.scale": "model-00000.safetensors", "layers.1.post_f.scale": "model-00000.safetensors", "layers.1.attn.q.weight": "model-00000.safetensors", "layers.1.attn.k.weight": "model-00000.safetensors", "layers.1.attn.v.weight": "model-00000.safetensors", "layers.1.attn.o.weight": "model-00000.safetensors", "layers.1.attn.qn.scale": "model-00000.safetensors", "layers.1.attn.kn.scale": "model-00000.safetensors", "layers.1.ffn.w_gate": "model-00000.safetensors", "layers.1.ffn.w_up": "model-00000.safetensors", "layers.1.ffn.w_down": "model-00000.safetensors", "layers.1.ffn.expert_bias": "model-00000.safetensors", "layers.1.ffn.router.weight": "model-00000.safetensors", "layers.1.ffn.shared.gate.weight": "model-00000.safetensors", "layers.1.ffn.shared.up.weight": "model-00000.safetensors", "layers.1.ffn.shared.down.weight": "model-00000.safetensors", "layers.2.pre_a.scale": "model-00000.safetensors", "layers.2.post_a.scale": "model-00000.safetensors", "layers.2.pre_f.scale": "model-00000.safetensors", "layers.2.post_f.scale": "model-00000.safetensors", "layers.2.attn.q.weight": "model-00000.safetensors", "layers.2.attn.k.weight": "model-00000.safetensors", "layers.2.attn.v.weight": "model-00000.safetensors", "layers.2.attn.o.weight": "model-00000.safetensors", "layers.2.attn.qn.scale": "model-00000.safetensors", "layers.2.attn.kn.scale": "model-00000.safetensors", "layers.2.ffn.w_gate": "model-00000.safetensors", "layers.2.ffn.w_up": "model-00000.safetensors", "layers.2.ffn.w_down": "model-00000.safetensors", "layers.2.ffn.expert_bias": "model-00000.safetensors", "layers.2.ffn.router.weight": "model-00000.safetensors", "layers.2.ffn.shared.gate.weight": "model-00000.safetensors", "layers.2.ffn.shared.up.weight": "model-00000.safetensors", "layers.2.ffn.shared.down.weight": "model-00000.safetensors", "layers.3.pre_a.scale": "model-00000.safetensors", "layers.3.post_a.scale": "model-00000.safetensors", "layers.3.pre_f.scale": "model-00000.safetensors", "layers.3.post_f.scale": "model-00000.safetensors", "layers.3.attn.q.weight": "model-00000.safetensors", "layers.3.attn.k.weight": "model-00000.safetensors", "layers.3.attn.v.weight": "model-00000.safetensors", "layers.3.attn.o.weight": "model-00000.safetensors", "layers.3.attn.qn.scale": "model-00000.safetensors", "layers.3.attn.kn.scale": "model-00000.safetensors", "layers.3.ffn.w_gate": "model-00000.safetensors", "layers.3.ffn.w_up": "model-00000.safetensors", "layers.3.ffn.w_down": "model-00000.safetensors", "layers.3.ffn.expert_bias": "model-00000.safetensors", "layers.3.ffn.router.weight": "model-00000.safetensors", "layers.3.ffn.shared.gate.weight": "model-00000.safetensors", "layers.3.ffn.shared.up.weight": "model-00000.safetensors", "layers.3.ffn.shared.down.weight": "model-00000.safetensors", "layers.4.pre_a.scale": "model-00000.safetensors", "layers.4.post_a.scale": "model-00000.safetensors", "layers.4.pre_f.scale": "model-00000.safetensors", "layers.4.post_f.scale": "model-00000.safetensors", "layers.4.attn.q.weight": "model-00000.safetensors", "layers.4.attn.k.weight": "model-00000.safetensors", "layers.4.attn.v.weight": "model-00000.safetensors", "layers.4.attn.o.weight": "model-00000.safetensors", "layers.4.attn.qn.scale": "model-00000.safetensors", "layers.4.attn.kn.scale": "model-00000.safetensors", "layers.4.ffn.w_gate": "model-00000.safetensors", "layers.4.ffn.w_up": "model-00000.safetensors", "layers.4.ffn.w_down": "model-00000.safetensors", "layers.4.ffn.expert_bias": "model-00000.safetensors", "layers.4.ffn.router.weight": "model-00000.safetensors", "layers.4.ffn.shared.gate.weight": "model-00000.safetensors", "layers.4.ffn.shared.up.weight": "model-00000.safetensors", "layers.4.ffn.shared.down.weight": "model-00000.safetensors", "layers.5.pre_a.scale": "model-00000.safetensors", "layers.5.post_a.scale": "model-00000.safetensors", "layers.5.pre_f.scale": "model-00000.safetensors", "layers.5.post_f.scale": "model-00000.safetensors", "layers.5.attn.q.weight": "model-00000.safetensors", "layers.5.attn.k.weight": "model-00000.safetensors", "layers.5.attn.v.weight": "model-00000.safetensors", "layers.5.attn.o.weight": "model-00000.safetensors", "layers.5.attn.qn.scale": "model-00000.safetensors", "layers.5.attn.kn.scale": "model-00000.safetensors", "layers.5.ffn.w_gate": "model-00000.safetensors", "layers.5.ffn.w_up": "model-00000.safetensors", "layers.5.ffn.w_down": "model-00000.safetensors", "layers.5.ffn.expert_bias": "model-00000.safetensors", "layers.5.ffn.router.weight": "model-00000.safetensors", "layers.5.ffn.shared.gate.weight": "model-00000.safetensors", "layers.5.ffn.shared.up.weight": "model-00000.safetensors", "layers.5.ffn.shared.down.weight": "model-00000.safetensors", "layers.6.pre_a.scale": "model-00000.safetensors", "layers.6.post_a.scale": "model-00000.safetensors", "layers.6.pre_f.scale": "model-00000.safetensors", "layers.6.post_f.scale": "model-00000.safetensors", "layers.6.attn.q.weight": "model-00000.safetensors", "layers.6.attn.k.weight": "model-00000.safetensors", "layers.6.attn.v.weight": "model-00000.safetensors", "layers.6.attn.o.weight": "model-00000.safetensors", "layers.6.attn.qn.scale": "model-00000.safetensors", "layers.6.attn.kn.scale": "model-00000.safetensors", "layers.6.ffn.w_gate": "model-00000.safetensors", "layers.6.ffn.w_up": "model-00000.safetensors", "layers.6.ffn.w_down": "model-00000.safetensors", "layers.6.ffn.expert_bias": "model-00000.safetensors", "layers.6.ffn.router.weight": "model-00000.safetensors", "layers.6.ffn.shared.gate.weight": "model-00000.safetensors", "layers.6.ffn.shared.up.weight": "model-00000.safetensors", "layers.6.ffn.shared.down.weight": "model-00000.safetensors", "layers.7.pre_a.scale": "model-00000.safetensors", "layers.7.post_a.scale": "model-00000.safetensors", "layers.7.pre_f.scale": "model-00000.safetensors", "layers.7.post_f.scale": "model-00000.safetensors", "layers.7.attn.q.weight": "model-00000.safetensors", "layers.7.attn.k.weight": "model-00000.safetensors", "layers.7.attn.v.weight": "model-00000.safetensors", "layers.7.attn.o.weight": "model-00000.safetensors", "layers.7.attn.qn.scale": "model-00000.safetensors", "layers.7.attn.kn.scale": "model-00000.safetensors", "layers.7.ffn.w_gate": "model-00000.safetensors", "layers.7.ffn.w_up": "model-00000.safetensors", "layers.7.ffn.w_down": "model-00000.safetensors", "layers.7.ffn.expert_bias": "model-00001.safetensors", "layers.7.ffn.router.weight": "model-00001.safetensors", "layers.7.ffn.shared.gate.weight": "model-00001.safetensors", "layers.7.ffn.shared.up.weight": "model-00001.safetensors", "layers.7.ffn.shared.down.weight": "model-00001.safetensors", "layers.8.pre_a.scale": "model-00001.safetensors", "layers.8.post_a.scale": "model-00001.safetensors", "layers.8.pre_f.scale": "model-00001.safetensors", "layers.8.post_f.scale": "model-00001.safetensors", "layers.8.attn.q.weight": "model-00001.safetensors", "layers.8.attn.k.weight": "model-00001.safetensors", "layers.8.attn.v.weight": "model-00001.safetensors", "layers.8.attn.o.weight": "model-00001.safetensors", "layers.8.attn.qn.scale": "model-00001.safetensors", "layers.8.attn.kn.scale": "model-00001.safetensors", "layers.8.ffn.w_gate": "model-00001.safetensors", "layers.8.ffn.w_up": "model-00001.safetensors", "layers.8.ffn.w_down": "model-00001.safetensors", "layers.8.ffn.expert_bias": "model-00001.safetensors", "layers.8.ffn.router.weight": "model-00001.safetensors", "layers.8.ffn.shared.gate.weight": "model-00001.safetensors", "layers.8.ffn.shared.up.weight": "model-00001.safetensors", "layers.8.ffn.shared.down.weight": "model-00001.safetensors", "layers.9.pre_a.scale": "model-00001.safetensors", "layers.9.post_a.scale": "model-00001.safetensors", "layers.9.pre_f.scale": "model-00001.safetensors", "layers.9.post_f.scale": "model-00001.safetensors", "layers.9.attn.q.weight": "model-00001.safetensors", "layers.9.attn.k.weight": "model-00001.safetensors", "layers.9.attn.v.weight": "model-00001.safetensors", "layers.9.attn.o.weight": "model-00001.safetensors", "layers.9.attn.qn.scale": "model-00001.safetensors", "layers.9.attn.kn.scale": "model-00001.safetensors", "layers.9.ffn.w_gate": "model-00001.safetensors", "layers.9.ffn.w_up": "model-00001.safetensors", "layers.9.ffn.w_down": "model-00001.safetensors", "layers.9.ffn.expert_bias": "model-00001.safetensors", "layers.9.ffn.router.weight": "model-00001.safetensors", "layers.9.ffn.shared.gate.weight": "model-00001.safetensors", "layers.9.ffn.shared.up.weight": "model-00001.safetensors", "layers.9.ffn.shared.down.weight": "model-00001.safetensors", "layers.10.pre_a.scale": "model-00001.safetensors", "layers.10.post_a.scale": "model-00001.safetensors", "layers.10.pre_f.scale": "model-00001.safetensors", "layers.10.post_f.scale": "model-00001.safetensors", "layers.10.attn.q.weight": "model-00001.safetensors", "layers.10.attn.k.weight": "model-00001.safetensors", "layers.10.attn.v.weight": "model-00001.safetensors", "layers.10.attn.o.weight": "model-00001.safetensors", "layers.10.attn.qn.scale": "model-00001.safetensors", "layers.10.attn.kn.scale": "model-00001.safetensors", "layers.10.ffn.w_gate": "model-00001.safetensors", "layers.10.ffn.w_up": "model-00001.safetensors", "layers.10.ffn.w_down": "model-00001.safetensors", "layers.10.ffn.expert_bias": "model-00001.safetensors", "layers.10.ffn.router.weight": "model-00001.safetensors", "layers.10.ffn.shared.gate.weight": "model-00001.safetensors", "layers.10.ffn.shared.up.weight": "model-00001.safetensors", "layers.10.ffn.shared.down.weight": "model-00001.safetensors", "layers.11.pre_a.scale": "model-00001.safetensors", "layers.11.post_a.scale": "model-00001.safetensors", "layers.11.pre_f.scale": "model-00001.safetensors", "layers.11.post_f.scale": "model-00001.safetensors", "layers.11.attn.q.weight": "model-00001.safetensors", "layers.11.attn.k.weight": "model-00001.safetensors", "layers.11.attn.v.weight": "model-00001.safetensors", "layers.11.attn.o.weight": "model-00001.safetensors", "layers.11.attn.qn.scale": "model-00001.safetensors", "layers.11.attn.kn.scale": "model-00001.safetensors", "layers.11.ffn.w_gate": "model-00001.safetensors", "layers.11.ffn.w_up": "model-00001.safetensors", "layers.11.ffn.w_down": "model-00001.safetensors", "layers.11.ffn.expert_bias": "model-00001.safetensors", "layers.11.ffn.router.weight": "model-00001.safetensors", "layers.11.ffn.shared.gate.weight": "model-00001.safetensors", "layers.11.ffn.shared.up.weight": "model-00001.safetensors", "layers.11.ffn.shared.down.weight": "model-00001.safetensors", "layers.12.pre_a.scale": "model-00001.safetensors", "layers.12.post_a.scale": "model-00001.safetensors", "layers.12.pre_f.scale": "model-00001.safetensors", "layers.12.post_f.scale": "model-00001.safetensors", "layers.12.attn.q.weight": "model-00001.safetensors", "layers.12.attn.k.weight": "model-00001.safetensors", "layers.12.attn.v.weight": "model-00001.safetensors", "layers.12.attn.o.weight": "model-00001.safetensors", "layers.12.attn.qn.scale": "model-00001.safetensors", "layers.12.attn.kn.scale": "model-00001.safetensors", "layers.12.ffn.w_gate": "model-00001.safetensors", "layers.12.ffn.w_up": "model-00001.safetensors", "layers.12.ffn.w_down": "model-00001.safetensors", "layers.12.ffn.expert_bias": "model-00001.safetensors", "layers.12.ffn.router.weight": "model-00001.safetensors", "layers.12.ffn.shared.gate.weight": "model-00001.safetensors", "layers.12.ffn.shared.up.weight": "model-00001.safetensors", "layers.12.ffn.shared.down.weight": "model-00001.safetensors", "layers.13.pre_a.scale": "model-00001.safetensors", "layers.13.post_a.scale": "model-00001.safetensors", "layers.13.pre_f.scale": "model-00001.safetensors", "layers.13.post_f.scale": "model-00001.safetensors", "layers.13.attn.q.weight": "model-00001.safetensors", "layers.13.attn.k.weight": "model-00001.safetensors", "layers.13.attn.v.weight": "model-00001.safetensors", "layers.13.attn.o.weight": "model-00001.safetensors", "layers.13.attn.qn.scale": "model-00001.safetensors", "layers.13.attn.kn.scale": "model-00001.safetensors", "layers.13.ffn.w_gate": "model-00001.safetensors", "layers.13.ffn.w_up": "model-00001.safetensors", "layers.13.ffn.w_down": "model-00001.safetensors", "layers.13.ffn.expert_bias": "model-00001.safetensors", "layers.13.ffn.router.weight": "model-00001.safetensors", "layers.13.ffn.shared.gate.weight": "model-00001.safetensors", "layers.13.ffn.shared.up.weight": "model-00001.safetensors", "layers.13.ffn.shared.down.weight": "model-00001.safetensors", "layers.14.pre_a.scale": "model-00001.safetensors", "layers.14.post_a.scale": "model-00001.safetensors", "layers.14.pre_f.scale": "model-00001.safetensors", "layers.14.post_f.scale": "model-00001.safetensors", "layers.14.attn.q.weight": "model-00001.safetensors", "layers.14.attn.k.weight": "model-00001.safetensors", "layers.14.attn.v.weight": "model-00001.safetensors", "layers.14.attn.o.weight": "model-00001.safetensors", "layers.14.attn.qn.scale": "model-00001.safetensors", "layers.14.attn.kn.scale": "model-00001.safetensors", "layers.14.ffn.w_gate": "model-00001.safetensors", "layers.14.ffn.w_up": "model-00001.safetensors", "layers.14.ffn.w_down": "model-00001.safetensors", "layers.14.ffn.expert_bias": "model-00001.safetensors", "layers.14.ffn.router.weight": "model-00001.safetensors", "layers.14.ffn.shared.gate.weight": "model-00001.safetensors", "layers.14.ffn.shared.up.weight": "model-00001.safetensors", "layers.14.ffn.shared.down.weight": "model-00001.safetensors", "layers.15.pre_a.scale": "model-00001.safetensors", "layers.15.post_a.scale": "model-00001.safetensors", "layers.15.pre_f.scale": "model-00001.safetensors", "layers.15.post_f.scale": "model-00001.safetensors", "layers.15.attn.q.weight": "model-00001.safetensors", "layers.15.attn.k.weight": "model-00001.safetensors", "layers.15.attn.v.weight": "model-00001.safetensors", "layers.15.attn.o.weight": "model-00001.safetensors", "layers.15.attn.qn.scale": "model-00001.safetensors", "layers.15.attn.kn.scale": "model-00001.safetensors", "layers.15.ffn.w_gate": "model-00001.safetensors", "layers.15.ffn.w_up": "model-00002.safetensors", "layers.15.ffn.w_down": "model-00002.safetensors", "layers.15.ffn.expert_bias": "model-00002.safetensors", "layers.15.ffn.router.weight": "model-00002.safetensors", "layers.15.ffn.shared.gate.weight": "model-00002.safetensors", "layers.15.ffn.shared.up.weight": "model-00002.safetensors", "layers.15.ffn.shared.down.weight": "model-00002.safetensors", "layers.16.pre_a.scale": "model-00002.safetensors", "layers.16.post_a.scale": "model-00002.safetensors", "layers.16.pre_f.scale": "model-00002.safetensors", "layers.16.post_f.scale": "model-00002.safetensors", "layers.16.attn.q.weight": "model-00002.safetensors", "layers.16.attn.k.weight": "model-00002.safetensors", "layers.16.attn.v.weight": "model-00002.safetensors", "layers.16.attn.o.weight": "model-00002.safetensors", "layers.16.attn.qn.scale": "model-00002.safetensors", "layers.16.attn.kn.scale": "model-00002.safetensors", "layers.16.ffn.w_gate": "model-00002.safetensors", "layers.16.ffn.w_up": "model-00002.safetensors", "layers.16.ffn.w_down": "model-00002.safetensors", "layers.16.ffn.expert_bias": "model-00002.safetensors", "layers.16.ffn.router.weight": "model-00002.safetensors", "layers.16.ffn.shared.gate.weight": "model-00002.safetensors", "layers.16.ffn.shared.up.weight": "model-00002.safetensors", "layers.16.ffn.shared.down.weight": "model-00002.safetensors", "layers.17.pre_a.scale": "model-00002.safetensors", "layers.17.post_a.scale": "model-00002.safetensors", "layers.17.pre_f.scale": "model-00002.safetensors", "layers.17.post_f.scale": "model-00002.safetensors", "layers.17.attn.q.weight": "model-00002.safetensors", "layers.17.attn.k.weight": "model-00002.safetensors", "layers.17.attn.v.weight": "model-00002.safetensors", "layers.17.attn.o.weight": "model-00002.safetensors", "layers.17.attn.qn.scale": "model-00002.safetensors", "layers.17.attn.kn.scale": "model-00002.safetensors", "layers.17.ffn.w_gate": "model-00002.safetensors", "layers.17.ffn.w_up": "model-00002.safetensors", "layers.17.ffn.w_down": "model-00002.safetensors", "layers.17.ffn.expert_bias": "model-00002.safetensors", "layers.17.ffn.router.weight": "model-00002.safetensors", "layers.17.ffn.shared.gate.weight": "model-00002.safetensors", "layers.17.ffn.shared.up.weight": "model-00002.safetensors", "layers.17.ffn.shared.down.weight": "model-00002.safetensors", "layers.18.pre_a.scale": "model-00002.safetensors", "layers.18.post_a.scale": "model-00002.safetensors", "layers.18.pre_f.scale": "model-00002.safetensors", "layers.18.post_f.scale": "model-00002.safetensors", "layers.18.attn.q.weight": "model-00002.safetensors", "layers.18.attn.k.weight": "model-00002.safetensors", "layers.18.attn.v.weight": "model-00002.safetensors", "layers.18.attn.o.weight": "model-00002.safetensors", "layers.18.attn.qn.scale": "model-00002.safetensors", "layers.18.attn.kn.scale": "model-00002.safetensors", "layers.18.ffn.w_gate": "model-00002.safetensors", "layers.18.ffn.w_up": "model-00002.safetensors", "layers.18.ffn.w_down": "model-00002.safetensors", "layers.18.ffn.expert_bias": "model-00002.safetensors", "layers.18.ffn.router.weight": "model-00002.safetensors", "layers.18.ffn.shared.gate.weight": "model-00002.safetensors", "layers.18.ffn.shared.up.weight": "model-00002.safetensors", "layers.18.ffn.shared.down.weight": "model-00002.safetensors", "layers.19.pre_a.scale": "model-00002.safetensors", "layers.19.post_a.scale": "model-00002.safetensors", "layers.19.pre_f.scale": "model-00002.safetensors", "layers.19.post_f.scale": "model-00002.safetensors", "layers.19.attn.q.weight": "model-00002.safetensors", "layers.19.attn.k.weight": "model-00002.safetensors", "layers.19.attn.v.weight": "model-00002.safetensors", "layers.19.attn.o.weight": "model-00002.safetensors", "layers.19.attn.qn.scale": "model-00002.safetensors", "layers.19.attn.kn.scale": "model-00002.safetensors", "layers.19.ffn.w_gate": "model-00002.safetensors", "layers.19.ffn.w_up": "model-00002.safetensors", "layers.19.ffn.w_down": "model-00002.safetensors", "layers.19.ffn.expert_bias": "model-00002.safetensors", "layers.19.ffn.router.weight": "model-00002.safetensors", "layers.19.ffn.shared.gate.weight": "model-00002.safetensors", "layers.19.ffn.shared.up.weight": "model-00002.safetensors", "layers.19.ffn.shared.down.weight": "model-00002.safetensors", "layers.20.pre_a.scale": "model-00002.safetensors", "layers.20.post_a.scale": "model-00002.safetensors", "layers.20.pre_f.scale": "model-00002.safetensors", "layers.20.post_f.scale": "model-00002.safetensors", "layers.20.attn.q.weight": "model-00002.safetensors", "layers.20.attn.k.weight": "model-00002.safetensors", "layers.20.attn.v.weight": "model-00002.safetensors", "layers.20.attn.o.weight": "model-00002.safetensors", "layers.20.attn.qn.scale": "model-00002.safetensors", "layers.20.attn.kn.scale": "model-00002.safetensors", "layers.20.ffn.w_gate": "model-00002.safetensors", "layers.20.ffn.w_up": "model-00002.safetensors", "layers.20.ffn.w_down": "model-00002.safetensors", "layers.20.ffn.expert_bias": "model-00002.safetensors", "layers.20.ffn.router.weight": "model-00002.safetensors", "layers.20.ffn.shared.gate.weight": "model-00002.safetensors", "layers.20.ffn.shared.up.weight": "model-00002.safetensors", "layers.20.ffn.shared.down.weight": "model-00002.safetensors", "layers.21.pre_a.scale": "model-00002.safetensors", "layers.21.post_a.scale": "model-00002.safetensors", "layers.21.pre_f.scale": "model-00002.safetensors", "layers.21.post_f.scale": "model-00002.safetensors", "layers.21.attn.q.weight": "model-00002.safetensors", "layers.21.attn.k.weight": "model-00002.safetensors", "layers.21.attn.v.weight": "model-00002.safetensors", "layers.21.attn.o.weight": "model-00002.safetensors", "layers.21.attn.qn.scale": "model-00002.safetensors", "layers.21.attn.kn.scale": "model-00002.safetensors", "layers.21.ffn.w_gate": "model-00002.safetensors", "layers.21.ffn.w_up": "model-00002.safetensors", "layers.21.ffn.w_down": "model-00002.safetensors", "layers.21.ffn.expert_bias": "model-00002.safetensors", "layers.21.ffn.router.weight": "model-00002.safetensors", "layers.21.ffn.shared.gate.weight": "model-00002.safetensors", "layers.21.ffn.shared.up.weight": "model-00002.safetensors", "layers.21.ffn.shared.down.weight": "model-00002.safetensors", "layers.22.pre_a.scale": "model-00002.safetensors", "layers.22.post_a.scale": "model-00002.safetensors", "layers.22.pre_f.scale": "model-00002.safetensors", "layers.22.post_f.scale": "model-00002.safetensors", "layers.22.attn.q.weight": "model-00002.safetensors", "layers.22.attn.k.weight": "model-00002.safetensors", "layers.22.attn.v.weight": "model-00002.safetensors", "layers.22.attn.o.weight": "model-00002.safetensors", "layers.22.attn.qn.scale": "model-00002.safetensors", "layers.22.attn.kn.scale": "model-00002.safetensors", "layers.22.ffn.w_gate": "model-00002.safetensors", "layers.22.ffn.w_up": "model-00002.safetensors", "layers.22.ffn.w_down": "model-00002.safetensors", "layers.22.ffn.expert_bias": "model-00003.safetensors", "layers.22.ffn.router.weight": "model-00003.safetensors", "layers.22.ffn.shared.gate.weight": "model-00003.safetensors", "layers.22.ffn.shared.up.weight": "model-00003.safetensors", "layers.22.ffn.shared.down.weight": "model-00003.safetensors", "layers.23.pre_a.scale": "model-00003.safetensors", "layers.23.post_a.scale": "model-00003.safetensors", "layers.23.pre_f.scale": "model-00003.safetensors", "layers.23.post_f.scale": "model-00003.safetensors", "layers.23.attn.q.weight": "model-00003.safetensors", "layers.23.attn.k.weight": "model-00003.safetensors", "layers.23.attn.v.weight": "model-00003.safetensors", "layers.23.attn.o.weight": "model-00003.safetensors", "layers.23.attn.qn.scale": "model-00003.safetensors", "layers.23.attn.kn.scale": "model-00003.safetensors", "layers.23.ffn.w_gate": "model-00003.safetensors", "layers.23.ffn.w_up": "model-00003.safetensors", "layers.23.ffn.w_down": "model-00003.safetensors", "layers.23.ffn.expert_bias": "model-00003.safetensors", "layers.23.ffn.router.weight": "model-00003.safetensors", "layers.23.ffn.shared.gate.weight": "model-00003.safetensors", "layers.23.ffn.shared.up.weight": "model-00003.safetensors", "layers.23.ffn.shared.down.weight": "model-00003.safetensors", "layers.24.pre_a.scale": "model-00003.safetensors", "layers.24.post_a.scale": "model-00003.safetensors", "layers.24.pre_f.scale": "model-00003.safetensors", "layers.24.post_f.scale": "model-00003.safetensors", "layers.24.attn.q.weight": "model-00003.safetensors", "layers.24.attn.k.weight": "model-00003.safetensors", "layers.24.attn.v.weight": "model-00003.safetensors", "layers.24.attn.o.weight": "model-00003.safetensors", "layers.24.attn.qn.scale": "model-00003.safetensors", "layers.24.attn.kn.scale": "model-00003.safetensors", "layers.24.ffn.w_gate": "model-00003.safetensors", "layers.24.ffn.w_up": "model-00003.safetensors", "layers.24.ffn.w_down": "model-00003.safetensors", "layers.24.ffn.expert_bias": "model-00003.safetensors", "layers.24.ffn.router.weight": "model-00003.safetensors", "layers.24.ffn.shared.gate.weight": "model-00003.safetensors", "layers.24.ffn.shared.up.weight": "model-00003.safetensors", "layers.24.ffn.shared.down.weight": "model-00003.safetensors", "layers.25.pre_a.scale": "model-00003.safetensors", "layers.25.post_a.scale": "model-00003.safetensors", "layers.25.pre_f.scale": "model-00003.safetensors", "layers.25.post_f.scale": "model-00003.safetensors", "layers.25.attn.q.weight": "model-00003.safetensors", "layers.25.attn.k.weight": "model-00003.safetensors", "layers.25.attn.v.weight": "model-00003.safetensors", "layers.25.attn.o.weight": "model-00003.safetensors", "layers.25.attn.qn.scale": "model-00003.safetensors", "layers.25.attn.kn.scale": "model-00003.safetensors", "layers.25.ffn.w_gate": "model-00003.safetensors", "layers.25.ffn.w_up": "model-00003.safetensors", "layers.25.ffn.w_down": "model-00003.safetensors", "layers.25.ffn.expert_bias": "model-00003.safetensors", "layers.25.ffn.router.weight": "model-00003.safetensors", "layers.25.ffn.shared.gate.weight": "model-00003.safetensors", "layers.25.ffn.shared.up.weight": "model-00003.safetensors", "layers.25.ffn.shared.down.weight": "model-00003.safetensors", "layers.26.pre_a.scale": "model-00003.safetensors", "layers.26.post_a.scale": "model-00003.safetensors", "layers.26.pre_f.scale": "model-00003.safetensors", "layers.26.post_f.scale": "model-00003.safetensors", "layers.26.attn.q.weight": "model-00003.safetensors", "layers.26.attn.k.weight": "model-00003.safetensors", "layers.26.attn.v.weight": "model-00003.safetensors", "layers.26.attn.o.weight": "model-00003.safetensors", "layers.26.attn.qn.scale": "model-00003.safetensors", "layers.26.attn.kn.scale": "model-00003.safetensors", "layers.26.ffn.w_gate": "model-00003.safetensors", "layers.26.ffn.w_up": "model-00003.safetensors", "layers.26.ffn.w_down": "model-00003.safetensors", "layers.26.ffn.expert_bias": "model-00003.safetensors", "layers.26.ffn.router.weight": "model-00003.safetensors", "layers.26.ffn.shared.gate.weight": "model-00003.safetensors", "layers.26.ffn.shared.up.weight": "model-00003.safetensors", "layers.26.ffn.shared.down.weight": "model-00003.safetensors", "layers.27.pre_a.scale": "model-00003.safetensors", "layers.27.post_a.scale": "model-00003.safetensors", "layers.27.pre_f.scale": "model-00003.safetensors", "layers.27.post_f.scale": "model-00003.safetensors", "layers.27.attn.q.weight": "model-00003.safetensors", "layers.27.attn.k.weight": "model-00003.safetensors", "layers.27.attn.v.weight": "model-00003.safetensors", "layers.27.attn.o.weight": "model-00003.safetensors", "layers.27.attn.qn.scale": "model-00003.safetensors", "layers.27.attn.kn.scale": "model-00003.safetensors", "layers.27.ffn.w_gate": "model-00003.safetensors", "layers.27.ffn.w_up": "model-00003.safetensors", "layers.27.ffn.w_down": "model-00003.safetensors", "layers.27.ffn.expert_bias": "model-00003.safetensors", "layers.27.ffn.router.weight": "model-00003.safetensors", "layers.27.ffn.shared.gate.weight": "model-00003.safetensors", "layers.27.ffn.shared.up.weight": "model-00003.safetensors", "layers.27.ffn.shared.down.weight": "model-00003.safetensors", "layers.28.pre_a.scale": "model-00003.safetensors", "layers.28.post_a.scale": "model-00003.safetensors", "layers.28.pre_f.scale": "model-00003.safetensors", "layers.28.post_f.scale": "model-00003.safetensors", "layers.28.attn.q.weight": "model-00003.safetensors", "layers.28.attn.k.weight": "model-00003.safetensors", "layers.28.attn.v.weight": "model-00003.safetensors", "layers.28.attn.o.weight": "model-00003.safetensors", "layers.28.attn.qn.scale": "model-00003.safetensors", "layers.28.attn.kn.scale": "model-00003.safetensors", "layers.28.ffn.w_gate": "model-00003.safetensors", "layers.28.ffn.w_up": "model-00003.safetensors", "layers.28.ffn.w_down": "model-00003.safetensors", "layers.28.ffn.expert_bias": "model-00003.safetensors", "layers.28.ffn.router.weight": "model-00003.safetensors", "layers.28.ffn.shared.gate.weight": "model-00003.safetensors", "layers.28.ffn.shared.up.weight": "model-00003.safetensors", "layers.28.ffn.shared.down.weight": "model-00003.safetensors", "layers.29.pre_a.scale": "model-00003.safetensors", "layers.29.post_a.scale": "model-00003.safetensors", "layers.29.pre_f.scale": "model-00003.safetensors", "layers.29.post_f.scale": "model-00003.safetensors", "layers.29.attn.q.weight": "model-00003.safetensors", "layers.29.attn.k.weight": "model-00003.safetensors", "layers.29.attn.v.weight": "model-00003.safetensors", "layers.29.attn.o.weight": "model-00003.safetensors", "layers.29.attn.qn.scale": "model-00003.safetensors", "layers.29.attn.kn.scale": "model-00003.safetensors", "layers.29.ffn.w_gate": "model-00003.safetensors", "layers.29.ffn.w_up": "model-00003.safetensors", "layers.29.ffn.w_down": "model-00003.safetensors", "layers.29.ffn.expert_bias": "model-00003.safetensors", "layers.29.ffn.router.weight": "model-00003.safetensors", "layers.29.ffn.shared.gate.weight": "model-00003.safetensors", "layers.29.ffn.shared.up.weight": "model-00003.safetensors", "layers.29.ffn.shared.down.weight": "model-00003.safetensors", "norm.scale": "model-00003.safetensors", "lm_head.weight": "model-00003.safetensors", "mtp_norm_h.scale": "model-00004.safetensors", "mtp_norm_e.scale": "model-00004.safetensors", "mtp_proj.weight": "model-00004.safetensors", "mtp_block.pre_a.scale": "model-00004.safetensors", "mtp_block.post_a.scale": "model-00004.safetensors", "mtp_block.pre_f.scale": "model-00004.safetensors", "mtp_block.post_f.scale": "model-00004.safetensors", "mtp_block.attn.q.weight": "model-00004.safetensors", "mtp_block.attn.k.weight": "model-00004.safetensors", "mtp_block.attn.v.weight": "model-00004.safetensors", "mtp_block.attn.o.weight": "model-00004.safetensors", "mtp_block.attn.qn.scale": "model-00004.safetensors", "mtp_block.attn.kn.scale": "model-00004.safetensors", "mtp_block.ffn.gate.weight": "model-00004.safetensors", "mtp_block.ffn.up.weight": "model-00004.safetensors", "mtp_block.ffn.down.weight": "model-00004.safetensors"}}