Upload checkpoint step 147444

config.json

@@ -0,0 +1,61 @@
+{
+  "sequence_len": 8192,
+  "vocab_size": 131072,
+  "n_layer": 16,
+  "n_head": 8,
+  "n_kv_head": 8,
+  "n_embd": 1024,
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "moe_layer_interval": 3,
+  "moe_group_size": 4,
+  "moe_expert_intermediate_size": 1792,
+  "moe_adjugate_intermediate_size": 0,
+  "moe_adjugate_scale": 0.05,
+  "moe_router_aux_loss_coef": 0.015,
+  "moe_router_bias_lr": 0.001,
+  "moe_activation_checkpoint": true,
+  "moe_capacity_factor": 0.75,
+  "rotary_base": 10000.0,
+  "rotary_scaling_type": "yarn",
+  "rotary_scale_factor": 4.0,
+  "residual_scale": -1.0,
+  "attn_dropout": 0.01,
+  "label_smoothing": 0.0,
+  "z_loss_weight": 0.0,
+  "use_flash_attention": true,
+  "structured_head_names": null,
+  "structured_loss_weight": 0.0,
+  "structured_head_dims": null,
+  "structured_label_vocab": null,
+  "use_value_head": false,
+  "domain_router_dim": 32,
+  "num_domain_tags": 128,
+  "domain_router_features": {
+    "dataset": {
+      "capacity": 128,
+      "mode": "one_hot"
+    },
+    "quality": {
+      "capacity": 32,
+      "mode": "one_hot"
+    },
+    "specialty": {
+      "capacity": 64,
+      "mode": "one_hot"
+    },
+    "modality": {
+      "capacity": 32,
+      "mode": "one_hot"
+    },
+    "language": {
+      "capacity": 32,
+      "mode": "one_hot"
+    },
+    "origin": {
+      "capacity": 8,
+      "mode": "one_hot"
+    }
+  },
+  "router_context_dim": 302
+}

meta.json

@@ -0,0 +1,73 @@
+{
+  "step": 147444,
+  "val_loss": 1.0088086128234863,
+  "mmlu_acc": 0.3369140625,
+  "mmlu_pro_acc": 0.1455078125,
+  "arc_easy_acc": 0.4833984375,
+  "gsm8k_acc": 0.1171875,
+  "tool_ref_count": 246.0,
+  "tool_call_parse_rate": 1.0,
+  "tool_call_name_acc": 0.9349593495934959,
+  "tool_call_args_structured_rate": 1.0,
+  "model_config": {
+    "sequence_len": 8192,
+    "vocab_size": 131072,
+    "n_layer": 16,
+    "n_head": 8,
+    "n_kv_head": 8,
+    "n_embd": 1024,
+    "moe_num_experts": 8,
+    "moe_top_k": 2,
+    "moe_layer_interval": 3,
+    "moe_group_size": 4,
+    "moe_expert_intermediate_size": 1792,
+    "moe_adjugate_intermediate_size": 0,
+    "moe_adjugate_scale": 0.05,
+    "moe_router_aux_loss_coef": 0.015,
+    "moe_router_bias_lr": 0.001,
+    "moe_activation_checkpoint": true,
+    "moe_capacity_factor": 0.75,
+    "rotary_base": 10000.0,
+    "rotary_scaling_type": "yarn",
+    "rotary_scale_factor": 4.0,
+    "residual_scale": -1.0,
+    "attn_dropout": 0.01,
+    "label_smoothing": 0.0,
+    "z_loss_weight": 0.0,
+    "use_flash_attention": true,
+    "structured_head_names": null,
+    "structured_loss_weight": 0.0,
+    "structured_head_dims": null,
+    "structured_label_vocab": null,
+    "use_value_head": false,
+    "domain_router_dim": 32,
+    "num_domain_tags": 128,
+    "domain_router_features": {
+      "dataset": {
+        "capacity": 128,
+        "mode": "one_hot"
+      },
+      "quality": {
+        "capacity": 32,
+        "mode": "one_hot"
+      },
+      "specialty": {
+        "capacity": 64,
+        "mode": "one_hot"
+      },
+      "modality": {
+        "capacity": 32,
+        "mode": "one_hot"
+      },
+      "language": {
+        "capacity": 32,
+        "mode": "one_hot"
+      },
+      "origin": {
+        "capacity": 8,
+        "mode": "one_hot"
+      }
+    },
+    "router_context_dim": 302
+  }
+}

model-00001-of-00001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f3ee72cab079abe508234142b27a3e9e523f389e19e73aa49d780f78b781e34
+size 1845827576

model.safetensors.index.json

@@ -0,0 +1,228 @@
+{
+  "metadata": {
+    "total_size": 1845802880,
+    "num_shards": 1,
+    "step": 147444,
+    "format": "safetensors_state_dict"
+  },
+  "weight_map": {
+    "transformer.wte.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "lm_head.weight": "model-00001-of-00001.safetensors"
+  }
+}

token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:657d33e4365804c866b1c9e8c065c0f870428a0ece1f3b0771533dd6a0dc6076
+size 525865

tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d2e58be3e2ec81124dbf4941bbab6bd0715eca1a34e5a773953b60d6369e750
+size 1846082