Upload PACER merged Qwen coder models

README.md

@@ -0,0 +1,55 @@
+---
+library_name: transformers
+tags:
+- pacer
+- model-merging
+- merged-model
+- moe
+license: apache-2.0
+---
+
+# pacer-merge
+
+This model was created using **PACER (Permutation-Aligned Consensus Expert Routing)**.
+
+## Model Details
+
+**Merge Type:** PACER (Base-Free, Interference-Aware)
+
+**Source Models:**
+- `fluently/FluentlyQwen3-Coder-4B-0909`
+- `SamuelBang/AesCoder-4B`
+
+**Merge Configuration:**
+- Interference Threshold: `0.35`
+- Top-K Experts: `2`
+- Merged Layers: `0`
+- MoE Layers: `108`
+
+## How PACER Works
+
+PACER is a novel model merging framework that:
+1. **Aligns models geometrically** using Git Re-Basin
+2. **Computes a Consensus Barycenter** as a synthetic base
+3. **Analyzes interference** per layer
+4. **Merges low-interference layers** using DARE-TIES
+5. **Upcycles high-interference layers** to Mixture-of-Experts
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained("pacer-merge")
+tokenizer = AutoTokenizer.from_pretrained("pacer-merge")
+
+# Use the model
+inputs = tokenizer("Hello, world!", return_tensors="pt")
+outputs = model.generate(**inputs)
+```
+
+## Created With
+
+[PacerKit](https://github.com/yourusername/pacerkit) - PACER Model Merging Framework
+
+**Created:** 2025-12-09 21:46:52

config.json

@@ -0,0 +1,70 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 9728,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 40960,
+  "max_window_layers": 36,
+  "model_type": "qwen3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.3",
+  "unsloth_version": "2025.9.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "max_length": 40960,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.3"
+}

merge_config.json

@@ -0,0 +1,20 @@
+{
+  "project_name": "pacer-merge",
+  "models": [
+    "fluently/FluentlyQwen3-Coder-4B-0909",
+    "SamuelBang/AesCoder-4B"
+  ],
+  "pacer": {
+    "interference_threshold": 0.35,
+    "top_k_experts": 2
+  },
+  "summary": {
+    "total_layers": 108,
+    "merge_layers": 0,
+    "moe_layers": 108,
+    "avg_interference": 0.9777785492716012,
+    "max_interference": 0.9826418738812208,
+    "min_interference": 0.9674257636070251,
+    "threshold": 0.35
+  }
+}

merge_report.json

@@ -0,0 +1,888 @@
+{
+  "config": {
+    "project_name": "pacer-merge",
+    "models": [
+      "fluently/FluentlyQwen3-Coder-4B-0909",
+      "SamuelBang/AesCoder-4B"
+    ],
+    "interference_threshold": 0.35,
+    "top_k_experts": 2
+  },
+  "summary": {
+    "total_layers": 108,
+    "merge_layers": 0,
+    "moe_layers": 108,
+    "avg_interference": 0.9777785492716012,
+    "max_interference": 0.9826418738812208,
+    "min_interference": 0.9674257636070251,
+    "threshold": 0.35
+  },
+  "layer_decisions": {
+    "model.layers.0.mlp.gate_proj.weight": {
+      "score": 0.9758567679673433,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.0.mlp.up_proj.weight": {
+      "score": 0.9773118272423744,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.0.mlp.down_proj.weight": {
+      "score": 0.9756715409457684,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.1.mlp.gate_proj.weight": {
+      "score": 0.9674257636070251,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.1.mlp.up_proj.weight": {
+      "score": 0.9758508205413818,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.1.mlp.down_proj.weight": {
+      "score": 0.9768250994384289,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.2.mlp.gate_proj.weight": {
+      "score": 0.973158223554492,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.2.mlp.up_proj.weight": {
+      "score": 0.9724544435739517,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.2.mlp.down_proj.weight": {
+      "score": 0.975229199975729,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.3.mlp.gate_proj.weight": {
+      "score": 0.9705729000270367,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.3.mlp.up_proj.weight": {
+      "score": 0.9761323314160109,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.3.mlp.down_proj.weight": {
+      "score": 0.9766243025660515,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.4.mlp.gate_proj.weight": {
+      "score": 0.9693492259830236,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.4.mlp.up_proj.weight": {
+      "score": 0.9771679863333702,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.4.mlp.down_proj.weight": {
+      "score": 0.9777354244142771,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.5.mlp.gate_proj.weight": {
+      "score": 0.9719736948609352,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.5.mlp.up_proj.weight": {
+      "score": 0.9768269564956427,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.5.mlp.down_proj.weight": {
+      "score": 0.9775361530482769,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.6.mlp.gate_proj.weight": {
+      "score": 0.9740530084818602,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.6.mlp.up_proj.weight": {
+      "score": 0.9770794659852982,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.6.mlp.down_proj.weight": {
+      "score": 0.9774630032479763,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.7.mlp.gate_proj.weight": {
+      "score": 0.9748863540589809,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.7.mlp.up_proj.weight": {
+      "score": 0.9771392066031694,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.7.mlp.down_proj.weight": {
+      "score": 0.9776077885180712,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.8.mlp.gate_proj.weight": {
+      "score": 0.9771762136369944,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.8.mlp.up_proj.weight": {
+      "score": 0.9769909419119358,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.8.mlp.down_proj.weight": {
+      "score": 0.9772236570715904,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.9.mlp.gate_proj.weight": {
+      "score": 0.9760845918208361,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.9.mlp.up_proj.weight": {
+      "score": 0.9766342304646969,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.9.mlp.down_proj.weight": {
+      "score": 0.9773884564638138,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.10.mlp.gate_proj.weight": {
+      "score": 0.977036003023386,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.10.mlp.up_proj.weight": {
+      "score": 0.9762268159538507,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.10.mlp.down_proj.weight": {
+      "score": 0.9767854642122984,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.11.mlp.gate_proj.weight": {
+      "score": 0.9778869729489088,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.11.mlp.up_proj.weight": {
+      "score": 0.9764058664441109,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.11.mlp.down_proj.weight": {
+      "score": 0.9770964700728655,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.12.mlp.gate_proj.weight": {
+      "score": 0.9788605384528637,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.12.mlp.up_proj.weight": {
+      "score": 0.9766901917755604,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.12.mlp.down_proj.weight": {
+      "score": 0.9772241190075874,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.13.mlp.gate_proj.weight": {
+      "score": 0.9798912685364485,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.13.mlp.up_proj.weight": {
+      "score": 0.9771493915468454,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.13.mlp.down_proj.weight": {
+      "score": 0.9775336850434542,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.14.mlp.gate_proj.weight": {
+      "score": 0.9807573985308409,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.14.mlp.up_proj.weight": {
+      "score": 0.9772995170205832,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.14.mlp.down_proj.weight": {
+      "score": 0.9775482192635536,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.15.mlp.gate_proj.weight": {
+      "score": 0.9814920481294394,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.15.mlp.up_proj.weight": {
+      "score": 0.978249479085207,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.15.mlp.down_proj.weight": {
+      "score": 0.9782819077372551,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.16.mlp.gate_proj.weight": {
+      "score": 0.9815784655511379,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.16.mlp.up_proj.weight": {
+      "score": 0.9783014710992575,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.16.mlp.down_proj.weight": {
+      "score": 0.9782586451619864,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.17.mlp.gate_proj.weight": {
+      "score": 0.9818551931530237,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.17.mlp.up_proj.weight": {
+      "score": 0.978881973773241,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.17.mlp.down_proj.weight": {
+      "score": 0.9785361550748348,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.18.mlp.gate_proj.weight": {
+      "score": 0.9821842219680548,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.18.mlp.up_proj.weight": {
+      "score": 0.9791757967323065,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.18.mlp.down_proj.weight": {
+      "score": 0.9788669720292091,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.19.mlp.gate_proj.weight": {
+      "score": 0.9823547545820475,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.19.mlp.up_proj.weight": {
+      "score": 0.9793595056980848,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.19.mlp.down_proj.weight": {
+      "score": 0.9791094567626715,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.20.mlp.gate_proj.weight": {
+      "score": 0.9826161675155163,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.20.mlp.up_proj.weight": {
+      "score": 0.9792643412947655,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.20.mlp.down_proj.weight": {
+      "score": 0.9788958225399256,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.21.mlp.gate_proj.weight": {
+      "score": 0.9826418738812208,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.21.mlp.up_proj.weight": {
+      "score": 0.9789138380438089,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.21.mlp.down_proj.weight": {
+      "score": 0.9783939477056265,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.22.mlp.gate_proj.weight": {
+      "score": 0.9819891918450594,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.22.mlp.up_proj.weight": {
+      "score": 0.9783588405698538,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.22.mlp.down_proj.weight": {
+      "score": 0.9774981644004583,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.23.mlp.gate_proj.weight": {
+      "score": 0.9811321999877691,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.23.mlp.up_proj.weight": {
+      "score": 0.9782727509737015,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.23.mlp.down_proj.weight": {
+      "score": 0.9773894734680653,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.24.mlp.gate_proj.weight": {
+      "score": 0.9806375782936811,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.24.mlp.up_proj.weight": {
+      "score": 0.9782112184911966,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.24.mlp.down_proj.weight": {
+      "score": 0.9772476609796286,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.25.mlp.gate_proj.weight": {
+      "score": 0.9800966791808605,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.25.mlp.up_proj.weight": {
+      "score": 0.9781745038926601,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.25.mlp.down_proj.weight": {
+      "score": 0.9775232560932636,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.26.mlp.gate_proj.weight": {
+      "score": 0.9799431785941124,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.26.mlp.up_proj.weight": {
+      "score": 0.9781098868697882,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.26.mlp.down_proj.weight": {
+      "score": 0.9775767754763365,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.27.mlp.gate_proj.weight": {
+      "score": 0.9798190761357546,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.27.mlp.up_proj.weight": {
+      "score": 0.9779539816081524,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.27.mlp.down_proj.weight": {
+      "score": 0.9775818083435297,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.28.mlp.gate_proj.weight": {
+      "score": 0.9798130877315998,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.28.mlp.up_proj.weight": {
+      "score": 0.9780505858361721,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.28.mlp.down_proj.weight": {
+      "score": 0.9777195602655411,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.29.mlp.gate_proj.weight": {
+      "score": 0.97960414737463,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.29.mlp.up_proj.weight": {
+      "score": 0.9777578953653574,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.29.mlp.down_proj.weight": {
+      "score": 0.9774074014276266,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.30.mlp.gate_proj.weight": {
+      "score": 0.979683106765151,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.30.mlp.up_proj.weight": {
+      "score": 0.9778437279164791,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.30.mlp.down_proj.weight": {
+      "score": 0.9775861166417599,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.31.mlp.gate_proj.weight": {
+      "score": 0.9799774046987295,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.31.mlp.up_proj.weight": {
+      "score": 0.9778894502669573,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.31.mlp.down_proj.weight": {
+      "score": 0.9775977656245232,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.32.mlp.gate_proj.weight": {
+      "score": 0.9808168206363916,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.32.mlp.up_proj.weight": {
+      "score": 0.9781559966504574,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.32.mlp.down_proj.weight": {
+      "score": 0.9776150044053793,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.33.mlp.gate_proj.weight": {
+      "score": 0.9813535250723362,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.33.mlp.up_proj.weight": {
+      "score": 0.9779957178980112,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.33.mlp.down_proj.weight": {
+      "score": 0.9773503355681896,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.34.mlp.gate_proj.weight": {
+      "score": 0.9805151708424091,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.34.mlp.up_proj.weight": {
+      "score": 0.9770685620605946,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.34.mlp.down_proj.weight": {
+      "score": 0.9765571355819702,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.35.mlp.gate_proj.weight": {
+      "score": 0.9779612477868795,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.35.mlp.up_proj.weight": {
+      "score": 0.9755715448409319,
+      "decision": "upcycle_moe"
+    },
+    "model.layers.35.mlp.down_proj.weight": {
+      "score": 0.9756422452628613,
+      "decision": "upcycle_moe"
+    }
+  },
+  "moe_layers": {
+    "model.layers.0.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9758567679673433
+    },
+    "model.layers.0.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9773118272423744
+    },
+    "model.layers.0.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9756715409457684
+    },
+    "model.layers.1.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9674257636070251
+    },
+    "model.layers.1.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9758508205413818
+    },
+    "model.layers.1.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9768250994384289
+    },
+    "model.layers.2.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.973158223554492
+    },
+    "model.layers.2.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9724544435739517
+    },
+    "model.layers.2.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.975229199975729
+    },
+    "model.layers.3.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9705729000270367
+    },
+    "model.layers.3.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9761323314160109
+    },
+    "model.layers.3.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9766243025660515
+    },
+    "model.layers.4.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9693492259830236
+    },
+    "model.layers.4.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9771679863333702
+    },
+    "model.layers.4.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9777354244142771
+    },
+    "model.layers.5.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9719736948609352
+    },
+    "model.layers.5.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9768269564956427
+    },
+    "model.layers.5.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9775361530482769
+    },
+    "model.layers.6.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9740530084818602
+    },
+    "model.layers.6.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9770794659852982
+    },
+    "model.layers.6.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9774630032479763
+    },
+    "model.layers.7.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9748863540589809
+    },
+    "model.layers.7.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9771392066031694
+    },
+    "model.layers.7.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9776077885180712
+    },
+    "model.layers.8.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9771762136369944
+    },
+    "model.layers.8.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9769909419119358
+    },
+    "model.layers.8.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9772236570715904
+    },
+    "model.layers.9.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9760845918208361
+    },
+    "model.layers.9.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9766342304646969
+    },
+    "model.layers.9.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9773884564638138
+    },
+    "model.layers.10.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.977036003023386
+    },
+    "model.layers.10.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9762268159538507
+    },
+    "model.layers.10.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9767854642122984
+    },
+    "model.layers.11.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9778869729489088
+    },
+    "model.layers.11.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9764058664441109
+    },
+    "model.layers.11.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9770964700728655
+    },
+    "model.layers.12.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9788605384528637
+    },
+    "model.layers.12.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9766901917755604
+    },
+    "model.layers.12.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9772241190075874
+    },
+    "model.layers.13.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9798912685364485
+    },
+    "model.layers.13.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9771493915468454
+    },
+    "model.layers.13.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9775336850434542
+    },
+    "model.layers.14.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9807573985308409
+    },
+    "model.layers.14.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9772995170205832
+    },
+    "model.layers.14.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9775482192635536
+    },
+    "model.layers.15.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9814920481294394
+    },
+    "model.layers.15.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.978249479085207
+    },
+    "model.layers.15.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9782819077372551
+    },
+    "model.layers.16.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9815784655511379
+    },
+    "model.layers.16.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9783014710992575
+    },
+    "model.layers.16.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9782586451619864
+    },
+    "model.layers.17.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9818551931530237
+    },
+    "model.layers.17.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.978881973773241
+    },
+    "model.layers.17.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9785361550748348
+    },
+    "model.layers.18.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9821842219680548
+    },
+    "model.layers.18.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9791757967323065
+    },
+    "model.layers.18.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9788669720292091
+    },
+    "model.layers.19.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9823547545820475
+    },
+    "model.layers.19.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9793595056980848
+    },
+    "model.layers.19.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9791094567626715
+    },
+    "model.layers.20.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9826161675155163
+    },
+    "model.layers.20.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9792643412947655
+    },
+    "model.layers.20.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9788958225399256
+    },
+    "model.layers.21.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9826418738812208
+    },
+    "model.layers.21.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9789138380438089
+    },
+    "model.layers.21.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9783939477056265
+    },
+    "model.layers.22.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9819891918450594
+    },
+    "model.layers.22.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9783588405698538
+    },
+    "model.layers.22.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9774981644004583
+    },
+    "model.layers.23.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9811321999877691
+    },
+    "model.layers.23.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9782727509737015
+    },
+    "model.layers.23.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9773894734680653
+    },
+    "model.layers.24.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9806375782936811
+    },
+    "model.layers.24.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9782112184911966
+    },
+    "model.layers.24.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9772476609796286
+    },
+    "model.layers.25.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9800966791808605
+    },
+    "model.layers.25.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9781745038926601
+    },
+    "model.layers.25.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9775232560932636
+    },
+    "model.layers.26.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9799431785941124
+    },
+    "model.layers.26.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9781098868697882
+    },
+    "model.layers.26.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9775767754763365
+    },
+    "model.layers.27.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9798190761357546
+    },
+    "model.layers.27.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9779539816081524
+    },
+    "model.layers.27.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9775818083435297
+    },
+    "model.layers.28.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9798130877315998
+    },
+    "model.layers.28.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9780505858361721
+    },
+    "model.layers.28.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9777195602655411
+    },
+    "model.layers.29.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.97960414737463
+    },
+    "model.layers.29.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9777578953653574
+    },
+    "model.layers.29.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9774074014276266
+    },
+    "model.layers.30.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.979683106765151
+    },
+    "model.layers.30.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9778437279164791
+    },
+    "model.layers.30.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9775861166417599
+    },
+    "model.layers.31.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9799774046987295
+    },
+    "model.layers.31.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9778894502669573
+    },
+    "model.layers.31.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9775977656245232
+    },
+    "model.layers.32.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9808168206363916
+    },
+    "model.layers.32.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9781559966504574
+    },
+    "model.layers.32.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9776150044053793
+    },
+    "model.layers.33.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9813535250723362
+    },
+    "model.layers.33.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9779957178980112
+    },
+    "model.layers.33.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9773503355681896
+    },
+    "model.layers.34.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9805151708424091
+    },
+    "model.layers.34.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9770685620605946
+    },
+    "model.layers.34.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9765571355819702
+    },
+    "model.layers.35.mlp.gate_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9779612477868795
+    },
+    "model.layers.35.mlp.up_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9755715448409319
+    },
+    "model.layers.35.mlp.down_proj.weight": {
+      "num_experts": 2,
+      "score": 0.9756422452628613
+    }
+  }
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b89c47f8598b12a217beabdb6a895b03e0583934e605ae233e230d838c15cf75
+size 4967215360

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88cde30d6f41c57b2d874f48465956b7b75944efacc91174f6f1316b21c85951
+size 3077766632

model.safetensors.index.json

@@ -0,0 +1,406 @@
+{
+  "metadata": {
+    "total_parameters": 4022468096,
+    "total_size": 8044936192
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}