Add files using upload-large-folder tool

README.md

@@ -0,0 +1,51 @@
+---
+library_name: mlx
+license: apache-2.0
+base_model: tencent/Hy3
+base_model_relation: quantized
+pipeline_tag: text-generation
+tags:
+  - mlx
+  - hunyuan
+  - hy3
+  - moe
+  - text-generation
+  - imatrix
+---
+
+# Hy3-oQ4e
+
+oMLX **oQ4e** (imatrix-enhanced affine 4-bit, group size 64, sensitivity-driven mixed
+placements) quantization of [tencent/Hy3](https://huggingface.co/tencent/Hy3)
+(HunYuan V3, 80-layer MoE: 192 routed experts + 1 shared, sigmoid router with expert bias).
+
+- **158 GB** (from the 598 GB BF16 source) across 32 safetensors shards.
+- **imatrix**: 128×512-token calibration (876 entries incl. per-expert statistics for all
+  192 experts), collected by running an 8-bit affine proxy of the model — the BF16 source
+  exceeds 512 GB unified memory, so calibration ran on the half-size proxy and the resulting
+  activation statistics were applied to the BF16→4-bit quantization.
+- **Sensitivity**: data-driven per-layer measurement (uniform 4-bit disk proxy); the most
+  sensitive projections (final layers 76–79, early layers 2–4) receive 5-bit boosts.
+- MTP (`num_nextn_predict_layers`) weights are stripped; this is a pure decoder checkpoint.
+- Coherency-verified at temperature 0 (arithmetic, factual recall, code generation,
+  trick-question reasoning, strict format following): 6/6.
+
+## Requirements
+
+`model_type: hy_v3` support is **not yet merged** into mlx-lm — it requires
+[mlx-lm PR #1211](https://github.com/ml-explore/mlx-lm/pull/1211) (adds
+`mlx_lm/models/hy_v3.py`, the `hy_v3` tool parsers, and thinking-tag inference).
+Apply the PR (or install from its branch) before loading:
+
+```python
+from mlx_lm import load, generate
+
+model, tokenizer = load("unigilby/Hy3-oQ4e")
+prompt = tokenizer.apply_chat_template(
+    [{"role": "user", "content": "Hello!"}], add_generation_prompt=True
+)
+print(generate(model, tokenizer, prompt=prompt, max_tokens=200))
+```
+
+Quantized with [oMLX](https://github.com/jundot/omlx) `quantize_oq_streaming`
+(enhanced/imatrix mode) on a Mac Studio M3 Ultra.

chat_template.jinja

@@ -0,0 +1,222 @@
+{#- ----------‑‑‑ special token variables ‑‑‑---------- -#}
+{%- set HYTK = ':opensource' %}
+{%- set eos_token = '<｜hy_eos{}｜>'.format(HYTK) %}
+{%- set bos_token = '<｜hy_begin_of_sentence{}｜>'.format(HYTK) %}
+{%- set pad_token = '<｜hy_pad{}｜>'.format(HYTK) %}
+{%- set user_token = '<｜hy_User{}｜>'.format(HYTK) %}
+{%- set assistant_token = '<｜hy_Assistant{}｜>'.format(HYTK) %}
+{%- set think_begin_token = '<think{}>'.format(HYTK) %}
+{%- set think_end_token = '</think{}>'.format(HYTK) %}
+{%- set toolcalls_begin_token = '<tool_calls{}>'.format(HYTK) %}
+{%- set toolcalls_end_token = '</tool_calls{}>'.format(HYTK) %}
+{%- set toolcall_begin_token = '<tool_call{}>'.format(HYTK) %}
+{%- set toolcall_end_token = '</tool_call{}>'.format(HYTK) %}
+{%- set toolsep_token = '<tool_sep{}>'.format(HYTK) %}
+{%- set argkey_begin_token = '<arg_key{}>'.format(HYTK) %}
+{%- set argkey_end_token = '</arg_key{}>'.format(HYTK) %}
+{%- set argvalue_begin_token = '<arg_value{}>'.format(HYTK) %}
+{%- set argvalue_end_token = '</arg_value{}>'.format(HYTK) %}
+{%- set toolresponses_begin_token = '<tool_responses{}>'.format(HYTK) %}
+{%- set toolresponses_end_token = '</tool_responses{}>'.format(HYTK) %}
+{%- set toolresponse_begin_token = '<tool_response{}>'.format(HYTK) %}
+{%- set toolresponse_end_token = '</tool_response{}>'.format(HYTK) %}
+{%- set reasoning_mode_token = '<｜reasoning_mode{}｜>'.format(HYTK) %}
+
+{#- ----------‑‑‑ hyperparameters variables ‑‑‑---------- -#}
+{%- if not add_generation_prompt is defined %}
+    {%- set add_generation_prompt = false %}
+{%- endif %}
+{%- if not preserved_thinking is defined %}
+    {%- if not tools %}
+        {%- set preserved_thinking = false %}
+    {%- else %}
+        {%- set preserved_thinking = true %}
+    {%- endif %}
+{%- endif %}
+{%- if not is_training is defined %}
+    {%- set is_training = false %}
+{%- endif %}
+
+{%- if not reasoning_effort is defined %}
+    {%- set reasoning_effort = 'no_think' %}
+{%- elif reasoning_effort not in ['high', 'low', 'no_think'] %}
+    {%- if reasoning_effort is none %}
+        {{- raise_exception('reasoning_effort error : None, should be no_think/low/high') }}
+    {%- else %}
+        {{- raise_exception('reasoning_effort error : ' + reasoning_effort + ', should be no_think/low/high') }}
+    {%- endif %}
+{%- endif %}
+
+{%- if fallback_strategy is defined and fallback_strategy == 'reasoning_toolcall_retry' %}
+    {%- set reasoning_effort = 'high' %}
+    {%- set add_generation_prompt = false %}
+{%- endif %}
+{%- if not raw_last_assistant is defined %}
+    {%- set raw_last_assistant = false %}
+{%- endif %}
+
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- elif content is none -%}
+        {{- '' }}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- set ns = namespace(last_user_index=-1) %}
+{%- set sp_ns = namespace(system_prompt='', is_first_sp=true) %}
+{%- for message in messages %}
+    {%- if message['role'] == 'system' %}
+        {%- set sp_ns.system_prompt = sp_ns.system_prompt + visible_text(message['content']) %}
+    {%- endif %}
+    {%- if message['role'] == 'user' %}
+        {%- set ns.last_user_index = loop.index0 %}
+    {%- endif %}
+{%- endfor %}
+{%- if reasoning_effort is defined and reasoning_effort is string and reasoning_effort != '' and not tools %}
+    {%- set sp_ns.system_prompt = sp_ns.system_prompt + reasoning_mode_token + 'reasoning_effort:' + reasoning_effort %}
+{%- endif %}
+{{- bos_token }}
+{{- sp_ns.system_prompt }}
+{%- if tools %}
+    {%- if sp_ns.system_prompt != '' %}
+        {{- '\n\n# Tools\n\nYou may call one or more functions to assist with the user query.' }}
+    {%- else %}
+        {{- '# Tools\n\nYou may call one or more functions to assist with the user query.' }}
+    {%- endif %}
+    {{- '\n\nYou are provided with function signatures within <tools></tools> XML tags:' }}
+    {{- '\n<tools>\n' }}
+    {%- for tool in tools %}
+        {%- if loop.index0 > 0 %}
+            {{- '\n' }}
+        {%- endif %}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- '\n</tools>\n\n' }}
+    {{- 'For function call returns, you should first print ' + toolcalls_begin_token + '\n' }}
+    {{- 'For each function call, you should return object like:\n' }}
+    {{- toolcall_begin_token + '{function-name}' + toolsep_token + '\n' }}
+    {{- argkey_begin_token + '{arg-key-1}' + argkey_end_token + '\n' }}
+    {{- argvalue_begin_token + '{arg-value-1}' + argvalue_end_token + '\n' }}
+    {{- argkey_begin_token + '{arg-key-2}' + argkey_end_token + '\n' }}
+    {{- argvalue_begin_token + '{arg-value-2}' + argvalue_end_token + '\n' }}
+    {{- '...\n' }}
+    {{- toolcall_end_token + '\n' }}
+    {%- if reasoning_effort is defined and reasoning_effort is string and reasoning_effort != '' %}
+        {{- 'At the end of function call returns, you should print ' + toolcalls_end_token + reasoning_mode_token + 'reasoning_effort:' + reasoning_effort }}
+    {%- else %}
+        {{- 'At the end of function call returns, you should print ' + toolcalls_end_token }}
+    {%- endif %}
+{%- endif %}
+
+{%- set prev_ns = namespace(is_tool=false, is_tool_first=true) %}
+{%- set last_ns = namespace(last_is_assistant=false) %}
+{%- for message in messages %}
+    {%- if message['role'] == 'user' %}
+        {%- if prev_ns.is_tool %}
+            {{- toolresponses_end_token }}
+        {%- endif %}
+        {{- user_token + visible_text(message['content']) }}
+        {%- set prev_ns.is_tool = false %}
+    {%- endif %}
+    {%- if message['role'] == 'assistant' %}
+        {%- if is_training %}
+            {%- if 'reasoning_content' in message and message['reasoning_content'] is string %}
+                {%- set rc = message['reasoning_content'] %}
+            {%- elif 'reasoning' in message and message['reasoning'] is string %}
+                {%- set rc = message['reasoning'] %}
+            {%- else %}
+                {%- set rc = none %}
+            {%- endif %}
+            {%- if rc is not none %}
+                {%- set content = think_begin_token + rc + think_end_token + visible_text(message['content']) %}
+            {%- else %}
+                {%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
+            {%- endif %}
+        {%- else %}
+            {%- if ((preserved_thinking is defined and preserved_thinking) or loop.index0 > ns.last_user_index) %}
+                {%- if 'reasoning_content' in message and message['reasoning_content'] is string %}
+                    {%- set rc = message['reasoning_content'] %}
+                {%- elif 'reasoning' in message and message['reasoning'] is string %}
+                    {%- set rc = message['reasoning'] %}
+                {%- else %}
+                    {%- set rc = none %}
+                {%- endif %}
+                {%- if rc is not none %}
+                    {%- set content = think_begin_token + rc + think_end_token + visible_text(message['content']) %}
+                {%- else %}
+                    {%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
+                {%- endif %}
+            {%- else %}
+                {%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
+            {%- endif %}
+        {%- endif %}
+        {%- if prev_ns.is_tool %}
+            {{- toolresponses_end_token }}
+        {%- endif %}
+        {{- assistant_token }}
+        {%- if message['tool_calls'] is defined and message['tool_calls'] %}
+            {%- set prev_ns.is_tool_first = true %}
+            {{- content }}
+            {{- toolcalls_begin_token + '\n' }}
+            {%- for tool in message['tool_calls'] %}
+                {%- set arguments = tool['function']['arguments'] %}
+                {{- toolcall_begin_token + tool['function']['name'] + toolsep_token + '\n' }}
+                {%- for key, value in arguments.items() %}
+                    {{- argkey_begin_token + key + argkey_end_token + '\n' }}
+                    {%- if value is not string %}
+                        {%- set value = value | tojson(ensure_ascii=False) %}
+                    {%- endif %}
+                    {{- argvalue_begin_token + value + argvalue_end_token + '\n' }}
+                {%- endfor %}
+                {{- toolcall_end_token + '\n' }}
+            {%- endfor %}
+            {{- toolcalls_end_token + eos_token }}
+        {%- else %}
+            {%- if loop.last and raw_last_assistant %}
+                {{- visible_text(message['content']) }}
+            {%- elif not loop.last or is_training %}
+                {{- content + eos_token }}
+            {%- else %}
+                {{- content }}
+            {%- endif %}
+        {%- endif %}
+        {%- set prev_ns.is_tool = false %}
+    {%- endif %}
+    {%- if message['role'] == 'tool' %}
+        {%- set prev_ns.is_tool = true %}
+        {%- if prev_ns.is_tool_first %}
+            {{- toolresponses_begin_token + '\n' }}
+            {%- set prev_ns.is_tool_first = false %}
+        {%- endif %}
+        {{- toolresponse_begin_token + '\n' + visible_text(message['content']) + '\n' + toolresponse_end_token + '\n' }}
+    {%- endif %}
+    {%- if loop.last and message['role'] == 'assistant' %}
+        {%- set last_ns.last_is_assistant = true %}
+    {%- endif %}
+
+{%- endfor %}
+{%- if prev_ns.is_tool %}
+    {{- toolresponses_end_token }}
+{%- endif %}
+{%- if add_generation_prompt %}
+    {%- if not last_ns.last_is_assistant %}
+        {%- if reasoning_effort is defined and reasoning_effort in ['low', 'high'] %}
+            {{- assistant_token + think_begin_token }}
+        {%- elif reasoning_effort is defined and reasoning_effort == 'no_think' %}
+            {{- assistant_token + think_begin_token + think_end_token }}
+        {%- else %}
+            {{- assistant_token }}
+        {%- endif %}
+    {%- endif %}
+{%- endif %}

generation_config.json

@@ -0,0 +1,10 @@
+{
+    "bos_token_id": 120000,
+    "do_sample": true,
+    "eos_token_id": 120025,
+    "pad_token_id": 120002,
+    "temperature": 0.9,
+    "top_k": -1,
+    "top_p": 1,
+    "transformers_version": "5.6.0"
+}

model-00001-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2104cef5e9f8320def1d3618ed083fa6e936dc6eeb27a0a6c00dee330836ded5
+size 5029416223

model-00002-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45d8cc10bd8eb69c39784e3efc7c5760105ca23b58aa3dabd13e787e85fb88f4
+size 5078498037

model-00003-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dee2f66bb1c4194f35ac771673a294c453c38814070a9ea04c6ae75cde433750
+size 5435823661

model-00004-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c609ff6c549f846b5573ad6ccafca89e18393c2aacd7f981cc781abac5c1e3
+size 5435822792

model-00005-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf6aacfcf0fb9480801a221886337bf24f60a9026cfc1e9fab32a3319bc5e12
+size 5435823665

model-00006-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6e7538bd6e6d8bef4e7b8d2dc0901d7bbcd110390b9182df6ed2cb260cd02d5
+size 5435823679

model-00007-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edbe950e4abec41dc2a46109dbc1d7093f69f6a16746455bec76bed9fcd8798f
+size 5435822820

model-00008-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6350a8434b1f6823d3b31e15677fbda893af5f6524794c2af6e5f9f02e28500c
+size 5435823694

model-00009-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6167dd1fd263df281e6f5dd5a369a3107d6aed789ccdf6cad03ba84a909f8fff
+size 5435823688

model-00010-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d98fc38e2453e377bc143f5c8e7c5ab003751e57f60b0837012befb70433740c
+size 5435822816

model-00011-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09bcc4a6ab4a3f4a9b6c97630dcf6a29503f27230116a62eab0097d0de89843a
+size 5435823692

model-00012-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a00ac15e4feac148c7add1f9c83980bdcd1c5a744901c74396892102488ac031
+size 5435823686

model-00013-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f137230d486f1dccf398806664672e4b7f24cd680a6aae01c6049df316416437
+size 5435822806

model-00014-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c146d3629fed211e46672c6763160ca4f00f8fa3b352babfcf6bb2f9df08f2a
+size 5435823692

model-00015-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:163ca55e42e5ecd74a44f03797cc598a1ccac768fec1e5b39f40d756eb9f0598
+size 5435823686

model-00016-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce6b9e1d56e5f1780868c59c40a2204865de774b7409f7db72ee61fba2c0a0e
+size 5435822818

model-00017-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bc406cc47a7df33b295c1c9688a1e21ecda9efeac852293486c7997d2d92d26
+size 5435823692

model-00018-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36517cf8709e787747eb652ad164c2d308a1f4e5eecb80b5d9ecffb4942b2f1b
+size 5435823680

model-00019-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:150f4bbb87694398e9dd0efc62cf6da0c2bd785987e43aab4b6f45ea3b4f9937
+size 5435822814

model-00020-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9cfc8cbe9db5e5871e467a5e63d163cbf264bc4b873de3d4a615e16939d2c1e
+size 5435823698

model-00021-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4e9aa43c6c6071de45fb21501b075994d0106edffb96e5986d699562d38ef14
+size 5435823684

model-00022-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e40970dac9ab62542e2e4887de9371a2392ea7c797cacaa530791c019d6397c
+size 5435822812

model-00023-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed1970e6eaf6cbd49a8246ef2559b40f0abe0eabd1f6e3cee0dd92ef52a16c5c
+size 5435823700

model-00024-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f59dc4626b6422afcc8f8d0cadb0a6dc66255ac6276aec9b6e168f55f4ad5800
+size 5435823690

model-00025-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:569196b36d54d4f782f7a4bb8808be786951719350c729f184c462427f87bc56
+size 5435822814

model-00026-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c41bb9e2278e398cc42b0b16ff09f85621338afab4abe6d4e86f844fb4bb37f
+size 5435823700

model-00027-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:178d808eeb3b351d0f85cc591ea85a05692fb6343d85b7603b1ae608a14d9c68
+size 5435823682

model-00028-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9458f14561e4612292b8fe374f3ddee1ece258a166a940896c98e463a9cd9d0
+size 5435822800

model-00029-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72c8ae0d51282cad3333d713c466e68aef73f37e462f90ca4f3b7cb4e705016e
+size 5435823682

model-00030-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0b7402c31cd0a67bd77ecd5181b77ded43ed8c19c084f76e8088a1f542fcb65
+size 5435823678

model-00031-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad002baef82d869e566ed6ae022ff5f0fb58c80a65bb1084df4e19bc0efc2e4c
+size 5435822818

model-00032-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4055448acb9e869420e0aa29dafa3d372bb64aafa303f14a50513c553001ccf2
+size 2038433782

oq_imatrix_report.json

@@ -0,0 +1,888 @@
+{
+  "enabled": true,
+  "cache_path": "/Users/earl/models/tencent/.oqe_imatrix/Hy3-oQe-s128-l512.npz",
+  "cache_reused": true,
+  "entry_count": 876,
+  "calib_dataset": "oqe_code_multilingual",
+  "collection": {
+    "dataset": "oqe_code_multilingual",
+    "requested_samples": 128,
+    "seq_length": 512,
+    "adaptive": true,
+    "adaptive_step_samples": 128,
+    "adaptive_max_samples": 1024,
+    "available_samples": 1024,
+    "micro_batch_size": 12,
+    "micro_batches": 11,
+    "batch_plan": {
+      "micro_batch_size": 12,
+      "estimated_sample_bytes": 67108864,
+      "capture_budget_bytes": 805306368,
+      "system_available_bytes": 539079163904,
+      "metal_available_bytes": 515391651504,
+      "live_available_bytes": 515391651504,
+      "hidden_size": 4096,
+      "num_experts": 192,
+      "top_k": 8
+    },
+    "processed_samples": 128,
+    "installed_modules": 877,
+    "coverage_sufficient": true,
+    "coverage": {
+      "has_expert_counts": true,
+      "expert_modules": 237,
+      "total_experts": 45504,
+      "active_experts": 45504,
+      "zero_count_experts": 0,
+      "active_ratio": 1.0,
+      "min_count": 25,
+      "p05_count": 1065.0,
+      "p10_count": 1327.0,
+      "median_count": 2462.0,
+      "max_count": 28519,
+      "min_required_count": 16,
+      "required_percentile": 5
+    },
+    "rounds": [
+      {
+        "processed_samples": 128,
+        "coverage_sufficient": true,
+        "coverage": {
+          "has_expert_counts": true,
+          "expert_modules": 237,
+          "total_experts": 45504,
+          "active_experts": 45504,
+          "zero_count_experts": 0,
+          "active_ratio": 1.0,
+          "min_count": 25,
+          "p05_count": 1065.0,
+          "p10_count": 1327.0,
+          "median_count": 2462.0,
+          "max_count": 28519,
+          "min_required_count": 16,
+          "required_percentile": 5
+        }
+      }
+    ]
+  },
+  "expert_coverage": {
+    "has_expert_counts": true,
+    "expert_modules": 237,
+    "total_experts": 45504,
+    "active_experts": 45504,
+    "zero_count_experts": 0,
+    "active_ratio": 1.0,
+    "min_count": 25,
+    "p05_count": 1065.0,
+    "p10_count": 1327.0,
+    "median_count": 2462.0,
+    "max_count": 28519,
+    "min_required_count": 16,
+    "required_percentile": 5
+  },
+  "applied": [
+    "model.layers.0.mlp.down_proj",
+    "model.layers.0.mlp.gate_proj",
+    "model.layers.0.mlp.up_proj",
+    "model.layers.0.self_attn.k_proj",
+    "model.layers.0.self_attn.o_proj",
+    "model.layers.0.self_attn.q_proj",
+    "model.layers.0.self_attn.v_proj",
+    "model.layers.1.mlp.shared_mlp.down_proj",
+    "model.layers.1.mlp.shared_mlp.gate_proj",
+    "model.layers.1.mlp.shared_mlp.up_proj",
+    "model.layers.1.mlp.switch_mlp.down_proj",
+    "model.layers.1.mlp.switch_mlp.gate_proj",
+    "model.layers.1.mlp.switch_mlp.up_proj",
+    "model.layers.1.self_attn.k_proj",
+    "model.layers.1.self_attn.o_proj",
+    "model.layers.1.self_attn.q_proj",
+    "model.layers.1.self_attn.v_proj",
+    "model.layers.10.mlp.shared_mlp.down_proj",
+    "model.layers.10.mlp.shared_mlp.gate_proj",
+    "model.layers.10.mlp.shared_mlp.up_proj",
+    "model.layers.10.mlp.switch_mlp.down_proj",
+    "model.layers.10.mlp.switch_mlp.gate_proj",
+    "model.layers.10.mlp.switch_mlp.up_proj",
+    "model.layers.10.self_attn.k_proj",
+    "model.layers.10.self_attn.o_proj",
+    "model.layers.10.self_attn.q_proj",
+    "model.layers.10.self_attn.v_proj",
+    "model.layers.11.mlp.shared_mlp.down_proj",
+    "model.layers.11.mlp.shared_mlp.gate_proj",
+    "model.layers.11.mlp.shared_mlp.up_proj",
+    "model.layers.11.mlp.switch_mlp.down_proj",
+    "model.layers.11.mlp.switch_mlp.gate_proj",
+    "model.layers.11.mlp.switch_mlp.up_proj",
+    "model.layers.11.self_attn.k_proj",
+    "model.layers.11.self_attn.o_proj",
+    "model.layers.11.self_attn.q_proj",
+    "model.layers.11.self_attn.v_proj",
+    "model.layers.12.mlp.shared_mlp.down_proj",
+    "model.layers.12.mlp.shared_mlp.gate_proj",
+    "model.layers.12.mlp.shared_mlp.up_proj",
+    "model.layers.12.mlp.switch_mlp.down_proj",
+    "model.layers.12.mlp.switch_mlp.gate_proj",
+    "model.layers.12.mlp.switch_mlp.up_proj",
+    "model.layers.12.self_attn.k_proj",
+    "model.layers.12.self_attn.o_proj",
+    "model.layers.12.self_attn.q_proj",
+    "model.layers.12.self_attn.v_proj",
+    "model.layers.13.mlp.shared_mlp.down_proj",
+    "model.layers.13.mlp.shared_mlp.gate_proj",
+    "model.layers.13.mlp.shared_mlp.up_proj",
+    "model.layers.13.mlp.switch_mlp.down_proj",
+    "model.layers.13.mlp.switch_mlp.gate_proj",
+    "model.layers.13.mlp.switch_mlp.up_proj",
+    "model.layers.13.self_attn.k_proj",
+    "model.layers.13.self_attn.o_proj",
+    "model.layers.13.self_attn.q_proj",
+    "model.layers.13.self_attn.v_proj",
+    "model.layers.14.mlp.shared_mlp.down_proj",
+    "model.layers.14.mlp.shared_mlp.gate_proj",
+    "model.layers.14.mlp.shared_mlp.up_proj",
+    "model.layers.14.mlp.switch_mlp.down_proj",
+    "model.layers.14.mlp.switch_mlp.gate_proj",
+    "model.layers.14.mlp.switch_mlp.up_proj",
+    "model.layers.14.self_attn.k_proj",
+    "model.layers.14.self_attn.o_proj",
+    "model.layers.14.self_attn.q_proj",
+    "model.layers.14.self_attn.v_proj",
+    "model.layers.15.mlp.shared_mlp.down_proj",
+    "model.layers.15.mlp.shared_mlp.gate_proj",
+    "model.layers.15.mlp.shared_mlp.up_proj",
+    "model.layers.15.mlp.switch_mlp.down_proj",
+    "model.layers.15.mlp.switch_mlp.gate_proj",
+    "model.layers.15.mlp.switch_mlp.up_proj",
+    "model.layers.15.self_attn.k_proj",
+    "model.layers.15.self_attn.o_proj",
+    "model.layers.15.self_attn.q_proj",
+    "model.layers.15.self_attn.v_proj",
+    "model.layers.16.mlp.shared_mlp.down_proj",
+    "model.layers.16.mlp.shared_mlp.gate_proj",
+    "model.layers.16.mlp.shared_mlp.up_proj",
+    "model.layers.16.mlp.switch_mlp.down_proj",
+    "model.layers.16.mlp.switch_mlp.gate_proj",
+    "model.layers.16.mlp.switch_mlp.up_proj",
+    "model.layers.16.self_attn.k_proj",
+    "model.layers.16.self_attn.o_proj",
+    "model.layers.16.self_attn.q_proj",
+    "model.layers.16.self_attn.v_proj",
+    "model.layers.17.mlp.shared_mlp.down_proj",
+    "model.layers.17.mlp.shared_mlp.gate_proj",
+    "model.layers.17.mlp.shared_mlp.up_proj",
+    "model.layers.17.mlp.switch_mlp.down_proj",
+    "model.layers.17.mlp.switch_mlp.gate_proj",
+    "model.layers.17.mlp.switch_mlp.up_proj",
+    "model.layers.17.self_attn.k_proj",
+    "model.layers.17.self_attn.o_proj",
+    "model.layers.17.self_attn.q_proj",
+    "model.layers.17.self_attn.v_proj",
+    "model.layers.18.mlp.shared_mlp.down_proj",
+    "model.layers.18.mlp.shared_mlp.gate_proj",
+    "model.layers.18.mlp.shared_mlp.up_proj",
+    "model.layers.18.mlp.switch_mlp.down_proj",
+    "model.layers.18.mlp.switch_mlp.gate_proj",
+    "model.layers.18.mlp.switch_mlp.up_proj",
+    "model.layers.18.self_attn.k_proj",
+    "model.layers.18.self_attn.o_proj",
+    "model.layers.18.self_attn.q_proj",
+    "model.layers.18.self_attn.v_proj",
+    "model.layers.19.mlp.shared_mlp.down_proj",
+    "model.layers.19.mlp.shared_mlp.gate_proj",
+    "model.layers.19.mlp.shared_mlp.up_proj",
+    "model.layers.19.mlp.switch_mlp.down_proj",
+    "model.layers.19.mlp.switch_mlp.gate_proj",
+    "model.layers.19.mlp.switch_mlp.up_proj",
+    "model.layers.19.self_attn.k_proj",
+    "model.layers.19.self_attn.o_proj",
+    "model.layers.19.self_attn.q_proj",
+    "model.layers.19.self_attn.v_proj",
+    "model.layers.2.mlp.shared_mlp.down_proj",
+    "model.layers.2.mlp.shared_mlp.gate_proj",
+    "model.layers.2.mlp.shared_mlp.up_proj",
+    "model.layers.2.mlp.switch_mlp.down_proj",
+    "model.layers.2.mlp.switch_mlp.gate_proj",
+    "model.layers.2.mlp.switch_mlp.up_proj",
+    "model.layers.2.self_attn.k_proj",
+    "model.layers.2.self_attn.o_proj",
+    "model.layers.2.self_attn.q_proj",
+    "model.layers.2.self_attn.v_proj",
+    "model.layers.20.mlp.shared_mlp.down_proj",
+    "model.layers.20.mlp.shared_mlp.gate_proj",
+    "model.layers.20.mlp.shared_mlp.up_proj",
+    "model.layers.20.mlp.switch_mlp.down_proj",
+    "model.layers.20.mlp.switch_mlp.gate_proj",
+    "model.layers.20.mlp.switch_mlp.up_proj",
+    "model.layers.20.self_attn.k_proj",
+    "model.layers.20.self_attn.o_proj",
+    "model.layers.20.self_attn.q_proj",
+    "model.layers.20.self_attn.v_proj",
+    "model.layers.21.mlp.shared_mlp.down_proj",
+    "model.layers.21.mlp.shared_mlp.gate_proj",
+    "model.layers.21.mlp.shared_mlp.up_proj",
+    "model.layers.21.mlp.switch_mlp.down_proj",
+    "model.layers.21.mlp.switch_mlp.gate_proj",
+    "model.layers.21.mlp.switch_mlp.up_proj",
+    "model.layers.21.self_attn.k_proj",
+    "model.layers.21.self_attn.o_proj",
+    "model.layers.21.self_attn.q_proj",
+    "model.layers.21.self_attn.v_proj",
+    "model.layers.22.mlp.shared_mlp.down_proj",
+    "model.layers.22.mlp.shared_mlp.gate_proj",
+    "model.layers.22.mlp.shared_mlp.up_proj",
+    "model.layers.22.mlp.switch_mlp.down_proj",
+    "model.layers.22.mlp.switch_mlp.gate_proj",
+    "model.layers.22.mlp.switch_mlp.up_proj",
+    "model.layers.22.self_attn.k_proj",
+    "model.layers.22.self_attn.o_proj",
+    "model.layers.22.self_attn.q_proj",
+    "model.layers.22.self_attn.v_proj",
+    "model.layers.23.mlp.shared_mlp.down_proj",
+    "model.layers.23.mlp.shared_mlp.gate_proj",
+    "model.layers.23.mlp.shared_mlp.up_proj",
+    "model.layers.23.mlp.switch_mlp.down_proj",
+    "model.layers.23.mlp.switch_mlp.gate_proj",
+    "model.layers.23.mlp.switch_mlp.up_proj",
+    "model.layers.23.self_attn.k_proj",
+    "model.layers.23.self_attn.o_proj",
+    "model.layers.23.self_attn.q_proj",
+    "model.layers.23.self_attn.v_proj",
+    "model.layers.24.mlp.shared_mlp.down_proj",
+    "model.layers.24.mlp.shared_mlp.gate_proj",
+    "model.layers.24.mlp.shared_mlp.up_proj",
+    "model.layers.24.mlp.switch_mlp.down_proj",
+    "model.layers.24.mlp.switch_mlp.gate_proj",
+    "model.layers.24.mlp.switch_mlp.up_proj",
+    "model.layers.24.self_attn.k_proj",
+    "model.layers.24.self_attn.o_proj",
+    "model.layers.24.self_attn.q_proj",
+    "model.layers.24.self_attn.v_proj",
+    "model.layers.25.mlp.shared_mlp.down_proj",
+    "model.layers.25.mlp.shared_mlp.gate_proj",
+    "model.layers.25.mlp.shared_mlp.up_proj",
+    "model.layers.25.mlp.switch_mlp.down_proj",
+    "model.layers.25.mlp.switch_mlp.gate_proj",
+    "model.layers.25.mlp.switch_mlp.up_proj",
+    "model.layers.25.self_attn.k_proj",
+    "model.layers.25.self_attn.o_proj",
+    "model.layers.25.self_attn.q_proj",
+    "model.layers.25.self_attn.v_proj",
+    "model.layers.26.mlp.shared_mlp.down_proj",
+    "model.layers.26.mlp.shared_mlp.gate_proj",
+    "model.layers.26.mlp.shared_mlp.up_proj",
+    "model.layers.26.mlp.switch_mlp.down_proj",
+    "model.layers.26.mlp.switch_mlp.gate_proj",
+    "model.layers.26.mlp.switch_mlp.up_proj",
+    "model.layers.26.self_attn.k_proj",
+    "model.layers.26.self_attn.o_proj",
+    "model.layers.26.self_attn.q_proj",
+    "model.layers.26.self_attn.v_proj",
+    "model.layers.27.mlp.shared_mlp.down_proj",
+    "model.layers.27.mlp.shared_mlp.gate_proj",
+    "model.layers.27.mlp.shared_mlp.up_proj",
+    "model.layers.27.mlp.switch_mlp.down_proj",
+    "model.layers.27.mlp.switch_mlp.gate_proj",
+    "model.layers.27.mlp.switch_mlp.up_proj",
+    "model.layers.27.self_attn.k_proj",
+    "model.layers.27.self_attn.o_proj",
+    "model.layers.27.self_attn.q_proj",
+    "model.layers.27.self_attn.v_proj",
+    "model.layers.28.mlp.shared_mlp.down_proj",
+    "model.layers.28.mlp.shared_mlp.gate_proj",
+    "model.layers.28.mlp.shared_mlp.up_proj",
+    "model.layers.28.mlp.switch_mlp.down_proj",
+    "model.layers.28.mlp.switch_mlp.gate_proj",
+    "model.layers.28.mlp.switch_mlp.up_proj",
+    "model.layers.28.self_attn.k_proj",
+    "model.layers.28.self_attn.o_proj",
+    "model.layers.28.self_attn.q_proj",
+    "model.layers.28.self_attn.v_proj",
+    "model.layers.29.mlp.shared_mlp.down_proj",
+    "model.layers.29.mlp.shared_mlp.gate_proj",
+    "model.layers.29.mlp.shared_mlp.up_proj",
+    "model.layers.29.mlp.switch_mlp.down_proj",
+    "model.layers.29.mlp.switch_mlp.gate_proj",
+    "model.layers.29.mlp.switch_mlp.up_proj",
+    "model.layers.29.self_attn.k_proj",
+    "model.layers.29.self_attn.o_proj",
+    "model.layers.29.self_attn.q_proj",
+    "model.layers.29.self_attn.v_proj",
+    "model.layers.3.mlp.shared_mlp.down_proj",
+    "model.layers.3.mlp.shared_mlp.gate_proj",
+    "model.layers.3.mlp.shared_mlp.up_proj",
+    "model.layers.3.mlp.switch_mlp.down_proj",
+    "model.layers.3.mlp.switch_mlp.gate_proj",
+    "model.layers.3.mlp.switch_mlp.up_proj",
+    "model.layers.3.self_attn.k_proj",
+    "model.layers.3.self_attn.o_proj",
+    "model.layers.3.self_attn.q_proj",
+    "model.layers.3.self_attn.v_proj",
+    "model.layers.30.mlp.shared_mlp.down_proj",
+    "model.layers.30.mlp.shared_mlp.gate_proj",
+    "model.layers.30.mlp.shared_mlp.up_proj",
+    "model.layers.30.mlp.switch_mlp.down_proj",
+    "model.layers.30.mlp.switch_mlp.gate_proj",
+    "model.layers.30.mlp.switch_mlp.up_proj",
+    "model.layers.30.self_attn.k_proj",
+    "model.layers.30.self_attn.o_proj",
+    "model.layers.30.self_attn.q_proj",
+    "model.layers.30.self_attn.v_proj",
+    "model.layers.31.mlp.shared_mlp.down_proj",
+    "model.layers.31.mlp.shared_mlp.gate_proj",
+    "model.layers.31.mlp.shared_mlp.up_proj",
+    "model.layers.31.mlp.switch_mlp.down_proj",
+    "model.layers.31.mlp.switch_mlp.gate_proj",
+    "model.layers.31.mlp.switch_mlp.up_proj",
+    "model.layers.31.self_attn.k_proj",
+    "model.layers.31.self_attn.o_proj",
+    "model.layers.31.self_attn.q_proj",
+    "model.layers.31.self_attn.v_proj",
+    "model.layers.32.mlp.shared_mlp.down_proj",
+    "model.layers.32.mlp.shared_mlp.gate_proj",
+    "model.layers.32.mlp.shared_mlp.up_proj",
+    "model.layers.32.mlp.switch_mlp.down_proj",
+    "model.layers.32.mlp.switch_mlp.gate_proj",
+    "model.layers.32.mlp.switch_mlp.up_proj",
+    "model.layers.32.self_attn.k_proj",
+    "model.layers.32.self_attn.o_proj",
+    "model.layers.32.self_attn.q_proj",
+    "model.layers.32.self_attn.v_proj",
+    "model.layers.33.mlp.shared_mlp.down_proj",
+    "model.layers.33.mlp.shared_mlp.gate_proj",
+    "model.layers.33.mlp.shared_mlp.up_proj",
+    "model.layers.33.mlp.switch_mlp.down_proj",
+    "model.layers.33.mlp.switch_mlp.gate_proj",
+    "model.layers.33.mlp.switch_mlp.up_proj",
+    "model.layers.33.self_attn.k_proj",
+    "model.layers.33.self_attn.o_proj",
+    "model.layers.33.self_attn.q_proj",
+    "model.layers.33.self_attn.v_proj",
+    "model.layers.34.mlp.shared_mlp.down_proj",
+    "model.layers.34.mlp.shared_mlp.gate_proj",
+    "model.layers.34.mlp.shared_mlp.up_proj",
+    "model.layers.34.mlp.switch_mlp.down_proj",
+    "model.layers.34.mlp.switch_mlp.gate_proj",
+    "model.layers.34.mlp.switch_mlp.up_proj",
+    "model.layers.34.self_attn.k_proj",
+    "model.layers.34.self_attn.o_proj",
+    "model.layers.34.self_attn.q_proj",
+    "model.layers.34.self_attn.v_proj",
+    "model.layers.35.mlp.shared_mlp.down_proj",
+    "model.layers.35.mlp.shared_mlp.gate_proj",
+    "model.layers.35.mlp.shared_mlp.up_proj",
+    "model.layers.35.mlp.switch_mlp.down_proj",
+    "model.layers.35.mlp.switch_mlp.gate_proj",
+    "model.layers.35.mlp.switch_mlp.up_proj",
+    "model.layers.35.self_attn.k_proj",
+    "model.layers.35.self_attn.o_proj",
+    "model.layers.35.self_attn.q_proj",
+    "model.layers.35.self_attn.v_proj",
+    "model.layers.36.mlp.shared_mlp.down_proj",
+    "model.layers.36.mlp.shared_mlp.gate_proj",
+    "model.layers.36.mlp.shared_mlp.up_proj",
+    "model.layers.36.mlp.switch_mlp.down_proj",
+    "model.layers.36.mlp.switch_mlp.gate_proj",
+    "model.layers.36.mlp.switch_mlp.up_proj",
+    "model.layers.36.self_attn.k_proj",
+    "model.layers.36.self_attn.o_proj",
+    "model.layers.36.self_attn.q_proj",
+    "model.layers.36.self_attn.v_proj",
+    "model.layers.37.mlp.shared_mlp.down_proj",
+    "model.layers.37.mlp.shared_mlp.gate_proj",
+    "model.layers.37.mlp.shared_mlp.up_proj",
+    "model.layers.37.mlp.switch_mlp.down_proj",
+    "model.layers.37.mlp.switch_mlp.gate_proj",
+    "model.layers.37.mlp.switch_mlp.up_proj",
+    "model.layers.37.self_attn.k_proj",
+    "model.layers.37.self_attn.o_proj",
+    "model.layers.37.self_attn.q_proj",
+    "model.layers.37.self_attn.v_proj",
+    "model.layers.38.mlp.shared_mlp.down_proj",
+    "model.layers.38.mlp.shared_mlp.gate_proj",
+    "model.layers.38.mlp.shared_mlp.up_proj",
+    "model.layers.38.mlp.switch_mlp.down_proj",
+    "model.layers.38.mlp.switch_mlp.gate_proj",
+    "model.layers.38.mlp.switch_mlp.up_proj",
+    "model.layers.38.self_attn.k_proj",
+    "model.layers.38.self_attn.o_proj",
+    "model.layers.38.self_attn.q_proj",
+    "model.layers.38.self_attn.v_proj",
+    "model.layers.39.mlp.shared_mlp.down_proj",
+    "model.layers.39.mlp.shared_mlp.gate_proj",
+    "model.layers.39.mlp.shared_mlp.up_proj",
+    "model.layers.39.mlp.switch_mlp.down_proj",
+    "model.layers.39.mlp.switch_mlp.gate_proj",
+    "model.layers.39.mlp.switch_mlp.up_proj",
+    "model.layers.39.self_attn.k_proj",
+    "model.layers.39.self_attn.o_proj",
+    "model.layers.39.self_attn.q_proj",
+    "model.layers.39.self_attn.v_proj",
+    "model.layers.4.mlp.shared_mlp.down_proj",
+    "model.layers.4.mlp.shared_mlp.gate_proj",
+    "model.layers.4.mlp.shared_mlp.up_proj",
+    "model.layers.4.mlp.switch_mlp.down_proj",
+    "model.layers.4.mlp.switch_mlp.gate_proj",
+    "model.layers.4.mlp.switch_mlp.up_proj",
+    "model.layers.4.self_attn.k_proj",
+    "model.layers.4.self_attn.o_proj",
+    "model.layers.4.self_attn.q_proj",
+    "model.layers.4.self_attn.v_proj",
+    "model.layers.40.mlp.shared_mlp.down_proj",
+    "model.layers.40.mlp.shared_mlp.gate_proj",
+    "model.layers.40.mlp.shared_mlp.up_proj",
+    "model.layers.40.mlp.switch_mlp.down_proj",
+    "model.layers.40.mlp.switch_mlp.gate_proj",
+    "model.layers.40.mlp.switch_mlp.up_proj",
+    "model.layers.40.self_attn.k_proj",
+    "model.layers.40.self_attn.o_proj",
+    "model.layers.40.self_attn.q_proj",
+    "model.layers.40.self_attn.v_proj",
+    "model.layers.41.mlp.shared_mlp.down_proj",
+    "model.layers.41.mlp.shared_mlp.gate_proj",
+    "model.layers.41.mlp.shared_mlp.up_proj",
+    "model.layers.41.mlp.switch_mlp.down_proj",
+    "model.layers.41.mlp.switch_mlp.gate_proj",
+    "model.layers.41.mlp.switch_mlp.up_proj",
+    "model.layers.41.self_attn.k_proj",
+    "model.layers.41.self_attn.o_proj",
+    "model.layers.41.self_attn.q_proj",
+    "model.layers.41.self_attn.v_proj",
+    "model.layers.42.mlp.shared_mlp.down_proj",
+    "model.layers.42.mlp.shared_mlp.gate_proj",
+    "model.layers.42.mlp.shared_mlp.up_proj",
+    "model.layers.42.mlp.switch_mlp.down_proj",
+    "model.layers.42.mlp.switch_mlp.gate_proj",
+    "model.layers.42.mlp.switch_mlp.up_proj",
+    "model.layers.42.self_attn.k_proj",
+    "model.layers.42.self_attn.o_proj",
+    "model.layers.42.self_attn.q_proj",
+    "model.layers.42.self_attn.v_proj",
+    "model.layers.43.mlp.shared_mlp.down_proj",
+    "model.layers.43.mlp.shared_mlp.gate_proj",
+    "model.layers.43.mlp.shared_mlp.up_proj",
+    "model.layers.43.mlp.switch_mlp.down_proj",
+    "model.layers.43.mlp.switch_mlp.gate_proj",
+    "model.layers.43.mlp.switch_mlp.up_proj",
+    "model.layers.43.self_attn.k_proj",
+    "model.layers.43.self_attn.o_proj",
+    "model.layers.43.self_attn.q_proj",
+    "model.layers.43.self_attn.v_proj",
+    "model.layers.44.mlp.shared_mlp.down_proj",
+    "model.layers.44.mlp.shared_mlp.gate_proj",
+    "model.layers.44.mlp.shared_mlp.up_proj",
+    "model.layers.44.mlp.switch_mlp.down_proj",
+    "model.layers.44.mlp.switch_mlp.gate_proj",
+    "model.layers.44.mlp.switch_mlp.up_proj",
+    "model.layers.44.self_attn.k_proj",
+    "model.layers.44.self_attn.o_proj",
+    "model.layers.44.self_attn.q_proj",
+    "model.layers.44.self_attn.v_proj",
+    "model.layers.45.mlp.shared_mlp.down_proj",
+    "model.layers.45.mlp.shared_mlp.gate_proj",
+    "model.layers.45.mlp.shared_mlp.up_proj",
+    "model.layers.45.mlp.switch_mlp.down_proj",
+    "model.layers.45.mlp.switch_mlp.gate_proj",
+    "model.layers.45.mlp.switch_mlp.up_proj",
+    "model.layers.45.self_attn.k_proj",
+    "model.layers.45.self_attn.o_proj",
+    "model.layers.45.self_attn.q_proj",
+    "model.layers.45.self_attn.v_proj",
+    "model.layers.46.mlp.shared_mlp.down_proj",
+    "model.layers.46.mlp.shared_mlp.gate_proj",
+    "model.layers.46.mlp.shared_mlp.up_proj",
+    "model.layers.46.mlp.switch_mlp.down_proj",
+    "model.layers.46.mlp.switch_mlp.gate_proj",
+    "model.layers.46.mlp.switch_mlp.up_proj",
+    "model.layers.46.self_attn.k_proj",
+    "model.layers.46.self_attn.o_proj",
+    "model.layers.46.self_attn.q_proj",
+    "model.layers.46.self_attn.v_proj",
+    "model.layers.47.mlp.shared_mlp.down_proj",
+    "model.layers.47.mlp.shared_mlp.gate_proj",
+    "model.layers.47.mlp.shared_mlp.up_proj",
+    "model.layers.47.mlp.switch_mlp.down_proj",
+    "model.layers.47.mlp.switch_mlp.gate_proj",
+    "model.layers.47.mlp.switch_mlp.up_proj",
+    "model.layers.47.self_attn.k_proj",
+    "model.layers.47.self_attn.o_proj",
+    "model.layers.47.self_attn.q_proj",
+    "model.layers.47.self_attn.v_proj",
+    "model.layers.48.mlp.shared_mlp.down_proj",
+    "model.layers.48.mlp.shared_mlp.gate_proj",
+    "model.layers.48.mlp.shared_mlp.up_proj",
+    "model.layers.48.mlp.switch_mlp.down_proj",
+    "model.layers.48.mlp.switch_mlp.gate_proj",
+    "model.layers.48.mlp.switch_mlp.up_proj",
+    "model.layers.48.self_attn.k_proj",
+    "model.layers.48.self_attn.o_proj",
+    "model.layers.48.self_attn.q_proj",
+    "model.layers.48.self_attn.v_proj",
+    "model.layers.49.mlp.shared_mlp.down_proj",
+    "model.layers.49.mlp.shared_mlp.gate_proj",
+    "model.layers.49.mlp.shared_mlp.up_proj",
+    "model.layers.49.mlp.switch_mlp.down_proj",
+    "model.layers.49.mlp.switch_mlp.gate_proj",
+    "model.layers.49.mlp.switch_mlp.up_proj",
+    "model.layers.49.self_attn.k_proj",
+    "model.layers.49.self_attn.o_proj",
+    "model.layers.49.self_attn.q_proj",
+    "model.layers.49.self_attn.v_proj",
+    "model.layers.5.mlp.shared_mlp.down_proj",
+    "model.layers.5.mlp.shared_mlp.gate_proj",
+    "model.layers.5.mlp.shared_mlp.up_proj",
+    "model.layers.5.mlp.switch_mlp.down_proj",
+    "model.layers.5.mlp.switch_mlp.gate_proj",
+    "model.layers.5.mlp.switch_mlp.up_proj",
+    "model.layers.5.self_attn.k_proj",
+    "model.layers.5.self_attn.o_proj",
+    "model.layers.5.self_attn.q_proj",
+    "model.layers.5.self_attn.v_proj",
+    "model.layers.50.mlp.shared_mlp.down_proj",
+    "model.layers.50.mlp.shared_mlp.gate_proj",
+    "model.layers.50.mlp.shared_mlp.up_proj",
+    "model.layers.50.mlp.switch_mlp.down_proj",
+    "model.layers.50.mlp.switch_mlp.gate_proj",
+    "model.layers.50.mlp.switch_mlp.up_proj",
+    "model.layers.50.self_attn.k_proj",
+    "model.layers.50.self_attn.o_proj",
+    "model.layers.50.self_attn.q_proj",
+    "model.layers.50.self_attn.v_proj",
+    "model.layers.51.mlp.shared_mlp.down_proj",
+    "model.layers.51.mlp.shared_mlp.gate_proj",
+    "model.layers.51.mlp.shared_mlp.up_proj",
+    "model.layers.51.mlp.switch_mlp.down_proj",
+    "model.layers.51.mlp.switch_mlp.gate_proj",
+    "model.layers.51.mlp.switch_mlp.up_proj",
+    "model.layers.51.self_attn.k_proj",
+    "model.layers.51.self_attn.o_proj",
+    "model.layers.51.self_attn.q_proj",
+    "model.layers.51.self_attn.v_proj",
+    "model.layers.52.mlp.shared_mlp.down_proj",
+    "model.layers.52.mlp.shared_mlp.gate_proj",
+    "model.layers.52.mlp.shared_mlp.up_proj",
+    "model.layers.52.mlp.switch_mlp.down_proj",
+    "model.layers.52.mlp.switch_mlp.gate_proj",
+    "model.layers.52.mlp.switch_mlp.up_proj",
+    "model.layers.52.self_attn.k_proj",
+    "model.layers.52.self_attn.o_proj",
+    "model.layers.52.self_attn.q_proj",
+    "model.layers.52.self_attn.v_proj",
+    "model.layers.53.mlp.shared_mlp.down_proj",
+    "model.layers.53.mlp.shared_mlp.gate_proj",
+    "model.layers.53.mlp.shared_mlp.up_proj",
+    "model.layers.53.mlp.switch_mlp.down_proj",
+    "model.layers.53.mlp.switch_mlp.gate_proj",
+    "model.layers.53.mlp.switch_mlp.up_proj",
+    "model.layers.53.self_attn.k_proj",
+    "model.layers.53.self_attn.o_proj",
+    "model.layers.53.self_attn.q_proj",
+    "model.layers.53.self_attn.v_proj",
+    "model.layers.54.mlp.shared_mlp.down_proj",
+    "model.layers.54.mlp.shared_mlp.gate_proj",
+    "model.layers.54.mlp.shared_mlp.up_proj",
+    "model.layers.54.mlp.switch_mlp.down_proj",
+    "model.layers.54.mlp.switch_mlp.gate_proj",
+    "model.layers.54.mlp.switch_mlp.up_proj",
+    "model.layers.54.self_attn.k_proj",
+    "model.layers.54.self_attn.o_proj",
+    "model.layers.54.self_attn.q_proj",
+    "model.layers.54.self_attn.v_proj",
+    "model.layers.55.mlp.shared_mlp.down_proj",
+    "model.layers.55.mlp.shared_mlp.gate_proj",
+    "model.layers.55.mlp.shared_mlp.up_proj",
+    "model.layers.55.mlp.switch_mlp.down_proj",
+    "model.layers.55.mlp.switch_mlp.gate_proj",
+    "model.layers.55.mlp.switch_mlp.up_proj",
+    "model.layers.55.self_attn.k_proj",
+    "model.layers.55.self_attn.o_proj",
+    "model.layers.55.self_attn.q_proj",
+    "model.layers.55.self_attn.v_proj",
+    "model.layers.56.mlp.shared_mlp.down_proj",
+    "model.layers.56.mlp.shared_mlp.gate_proj",
+    "model.layers.56.mlp.shared_mlp.up_proj",
+    "model.layers.56.mlp.switch_mlp.down_proj",
+    "model.layers.56.mlp.switch_mlp.gate_proj",
+    "model.layers.56.mlp.switch_mlp.up_proj",
+    "model.layers.56.self_attn.k_proj",
+    "model.layers.56.self_attn.o_proj",
+    "model.layers.56.self_attn.q_proj",
+    "model.layers.56.self_attn.v_proj",
+    "model.layers.57.mlp.shared_mlp.down_proj",
+    "model.layers.57.mlp.shared_mlp.gate_proj",
+    "model.layers.57.mlp.shared_mlp.up_proj",
+    "model.layers.57.mlp.switch_mlp.down_proj",
+    "model.layers.57.mlp.switch_mlp.gate_proj",
+    "model.layers.57.mlp.switch_mlp.up_proj",
+    "model.layers.57.self_attn.k_proj",
+    "model.layers.57.self_attn.o_proj",
+    "model.layers.57.self_attn.q_proj",
+    "model.layers.57.self_attn.v_proj",
+    "model.layers.58.mlp.shared_mlp.down_proj",
+    "model.layers.58.mlp.shared_mlp.gate_proj",
+    "model.layers.58.mlp.shared_mlp.up_proj",
+    "model.layers.58.mlp.switch_mlp.down_proj",
+    "model.layers.58.mlp.switch_mlp.gate_proj",
+    "model.layers.58.mlp.switch_mlp.up_proj",
+    "model.layers.58.self_attn.k_proj",
+    "model.layers.58.self_attn.o_proj",
+    "model.layers.58.self_attn.q_proj",
+    "model.layers.58.self_attn.v_proj",
+    "model.layers.59.mlp.shared_mlp.down_proj",
+    "model.layers.59.mlp.shared_mlp.gate_proj",
+    "model.layers.59.mlp.shared_mlp.up_proj",
+    "model.layers.59.mlp.switch_mlp.down_proj",
+    "model.layers.59.mlp.switch_mlp.gate_proj",
+    "model.layers.59.mlp.switch_mlp.up_proj",
+    "model.layers.59.self_attn.k_proj",
+    "model.layers.59.self_attn.o_proj",
+    "model.layers.59.self_attn.q_proj",
+    "model.layers.59.self_attn.v_proj",
+    "model.layers.6.mlp.shared_mlp.down_proj",
+    "model.layers.6.mlp.shared_mlp.gate_proj",
+    "model.layers.6.mlp.shared_mlp.up_proj",
+    "model.layers.6.mlp.switch_mlp.down_proj",
+    "model.layers.6.mlp.switch_mlp.gate_proj",
+    "model.layers.6.mlp.switch_mlp.up_proj",
+    "model.layers.6.self_attn.k_proj",
+    "model.layers.6.self_attn.o_proj",
+    "model.layers.6.self_attn.q_proj",
+    "model.layers.6.self_attn.v_proj",
+    "model.layers.60.mlp.shared_mlp.down_proj",
+    "model.layers.60.mlp.shared_mlp.gate_proj",
+    "model.layers.60.mlp.shared_mlp.up_proj",
+    "model.layers.60.mlp.switch_mlp.down_proj",
+    "model.layers.60.mlp.switch_mlp.gate_proj",
+    "model.layers.60.mlp.switch_mlp.up_proj",
+    "model.layers.60.self_attn.k_proj",
+    "model.layers.60.self_attn.o_proj",
+    "model.layers.60.self_attn.q_proj",
+    "model.layers.60.self_attn.v_proj",
+    "model.layers.61.mlp.shared_mlp.down_proj",
+    "model.layers.61.mlp.shared_mlp.gate_proj",
+    "model.layers.61.mlp.shared_mlp.up_proj",
+    "model.layers.61.mlp.switch_mlp.down_proj",
+    "model.layers.61.mlp.switch_mlp.gate_proj",
+    "model.layers.61.mlp.switch_mlp.up_proj",
+    "model.layers.61.self_attn.k_proj",
+    "model.layers.61.self_attn.o_proj",
+    "model.layers.61.self_attn.q_proj",
+    "model.layers.61.self_attn.v_proj",
+    "model.layers.62.mlp.shared_mlp.down_proj",
+    "model.layers.62.mlp.shared_mlp.gate_proj",
+    "model.layers.62.mlp.shared_mlp.up_proj",
+    "model.layers.62.mlp.switch_mlp.down_proj",
+    "model.layers.62.mlp.switch_mlp.gate_proj",
+    "model.layers.62.mlp.switch_mlp.up_proj",
+    "model.layers.62.self_attn.k_proj",
+    "model.layers.62.self_attn.o_proj",
+    "model.layers.62.self_attn.q_proj",
+    "model.layers.62.self_attn.v_proj",
+    "model.layers.63.mlp.shared_mlp.down_proj",
+    "model.layers.63.mlp.shared_mlp.gate_proj",
+    "model.layers.63.mlp.shared_mlp.up_proj",
+    "model.layers.63.mlp.switch_mlp.down_proj",
+    "model.layers.63.mlp.switch_mlp.gate_proj",
+    "model.layers.63.mlp.switch_mlp.up_proj",
+    "model.layers.63.self_attn.k_proj",
+    "model.layers.63.self_attn.o_proj",
+    "model.layers.63.self_attn.q_proj",
+    "model.layers.63.self_attn.v_proj",
+    "model.layers.64.mlp.shared_mlp.down_proj",
+    "model.layers.64.mlp.shared_mlp.gate_proj",
+    "model.layers.64.mlp.shared_mlp.up_proj",
+    "model.layers.64.mlp.switch_mlp.down_proj",
+    "model.layers.64.mlp.switch_mlp.gate_proj",
+    "model.layers.64.mlp.switch_mlp.up_proj",
+    "model.layers.64.self_attn.k_proj",
+    "model.layers.64.self_attn.o_proj",
+    "model.layers.64.self_attn.q_proj",
+    "model.layers.64.self_attn.v_proj",
+    "model.layers.65.mlp.shared_mlp.down_proj",
+    "model.layers.65.mlp.shared_mlp.gate_proj",
+    "model.layers.65.mlp.shared_mlp.up_proj",
+    "model.layers.65.mlp.switch_mlp.down_proj",
+    "model.layers.65.mlp.switch_mlp.gate_proj",
+    "model.layers.65.mlp.switch_mlp.up_proj",
+    "model.layers.65.self_attn.k_proj",
+    "model.layers.65.self_attn.o_proj",
+    "model.layers.65.self_attn.q_proj",
+    "model.layers.65.self_attn.v_proj",
+    "model.layers.66.mlp.shared_mlp.down_proj",
+    "model.layers.66.mlp.shared_mlp.gate_proj",
+    "model.layers.66.mlp.shared_mlp.up_proj",
+    "model.layers.66.mlp.switch_mlp.down_proj",
+    "model.layers.66.mlp.switch_mlp.gate_proj",
+    "model.layers.66.mlp.switch_mlp.up_proj",
+    "model.layers.66.self_attn.k_proj",
+    "model.layers.66.self_attn.o_proj",
+    "model.layers.66.self_attn.q_proj",
+    "model.layers.66.self_attn.v_proj",
+    "model.layers.67.mlp.shared_mlp.down_proj",
+    "model.layers.67.mlp.shared_mlp.gate_proj",
+    "model.layers.67.mlp.shared_mlp.up_proj",
+    "model.layers.67.mlp.switch_mlp.down_proj",
+    "model.layers.67.mlp.switch_mlp.gate_proj",
+    "model.layers.67.mlp.switch_mlp.up_proj",
+    "model.layers.67.self_attn.k_proj",
+    "model.layers.67.self_attn.o_proj",
+    "model.layers.67.self_attn.q_proj",
+    "model.layers.67.self_attn.v_proj",
+    "model.layers.68.mlp.shared_mlp.down_proj",
+    "model.layers.68.mlp.shared_mlp.gate_proj",
+    "model.layers.68.mlp.shared_mlp.up_proj",
+    "model.layers.68.mlp.switch_mlp.down_proj",
+    "model.layers.68.mlp.switch_mlp.gate_proj",
+    "model.layers.68.mlp.switch_mlp.up_proj",
+    "model.layers.68.self_attn.k_proj",
+    "model.layers.68.self_attn.o_proj",
+    "model.layers.68.self_attn.q_proj",
+    "model.layers.68.self_attn.v_proj",
+    "model.layers.69.mlp.shared_mlp.down_proj",
+    "model.layers.69.mlp.shared_mlp.gate_proj",
+    "model.layers.69.mlp.shared_mlp.up_proj",
+    "model.layers.69.mlp.switch_mlp.down_proj",
+    "model.layers.69.mlp.switch_mlp.gate_proj",
+    "model.layers.69.mlp.switch_mlp.up_proj",
+    "model.layers.69.self_attn.k_proj",
+    "model.layers.69.self_attn.o_proj",
+    "model.layers.69.self_attn.q_proj",
+    "model.layers.69.self_attn.v_proj",
+    "model.layers.7.mlp.shared_mlp.down_proj",
+    "model.layers.7.mlp.shared_mlp.gate_proj",
+    "model.layers.7.mlp.shared_mlp.up_proj",
+    "model.layers.7.mlp.switch_mlp.down_proj",
+    "model.layers.7.mlp.switch_mlp.gate_proj",
+    "model.layers.7.mlp.switch_mlp.up_proj",
+    "model.layers.7.self_attn.k_proj",
+    "model.layers.7.self_attn.o_proj",
+    "model.layers.7.self_attn.q_proj",
+    "model.layers.7.self_attn.v_proj",
+    "model.layers.70.mlp.shared_mlp.down_proj",
+    "model.layers.70.mlp.shared_mlp.gate_proj",
+    "model.layers.70.mlp.shared_mlp.up_proj",
+    "model.layers.70.mlp.switch_mlp.down_proj",
+    "model.layers.70.mlp.switch_mlp.gate_proj",
+    "model.layers.70.mlp.switch_mlp.up_proj",
+    "model.layers.70.self_attn.k_proj",
+    "model.layers.70.self_attn.o_proj",
+    "model.layers.70.self_attn.q_proj",
+    "model.layers.70.self_attn.v_proj",
+    "model.layers.71.mlp.shared_mlp.down_proj",
+    "model.layers.71.mlp.shared_mlp.gate_proj",
+    "model.layers.71.mlp.shared_mlp.up_proj",
+    "model.layers.71.mlp.switch_mlp.down_proj",
+    "model.layers.71.mlp.switch_mlp.gate_proj",
+    "model.layers.71.mlp.switch_mlp.up_proj",
+    "model.layers.71.self_attn.k_proj",
+    "model.layers.71.self_attn.o_proj",
+    "model.layers.71.self_attn.q_proj",
+    "model.layers.71.self_attn.v_proj",
+    "model.layers.72.mlp.shared_mlp.down_proj",
+    "model.layers.72.mlp.shared_mlp.gate_proj",
+    "model.layers.72.mlp.shared_mlp.up_proj",
+    "model.layers.72.mlp.switch_mlp.down_proj",
+    "model.layers.72.mlp.switch_mlp.gate_proj",
+    "model.layers.72.mlp.switch_mlp.up_proj",
+    "model.layers.72.self_attn.k_proj",
+    "model.layers.72.self_attn.o_proj",
+    "model.layers.72.self_attn.q_proj",
+    "model.layers.72.self_attn.v_proj",
+    "model.layers.73.mlp.shared_mlp.down_proj",
+    "model.layers.73.mlp.shared_mlp.gate_proj",
+    "model.layers.73.mlp.shared_mlp.up_proj",
+    "model.layers.73.mlp.switch_mlp.down_proj",
+    "model.layers.73.mlp.switch_mlp.gate_proj",
+    "model.layers.73.mlp.switch_mlp.up_proj",
+    "model.layers.73.self_attn.k_proj",
+    "model.layers.73.self_attn.o_proj",
+    "model.layers.73.self_attn.q_proj",
+    "model.layers.73.self_attn.v_proj",
+    "model.layers.74.mlp.shared_mlp.down_proj",
+    "model.layers.74.mlp.shared_mlp.gate_proj",
+    "model.layers.74.mlp.shared_mlp.up_proj",
+    "model.layers.74.mlp.switch_mlp.down_proj",
+    "model.layers.74.mlp.switch_mlp.gate_proj",
+    "model.layers.74.mlp.switch_mlp.up_proj",
+    "model.layers.74.self_attn.k_proj",
+    "model.layers.74.self_attn.o_proj",
+    "model.layers.74.self_attn.q_proj",
+    "model.layers.74.self_attn.v_proj",
+    "model.layers.75.mlp.shared_mlp.down_proj",
+    "model.layers.75.mlp.shared_mlp.gate_proj",
+    "model.layers.75.mlp.shared_mlp.up_proj",
+    "model.layers.75.mlp.switch_mlp.down_proj",
+    "model.layers.75.mlp.switch_mlp.gate_proj",
+    "model.layers.75.mlp.switch_mlp.up_proj",
+    "model.layers.75.self_attn.k_proj",
+    "model.layers.75.self_attn.o_proj",
+    "model.layers.75.self_attn.q_proj",
+    "model.layers.75.self_attn.v_proj",
+    "model.layers.76.mlp.shared_mlp.down_proj",
+    "model.layers.76.mlp.shared_mlp.gate_proj",
+    "model.layers.76.mlp.shared_mlp.up_proj",
+    "model.layers.76.mlp.switch_mlp.down_proj",
+    "model.layers.76.mlp.switch_mlp.gate_proj",
+    "model.layers.76.mlp.switch_mlp.up_proj",
+    "model.layers.76.self_attn.k_proj",
+    "model.layers.76.self_attn.o_proj",
+    "model.layers.76.self_attn.q_proj",
+    "model.layers.76.self_attn.v_proj",
+    "model.layers.77.mlp.shared_mlp.down_proj",
+    "model.layers.77.mlp.shared_mlp.gate_proj",
+    "model.layers.77.mlp.shared_mlp.up_proj",
+    "model.layers.77.mlp.switch_mlp.down_proj",
+    "model.layers.77.mlp.switch_mlp.gate_proj",
+    "model.layers.77.mlp.switch_mlp.up_proj",
+    "model.layers.77.self_attn.k_proj",
+    "model.layers.77.self_attn.o_proj",
+    "model.layers.77.self_attn.q_proj",
+    "model.layers.77.self_attn.v_proj",
+    "model.layers.78.mlp.shared_mlp.down_proj",
+    "model.layers.78.mlp.shared_mlp.gate_proj",
+    "model.layers.78.mlp.shared_mlp.up_proj",
+    "model.layers.78.mlp.switch_mlp.down_proj",
+    "model.layers.78.mlp.switch_mlp.gate_proj",
+    "model.layers.78.mlp.switch_mlp.up_proj",
+    "model.layers.78.self_attn.k_proj",
+    "model.layers.78.self_attn.o_proj",
+    "model.layers.78.self_attn.q_proj",
+    "model.layers.78.self_attn.v_proj",
+    "model.layers.79.mlp.shared_mlp.down_proj",
+    "model.layers.79.mlp.shared_mlp.gate_proj",
+    "model.layers.79.mlp.shared_mlp.up_proj",
+    "model.layers.79.mlp.switch_mlp.down_proj",
+    "model.layers.79.mlp.switch_mlp.gate_proj",
+    "model.layers.79.mlp.switch_mlp.up_proj",
+    "model.layers.79.self_attn.k_proj",
+    "model.layers.79.self_attn.o_proj",
+    "model.layers.79.self_attn.q_proj",
+    "model.layers.79.self_attn.v_proj",
+    "model.layers.8.mlp.shared_mlp.down_proj",
+    "model.layers.8.mlp.shared_mlp.gate_proj",
+    "model.layers.8.mlp.shared_mlp.up_proj",
+    "model.layers.8.mlp.switch_mlp.down_proj",
+    "model.layers.8.mlp.switch_mlp.gate_proj",
+    "model.layers.8.mlp.switch_mlp.up_proj",
+    "model.layers.8.self_attn.k_proj",
+    "model.layers.8.self_attn.o_proj",
+    "model.layers.8.self_attn.q_proj",
+    "model.layers.8.self_attn.v_proj",
+    "model.layers.9.mlp.shared_mlp.down_proj",
+    "model.layers.9.mlp.shared_mlp.gate_proj",
+    "model.layers.9.mlp.shared_mlp.up_proj",
+    "model.layers.9.mlp.switch_mlp.down_proj",
+    "model.layers.9.mlp.switch_mlp.gate_proj",
+    "model.layers.9.mlp.switch_mlp.up_proj",
+    "model.layers.9.self_attn.k_proj",
+    "model.layers.9.self_attn.o_proj",
+    "model.layers.9.self_attn.q_proj",
+    "model.layers.9.self_attn.v_proj"
+  ],
+  "missing": [
+    "lm_head",
+    "model.embed_tokens"
+  ],
+  "mismatched": [],
+  "zero_count_experts": 0
+}