Upload folder using huggingface_hub

bench.txt

@@ -0,0 +1,27 @@
+hf (pretrained=/workspace/output/glm4_7_30b/hf_temp_07i,trust_remote_code=True), gen_kwargs: (max_gen_toks=10), limit: None, num_fewshot: None, batch_size: 8
+|    Tasks    |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|-------------|------:|------|-----:|-----:|---|-----:|---|------|
+|niah_single_1|      1|none  |     0| 16384|↑  | 0.992|±  |   N/A|
+|             |       |none  |     0| 24576|   | 0.988|±  |0.0049|
+|             |       |none  |     0| 32768|↑  | 0.962|±  |   N/A|
+|             |       |none  |     0|  4096|↑  |-1.000|±  |   N/A|
+|             |       |none  |     0| 48000|   | 0.564|±  |0.0222|
+|             |       |none  |     0| 65536|↑  | 0.000|±  |   N/A|
+|             |       |none  |     0|  8192|↑  | 0.998|±  |   N/A|
+
+hf (pretrained=/workspace/output/glm4_7_30b/hf_temp_07i,trust_remote_code=True), gen_kwargs: (max_gen_toks=10), limit: None, num_fewshot: None, batch_size: 8
+|    Tasks    |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|-------------|------:|------|-----:|-----:|---|-----:|---|------|
+|niah_single_2|      1|none  |     0| 16384|↑  | 0.852|±  |   N/A|
+|             |       |none  |     0| 24576|   | 0.750|±  |0.0194|
+|             |       |none  |     0| 32768|↑  | 0.390|±  |   N/A|
+|             |       |none  |     0|  4096|↑  |-1.000|±  |   N/A|
+|             |       |none  |     0|  8192|↑  | 0.972|±  |   N/A|
+
+hf (pretrained=/workspace/output/glm4_7_30b/hf_temp_07i,trust_remote_code=True), gen_kwargs: (max_gen_toks=64), limit: None, num_fewshot: None, batch_size: 8
+|    Tasks    |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|-------------|------:|------|-----:|-----:|---|-----:|---|------|
+|niah_single_3|      1|none  |     0| 16384|↑  | 0.898|±  |   N/A|
+|             |       |none  |     0| 24576|   | 0.842|±  |0.0163|
+|             |       |none  |     0|  4096|↑  |-1.000|±  |   N/A|
+|             |       |none  |     0|  8192|↑  | 0.946|±  |   N/A|

config.json

@@ -12,7 +12,7 @@
   "transformer_layers": [-1], 
   
   "rwkv_layers": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46],
-  "tiny_attention_layers": [21, 23, 25, 27, 28, 30, 33, 37, 38, 41, 43],
+  "tiny_attention_layers": [0,7,12,16,21,25,29,32,35,36,37,38,39,40,41,42,43,44,45,46],
   "rwkv_architecture": "hxa07i",
   "enable_qk_norm": true,
   "nope_in_transformer": true,
@@ -20,8 +20,8 @@
   "tiny_head_dim": 128,
   "tiny_n_heads": 4,
   "tiny_kv_heads": 2,
-  "lora_rank_decay": 512,
-  "lora_rank_iclr":256,
+  "lora_rank_decay": 384,
+  "lora_rank_iclr":128,
   "lora_rank_value_residual_mix":-1,
   "lora_rank_key_residual_mix":-1,
   "lora_rank_gate":384,

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 60740379392,
+    "total_size": 60630414592,
     "format": "safetensors"
   },
   "weight_map": {
@@ -23,6 +23,14 @@
     "model.layers.0.self_attn.receptance_a_layernorm.weight": "model-00001-of-00016.safetensors",
     "model.layers.0.self_attn.receptance_a_proj.weight": "model-00001-of-00016.safetensors",
     "model.layers.0.self_attn.receptance_b_proj.weight": "model-00001-of-00016.safetensors",
+    "model.layers.0.self_attn.tiny_g1": "model-00001-of-00016.safetensors",
+    "model.layers.0.self_attn.tiny_g2": "model-00001-of-00016.safetensors",
+    "model.layers.0.self_attn.tiny_k_norm.weight": "model-00001-of-00016.safetensors",
+    "model.layers.0.self_attn.tiny_k_proj.weight": "model-00001-of-00016.safetensors",
+    "model.layers.0.self_attn.tiny_o_proj.weight": "model-00001-of-00016.safetensors",
+    "model.layers.0.self_attn.tiny_q_norm.weight": "model-00001-of-00016.safetensors",
+    "model.layers.0.self_attn.tiny_q_proj.weight": "model-00001-of-00016.safetensors",
+    "model.layers.0.self_attn.tiny_v_proj.weight": "model-00001-of-00016.safetensors",
     "model.layers.0.self_attn.w0": "model-00001-of-00016.safetensors",
     "model.layers.0.self_attn.w1": "model-00001-of-00016.safetensors",
     "model.layers.0.self_attn.w2": "model-00001-of-00016.safetensors",
@@ -191,6 +199,14 @@
     "model.layers.7.self_attn.receptance_a_layernorm.weight": "model-00003-of-00016.safetensors",
     "model.layers.7.self_attn.receptance_a_proj.weight": "model-00003-of-00016.safetensors",
     "model.layers.7.self_attn.receptance_b_proj.weight": "model-00003-of-00016.safetensors",
+    "model.layers.7.self_attn.tiny_g1": "model-00003-of-00016.safetensors",
+    "model.layers.7.self_attn.tiny_g2": "model-00003-of-00016.safetensors",
+    "model.layers.7.self_attn.tiny_k_norm.weight": "model-00003-of-00016.safetensors",
+    "model.layers.7.self_attn.tiny_k_proj.weight": "model-00003-of-00016.safetensors",
+    "model.layers.7.self_attn.tiny_o_proj.weight": "model-00003-of-00016.safetensors",
+    "model.layers.7.self_attn.tiny_q_norm.weight": "model-00003-of-00016.safetensors",
+    "model.layers.7.self_attn.tiny_q_proj.weight": "model-00003-of-00016.safetensors",
+    "model.layers.7.self_attn.tiny_v_proj.weight": "model-00003-of-00016.safetensors",
     "model.layers.7.self_attn.w0": "model-00003-of-00016.safetensors",
     "model.layers.7.self_attn.w1": "model-00003-of-00016.safetensors",
     "model.layers.7.self_attn.w2": "model-00003-of-00016.safetensors",
@@ -311,6 +327,14 @@
     "model.layers.12.self_attn.receptance_a_layernorm.weight": "model-00005-of-00016.safetensors",
     "model.layers.12.self_attn.receptance_a_proj.weight": "model-00005-of-00016.safetensors",
     "model.layers.12.self_attn.receptance_b_proj.weight": "model-00005-of-00016.safetensors",
+    "model.layers.12.self_attn.tiny_g1": "model-00005-of-00016.safetensors",
+    "model.layers.12.self_attn.tiny_g2": "model-00005-of-00016.safetensors",
+    "model.layers.12.self_attn.tiny_k_norm.weight": "model-00005-of-00016.safetensors",
+    "model.layers.12.self_attn.tiny_k_proj.weight": "model-00005-of-00016.safetensors",
+    "model.layers.12.self_attn.tiny_o_proj.weight": "model-00005-of-00016.safetensors",
+    "model.layers.12.self_attn.tiny_q_norm.weight": "model-00005-of-00016.safetensors",
+    "model.layers.12.self_attn.tiny_q_proj.weight": "model-00005-of-00016.safetensors",
+    "model.layers.12.self_attn.tiny_v_proj.weight": "model-00005-of-00016.safetensors",
     "model.layers.12.self_attn.w0": "model-00005-of-00016.safetensors",
     "model.layers.12.self_attn.w1": "model-00005-of-00016.safetensors",
     "model.layers.12.self_attn.w2": "model-00005-of-00016.safetensors",
@@ -407,6 +431,14 @@
     "model.layers.16.self_attn.receptance_a_layernorm.weight": "model-00006-of-00016.safetensors",
     "model.layers.16.self_attn.receptance_a_proj.weight": "model-00006-of-00016.safetensors",
     "model.layers.16.self_attn.receptance_b_proj.weight": "model-00006-of-00016.safetensors",
+    "model.layers.16.self_attn.tiny_g1": "model-00006-of-00016.safetensors",
+    "model.layers.16.self_attn.tiny_g2": "model-00006-of-00016.safetensors",
+    "model.layers.16.self_attn.tiny_k_norm.weight": "model-00006-of-00016.safetensors",
+    "model.layers.16.self_attn.tiny_k_proj.weight": "model-00006-of-00016.safetensors",
+    "model.layers.16.self_attn.tiny_o_proj.weight": "model-00006-of-00016.safetensors",
+    "model.layers.16.self_attn.tiny_q_norm.weight": "model-00006-of-00016.safetensors",
+    "model.layers.16.self_attn.tiny_q_proj.weight": "model-00006-of-00016.safetensors",
+    "model.layers.16.self_attn.tiny_v_proj.weight": "model-00006-of-00016.safetensors",
     "model.layers.16.self_attn.w0": "model-00006-of-00016.safetensors",
     "model.layers.16.self_attn.w1": "model-00006-of-00016.safetensors",
     "model.layers.16.self_attn.w2": "model-00006-of-00016.safetensors",
@@ -583,14 +615,6 @@
     "model.layers.23.self_attn.receptance_a_layernorm.weight": "model-00008-of-00016.safetensors",
     "model.layers.23.self_attn.receptance_a_proj.weight": "model-00008-of-00016.safetensors",
     "model.layers.23.self_attn.receptance_b_proj.weight": "model-00008-of-00016.safetensors",
-    "model.layers.23.self_attn.tiny_g1": "model-00008-of-00016.safetensors",
-    "model.layers.23.self_attn.tiny_g2": "model-00008-of-00016.safetensors",
-    "model.layers.23.self_attn.tiny_k_norm.weight": "model-00008-of-00016.safetensors",
-    "model.layers.23.self_attn.tiny_k_proj.weight": "model-00008-of-00016.safetensors",
-    "model.layers.23.self_attn.tiny_o_proj.weight": "model-00008-of-00016.safetensors",
-    "model.layers.23.self_attn.tiny_q_norm.weight": "model-00008-of-00016.safetensors",
-    "model.layers.23.self_attn.tiny_q_proj.weight": "model-00008-of-00016.safetensors",
-    "model.layers.23.self_attn.tiny_v_proj.weight": "model-00008-of-00016.safetensors",
     "model.layers.23.self_attn.w0": "model-00008-of-00016.safetensors",
     "model.layers.23.self_attn.w1": "model-00008-of-00016.safetensors",
     "model.layers.23.self_attn.w2": "model-00008-of-00016.safetensors",
@@ -695,14 +719,6 @@
     "model.layers.27.self_attn.receptance_a_layernorm.weight": "model-00010-of-00016.safetensors",
     "model.layers.27.self_attn.receptance_a_proj.weight": "model-00010-of-00016.safetensors",
     "model.layers.27.self_attn.receptance_b_proj.weight": "model-00010-of-00016.safetensors",
-    "model.layers.27.self_attn.tiny_g1": "model-00010-of-00016.safetensors",
-    "model.layers.27.self_attn.tiny_g2": "model-00010-of-00016.safetensors",
-    "model.layers.27.self_attn.tiny_k_norm.weight": "model-00010-of-00016.safetensors",
-    "model.layers.27.self_attn.tiny_k_proj.weight": "model-00010-of-00016.safetensors",
-    "model.layers.27.self_attn.tiny_o_proj.weight": "model-00010-of-00016.safetensors",
-    "model.layers.27.self_attn.tiny_q_norm.weight": "model-00010-of-00016.safetensors",
-    "model.layers.27.self_attn.tiny_q_proj.weight": "model-00010-of-00016.safetensors",
-    "model.layers.27.self_attn.tiny_v_proj.weight": "model-00010-of-00016.safetensors",
     "model.layers.27.self_attn.w0": "model-00010-of-00016.safetensors",
     "model.layers.27.self_attn.w1": "model-00010-of-00016.safetensors",
     "model.layers.27.self_attn.w2": "model-00010-of-00016.safetensors",
@@ -727,14 +743,6 @@
     "model.layers.28.self_attn.receptance_a_layernorm.weight": "model-00010-of-00016.safetensors",
     "model.layers.28.self_attn.receptance_a_proj.weight": "model-00010-of-00016.safetensors",
     "model.layers.28.self_attn.receptance_b_proj.weight": "model-00010-of-00016.safetensors",
-    "model.layers.28.self_attn.tiny_g1": "model-00010-of-00016.safetensors",
-    "model.layers.28.self_attn.tiny_g2": "model-00010-of-00016.safetensors",
-    "model.layers.28.self_attn.tiny_k_norm.weight": "model-00010-of-00016.safetensors",
-    "model.layers.28.self_attn.tiny_k_proj.weight": "model-00010-of-00016.safetensors",
-    "model.layers.28.self_attn.tiny_o_proj.weight": "model-00010-of-00016.safetensors",
-    "model.layers.28.self_attn.tiny_q_norm.weight": "model-00010-of-00016.safetensors",
-    "model.layers.28.self_attn.tiny_q_proj.weight": "model-00010-of-00016.safetensors",
-    "model.layers.28.self_attn.tiny_v_proj.weight": "model-00010-of-00016.safetensors",
     "model.layers.28.self_attn.w0": "model-00010-of-00016.safetensors",
     "model.layers.28.self_attn.w1": "model-00010-of-00016.safetensors",
     "model.layers.28.self_attn.w2": "model-00010-of-00016.safetensors",
@@ -759,6 +767,14 @@
     "model.layers.29.self_attn.receptance_a_layernorm.weight": "model-00010-of-00016.safetensors",
     "model.layers.29.self_attn.receptance_a_proj.weight": "model-00010-of-00016.safetensors",
     "model.layers.29.self_attn.receptance_b_proj.weight": "model-00010-of-00016.safetensors",
+    "model.layers.29.self_attn.tiny_g1": "model-00010-of-00016.safetensors",
+    "model.layers.29.self_attn.tiny_g2": "model-00010-of-00016.safetensors",
+    "model.layers.29.self_attn.tiny_k_norm.weight": "model-00010-of-00016.safetensors",
+    "model.layers.29.self_attn.tiny_k_proj.weight": "model-00010-of-00016.safetensors",
+    "model.layers.29.self_attn.tiny_o_proj.weight": "model-00010-of-00016.safetensors",
+    "model.layers.29.self_attn.tiny_q_norm.weight": "model-00010-of-00016.safetensors",
+    "model.layers.29.self_attn.tiny_q_proj.weight": "model-00010-of-00016.safetensors",
+    "model.layers.29.self_attn.tiny_v_proj.weight": "model-00010-of-00016.safetensors",
     "model.layers.29.self_attn.w0": "model-00010-of-00016.safetensors",
     "model.layers.29.self_attn.w1": "model-00010-of-00016.safetensors",
     "model.layers.29.self_attn.w2": "model-00010-of-00016.safetensors",
@@ -783,14 +799,6 @@
     "model.layers.30.self_attn.receptance_a_layernorm.weight": "model-00011-of-00016.safetensors",
     "model.layers.30.self_attn.receptance_a_proj.weight": "model-00011-of-00016.safetensors",
     "model.layers.30.self_attn.receptance_b_proj.weight": "model-00011-of-00016.safetensors",
-    "model.layers.30.self_attn.tiny_g1": "model-00011-of-00016.safetensors",
-    "model.layers.30.self_attn.tiny_g2": "model-00011-of-00016.safetensors",
-    "model.layers.30.self_attn.tiny_k_norm.weight": "model-00011-of-00016.safetensors",
-    "model.layers.30.self_attn.tiny_k_proj.weight": "model-00011-of-00016.safetensors",
-    "model.layers.30.self_attn.tiny_o_proj.weight": "model-00011-of-00016.safetensors",
-    "model.layers.30.self_attn.tiny_q_norm.weight": "model-00011-of-00016.safetensors",
-    "model.layers.30.self_attn.tiny_q_proj.weight": "model-00011-of-00016.safetensors",
-    "model.layers.30.self_attn.tiny_v_proj.weight": "model-00011-of-00016.safetensors",
     "model.layers.30.self_attn.w0": "model-00011-of-00016.safetensors",
     "model.layers.30.self_attn.w1": "model-00011-of-00016.safetensors",
     "model.layers.30.self_attn.w2": "model-00011-of-00016.safetensors",
@@ -839,6 +847,14 @@
     "model.layers.32.self_attn.receptance_a_layernorm.weight": "model-00011-of-00016.safetensors",
     "model.layers.32.self_attn.receptance_a_proj.weight": "model-00011-of-00016.safetensors",
     "model.layers.32.self_attn.receptance_b_proj.weight": "model-00011-of-00016.safetensors",
+    "model.layers.32.self_attn.tiny_g1": "model-00011-of-00016.safetensors",
+    "model.layers.32.self_attn.tiny_g2": "model-00011-of-00016.safetensors",
+    "model.layers.32.self_attn.tiny_k_norm.weight": "model-00011-of-00016.safetensors",
+    "model.layers.32.self_attn.tiny_k_proj.weight": "model-00011-of-00016.safetensors",
+    "model.layers.32.self_attn.tiny_o_proj.weight": "model-00011-of-00016.safetensors",
+    "model.layers.32.self_attn.tiny_q_norm.weight": "model-00011-of-00016.safetensors",
+    "model.layers.32.self_attn.tiny_q_proj.weight": "model-00011-of-00016.safetensors",
+    "model.layers.32.self_attn.tiny_v_proj.weight": "model-00011-of-00016.safetensors",
     "model.layers.32.self_attn.w0": "model-00011-of-00016.safetensors",
     "model.layers.32.self_attn.w1": "model-00011-of-00016.safetensors",
     "model.layers.32.self_attn.w2": "model-00011-of-00016.safetensors",
@@ -863,14 +879,6 @@
     "model.layers.33.self_attn.receptance_a_layernorm.weight": "model-00012-of-00016.safetensors",
     "model.layers.33.self_attn.receptance_a_proj.weight": "model-00012-of-00016.safetensors",
     "model.layers.33.self_attn.receptance_b_proj.weight": "model-00012-of-00016.safetensors",
-    "model.layers.33.self_attn.tiny_g1": "model-00012-of-00016.safetensors",
-    "model.layers.33.self_attn.tiny_g2": "model-00012-of-00016.safetensors",
-    "model.layers.33.self_attn.tiny_k_norm.weight": "model-00012-of-00016.safetensors",
-    "model.layers.33.self_attn.tiny_k_proj.weight": "model-00012-of-00016.safetensors",
-    "model.layers.33.self_attn.tiny_o_proj.weight": "model-00012-of-00016.safetensors",
-    "model.layers.33.self_attn.tiny_q_norm.weight": "model-00012-of-00016.safetensors",
-    "model.layers.33.self_attn.tiny_q_proj.weight": "model-00012-of-00016.safetensors",
-    "model.layers.33.self_attn.tiny_v_proj.weight": "model-00012-of-00016.safetensors",
     "model.layers.33.self_attn.w0": "model-00012-of-00016.safetensors",
     "model.layers.33.self_attn.w1": "model-00012-of-00016.safetensors",
     "model.layers.33.self_attn.w2": "model-00012-of-00016.safetensors",
@@ -919,6 +927,14 @@
     "model.layers.35.self_attn.receptance_a_layernorm.weight": "model-00012-of-00016.safetensors",
     "model.layers.35.self_attn.receptance_a_proj.weight": "model-00012-of-00016.safetensors",
     "model.layers.35.self_attn.receptance_b_proj.weight": "model-00012-of-00016.safetensors",
+    "model.layers.35.self_attn.tiny_g1": "model-00012-of-00016.safetensors",
+    "model.layers.35.self_attn.tiny_g2": "model-00012-of-00016.safetensors",
+    "model.layers.35.self_attn.tiny_k_norm.weight": "model-00012-of-00016.safetensors",
+    "model.layers.35.self_attn.tiny_k_proj.weight": "model-00012-of-00016.safetensors",
+    "model.layers.35.self_attn.tiny_o_proj.weight": "model-00012-of-00016.safetensors",
+    "model.layers.35.self_attn.tiny_q_norm.weight": "model-00012-of-00016.safetensors",
+    "model.layers.35.self_attn.tiny_q_proj.weight": "model-00012-of-00016.safetensors",
+    "model.layers.35.self_attn.tiny_v_proj.weight": "model-00012-of-00016.safetensors",
     "model.layers.35.self_attn.w0": "model-00012-of-00016.safetensors",
     "model.layers.35.self_attn.w1": "model-00012-of-00016.safetensors",
     "model.layers.35.self_attn.w2": "model-00012-of-00016.safetensors",
@@ -943,6 +959,14 @@
     "model.layers.36.self_attn.receptance_a_layernorm.weight": "model-00013-of-00016.safetensors",
     "model.layers.36.self_attn.receptance_a_proj.weight": "model-00013-of-00016.safetensors",
     "model.layers.36.self_attn.receptance_b_proj.weight": "model-00013-of-00016.safetensors",
+    "model.layers.36.self_attn.tiny_g1": "model-00013-of-00016.safetensors",
+    "model.layers.36.self_attn.tiny_g2": "model-00013-of-00016.safetensors",
+    "model.layers.36.self_attn.tiny_k_norm.weight": "model-00013-of-00016.safetensors",
+    "model.layers.36.self_attn.tiny_k_proj.weight": "model-00013-of-00016.safetensors",
+    "model.layers.36.self_attn.tiny_o_proj.weight": "model-00013-of-00016.safetensors",
+    "model.layers.36.self_attn.tiny_q_norm.weight": "model-00013-of-00016.safetensors",
+    "model.layers.36.self_attn.tiny_q_proj.weight": "model-00013-of-00016.safetensors",
+    "model.layers.36.self_attn.tiny_v_proj.weight": "model-00013-of-00016.safetensors",
     "model.layers.36.self_attn.w0": "model-00013-of-00016.safetensors",
     "model.layers.36.self_attn.w1": "model-00013-of-00016.safetensors",
     "model.layers.36.self_attn.w2": "model-00013-of-00016.safetensors",
@@ -1031,6 +1055,14 @@
     "model.layers.39.self_attn.receptance_a_layernorm.weight": "model-00014-of-00016.safetensors",
     "model.layers.39.self_attn.receptance_a_proj.weight": "model-00014-of-00016.safetensors",
     "model.layers.39.self_attn.receptance_b_proj.weight": "model-00014-of-00016.safetensors",
+    "model.layers.39.self_attn.tiny_g1": "model-00014-of-00016.safetensors",
+    "model.layers.39.self_attn.tiny_g2": "model-00014-of-00016.safetensors",
+    "model.layers.39.self_attn.tiny_k_norm.weight": "model-00014-of-00016.safetensors",
+    "model.layers.39.self_attn.tiny_k_proj.weight": "model-00014-of-00016.safetensors",
+    "model.layers.39.self_attn.tiny_o_proj.weight": "model-00014-of-00016.safetensors",
+    "model.layers.39.self_attn.tiny_q_norm.weight": "model-00014-of-00016.safetensors",
+    "model.layers.39.self_attn.tiny_q_proj.weight": "model-00014-of-00016.safetensors",
+    "model.layers.39.self_attn.tiny_v_proj.weight": "model-00014-of-00016.safetensors",
     "model.layers.39.self_attn.w0": "model-00014-of-00016.safetensors",
     "model.layers.39.self_attn.w1": "model-00014-of-00016.safetensors",
     "model.layers.39.self_attn.w2": "model-00014-of-00016.safetensors",
@@ -1055,6 +1087,14 @@
     "model.layers.40.self_attn.receptance_a_layernorm.weight": "model-00014-of-00016.safetensors",
     "model.layers.40.self_attn.receptance_a_proj.weight": "model-00014-of-00016.safetensors",
     "model.layers.40.self_attn.receptance_b_proj.weight": "model-00014-of-00016.safetensors",
+    "model.layers.40.self_attn.tiny_g1": "model-00014-of-00016.safetensors",
+    "model.layers.40.self_attn.tiny_g2": "model-00014-of-00016.safetensors",
+    "model.layers.40.self_attn.tiny_k_norm.weight": "model-00014-of-00016.safetensors",
+    "model.layers.40.self_attn.tiny_k_proj.weight": "model-00014-of-00016.safetensors",
+    "model.layers.40.self_attn.tiny_o_proj.weight": "model-00014-of-00016.safetensors",
+    "model.layers.40.self_attn.tiny_q_norm.weight": "model-00014-of-00016.safetensors",
+    "model.layers.40.self_attn.tiny_q_proj.weight": "model-00014-of-00016.safetensors",
+    "model.layers.40.self_attn.tiny_v_proj.weight": "model-00014-of-00016.safetensors",
     "model.layers.40.self_attn.w0": "model-00014-of-00016.safetensors",
     "model.layers.40.self_attn.w1": "model-00014-of-00016.safetensors",
     "model.layers.40.self_attn.w2": "model-00014-of-00016.safetensors",
@@ -1111,6 +1151,14 @@
     "model.layers.42.self_attn.receptance_a_layernorm.weight": "model-00015-of-00016.safetensors",
     "model.layers.42.self_attn.receptance_a_proj.weight": "model-00015-of-00016.safetensors",
     "model.layers.42.self_attn.receptance_b_proj.weight": "model-00015-of-00016.safetensors",
+    "model.layers.42.self_attn.tiny_g1": "model-00015-of-00016.safetensors",
+    "model.layers.42.self_attn.tiny_g2": "model-00015-of-00016.safetensors",
+    "model.layers.42.self_attn.tiny_k_norm.weight": "model-00015-of-00016.safetensors",
+    "model.layers.42.self_attn.tiny_k_proj.weight": "model-00015-of-00016.safetensors",
+    "model.layers.42.self_attn.tiny_o_proj.weight": "model-00015-of-00016.safetensors",
+    "model.layers.42.self_attn.tiny_q_norm.weight": "model-00015-of-00016.safetensors",
+    "model.layers.42.self_attn.tiny_q_proj.weight": "model-00015-of-00016.safetensors",
+    "model.layers.42.self_attn.tiny_v_proj.weight": "model-00015-of-00016.safetensors",
     "model.layers.42.self_attn.w0": "model-00015-of-00016.safetensors",
     "model.layers.42.self_attn.w1": "model-00015-of-00016.safetensors",
     "model.layers.42.self_attn.w2": "model-00015-of-00016.safetensors",
@@ -1167,6 +1215,14 @@
     "model.layers.44.self_attn.receptance_a_layernorm.weight": "model-00015-of-00016.safetensors",
     "model.layers.44.self_attn.receptance_a_proj.weight": "model-00015-of-00016.safetensors",
     "model.layers.44.self_attn.receptance_b_proj.weight": "model-00015-of-00016.safetensors",
+    "model.layers.44.self_attn.tiny_g1": "model-00015-of-00016.safetensors",
+    "model.layers.44.self_attn.tiny_g2": "model-00015-of-00016.safetensors",
+    "model.layers.44.self_attn.tiny_k_norm.weight": "model-00015-of-00016.safetensors",
+    "model.layers.44.self_attn.tiny_k_proj.weight": "model-00015-of-00016.safetensors",
+    "model.layers.44.self_attn.tiny_o_proj.weight": "model-00015-of-00016.safetensors",
+    "model.layers.44.self_attn.tiny_q_norm.weight": "model-00015-of-00016.safetensors",
+    "model.layers.44.self_attn.tiny_q_proj.weight": "model-00015-of-00016.safetensors",
+    "model.layers.44.self_attn.tiny_v_proj.weight": "model-00015-of-00016.safetensors",
     "model.layers.44.self_attn.w0": "model-00015-of-00016.safetensors",
     "model.layers.44.self_attn.w1": "model-00015-of-00016.safetensors",
     "model.layers.44.self_attn.w2": "model-00015-of-00016.safetensors",
@@ -1191,6 +1247,14 @@
     "model.layers.45.self_attn.receptance_a_layernorm.weight": "model-00016-of-00016.safetensors",
     "model.layers.45.self_attn.receptance_a_proj.weight": "model-00016-of-00016.safetensors",
     "model.layers.45.self_attn.receptance_b_proj.weight": "model-00016-of-00016.safetensors",
+    "model.layers.45.self_attn.tiny_g1": "model-00016-of-00016.safetensors",
+    "model.layers.45.self_attn.tiny_g2": "model-00016-of-00016.safetensors",
+    "model.layers.45.self_attn.tiny_k_norm.weight": "model-00016-of-00016.safetensors",
+    "model.layers.45.self_attn.tiny_k_proj.weight": "model-00016-of-00016.safetensors",
+    "model.layers.45.self_attn.tiny_o_proj.weight": "model-00016-of-00016.safetensors",
+    "model.layers.45.self_attn.tiny_q_norm.weight": "model-00016-of-00016.safetensors",
+    "model.layers.45.self_attn.tiny_q_proj.weight": "model-00016-of-00016.safetensors",
+    "model.layers.45.self_attn.tiny_v_proj.weight": "model-00016-of-00016.safetensors",
     "model.layers.45.self_attn.w0": "model-00016-of-00016.safetensors",
     "model.layers.45.self_attn.w1": "model-00016-of-00016.safetensors",
     "model.layers.45.self_attn.w2": "model-00016-of-00016.safetensors",
@@ -1215,6 +1279,14 @@
     "model.layers.46.self_attn.receptance_a_layernorm.weight": "model-00016-of-00016.safetensors",
     "model.layers.46.self_attn.receptance_a_proj.weight": "model-00016-of-00016.safetensors",
     "model.layers.46.self_attn.receptance_b_proj.weight": "model-00016-of-00016.safetensors",
+    "model.layers.46.self_attn.tiny_g1": "model-00016-of-00016.safetensors",
+    "model.layers.46.self_attn.tiny_g2": "model-00016-of-00016.safetensors",
+    "model.layers.46.self_attn.tiny_k_norm.weight": "model-00016-of-00016.safetensors",
+    "model.layers.46.self_attn.tiny_k_proj.weight": "model-00016-of-00016.safetensors",
+    "model.layers.46.self_attn.tiny_o_proj.weight": "model-00016-of-00016.safetensors",
+    "model.layers.46.self_attn.tiny_q_norm.weight": "model-00016-of-00016.safetensors",
+    "model.layers.46.self_attn.tiny_q_proj.weight": "model-00016-of-00016.safetensors",
+    "model.layers.46.self_attn.tiny_v_proj.weight": "model-00016-of-00016.safetensors",
     "model.layers.46.self_attn.w0": "model-00016-of-00016.safetensors",
     "model.layers.46.self_attn.w1": "model-00016-of-00016.safetensors",
     "model.layers.46.self_attn.w2": "model-00016-of-00016.safetensors",