Add files using upload-large-folder tool

config.json

@@ -0,0 +1,1879 @@
+{
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13696,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 40,
+  "rms_norm_eps": 0.0078125,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "talkie_attn_gains": [
+    0.00051116943359375,
+    0.000835418701171875,
+    0.00040435791015625,
+    3.719329833984375e-05,
+    0.000370025634765625,
+    0.00176239013671875,
+    0.0012664794921875,
+    0.0023651123046875,
+    0.005126953125,
+    0.006927490234375,
+    0.0079345703125,
+    0.01177978515625,
+    0.012939453125,
+    0.0133056640625,
+    0.01422119140625,
+    0.035400390625,
+    0.050048828125,
+    0.07470703125,
+    0.0791015625,
+    0.09521484375,
+    0.140625,
+    0.19921875,
+    0.2421875,
+    0.40625,
+    0.3515625,
+    0.578125,
+    0.71484375,
+    0.80078125,
+    1.078125,
+    1.3203125,
+    1.1484375,
+    1.453125,
+    1.8984375,
+    3.171875,
+    2.921875,
+    2.890625,
+    2.421875,
+    3.5,
+    1.7265625,
+    1.2734375
+  ],
+  "talkie_embed_skips": [
+    -0.39453125,
+    -0.34765625,
+    -0.255859375,
+    0.00020503997802734375,
+    0.0003719329833984375,
+    0.0002765655517578125,
+    0.000347137451171875,
+    0.00049591064453125,
+    0.000507354736328125,
+    0.0009765625,
+    0.00063323974609375,
+    0.0002574920654296875,
+    0.000606536865234375,
+    0.0016021728515625,
+    0.00153350830078125,
+    0.00183868408203125,
+    0.00182342529296875,
+    0.00194549560546875,
+    0.0011444091796875,
+    -0.00213623046875,
+    0.0023651123046875,
+    0.01068115234375,
+    0.00982666015625,
+    0.006317138671875,
+    0.004425048828125,
+    0.0302734375,
+    0.037353515625,
+    -0.00811767578125,
+    -0.01055908203125,
+    0.003814697265625,
+    0.0289306640625,
+    0.1513671875,
+    0.5859375,
+    -0.37109375,
+    -0.035400390625,
+    -0.021728515625,
+    0.2578125,
+    0.44921875,
+    -0.7265625,
+    0.01177978515625
+  ],
+  "talkie_embedding_skip": true,
+  "talkie_head_gains": [
+    [
+      0.9921875,
+      1.0859375,
+      1.28125,
+      1.0859375,
+      1.21875,
+      1.234375,
+      0.84375,
+      1.3125,
+      1.1171875,
+      1.2265625,
+      1.015625,
+      1.390625,
+      1.3828125,
+      0.93359375,
+      1.359375,
+      0.89453125,
+      1.25,
+      0.96484375,
+      0.99609375,
+      1.1640625,
+      1.2265625,
+      1.0078125,
+      1.1328125,
+      1.1640625,
+      1.109375,
+      1.0078125,
+      1.1640625,
+      1.15625,
+      1.046875,
+      1.15625,
+      1.15625,
+      1.453125,
+      0.91015625,
+      1.8515625,
+      0.50390625,
+      1.09375,
+      1.0,
+      1.34375,
+      0.92578125,
+      1.46875
+    ],
+    [
+      1.125,
+      1.1640625,
+      1.6875,
+      1.09375,
+      1.109375,
+      2.078125,
+      1.1484375,
+      0.76953125,
+      1.03125,
+      1.03125,
+      1.3359375,
+      1.0234375,
+      1.125,
+      1.171875,
+      2.921875,
+      0.98828125,
+      1.609375,
+      2.03125,
+      0.921875,
+      1.046875,
+      1.21875,
+      1.296875,
+      1.2421875,
+      2.21875,
+      1.921875,
+      1.4375,
+      1.453125,
+      0.83203125,
+      1.453125,
+      1.3359375,
+      1.109375,
+      1.8359375,
+      0.94921875,
+      1.1640625,
+      1.4921875,
+      1.8828125,
+      1.90625,
+      1.21875,
+      1.2890625,
+      1.1640625
+    ],
+    [
+      1.6953125,
+      2.109375,
+      1.296875,
+      0.8984375,
+      1.3671875,
+      1.6875,
+      1.1484375,
+      1.1015625,
+      0.796875,
+      0.91015625,
+      1.15625,
+      1.3828125,
+      0.99609375,
+      3.125,
+      1.1796875,
+      2.0,
+      1.0703125,
+      1.078125,
+      0.73828125,
+      1.0078125,
+      2.34375,
+      1.1796875,
+      1.8359375,
+      2.375,
+      1.65625,
+      1.15625,
+      1.890625,
+      1.4921875,
+      0.87890625,
+      0.94140625,
+      0.80078125,
+      0.8515625,
+      1.3671875,
+      1.2421875,
+      1.125,
+      1.0546875,
+      1.515625,
+      1.625,
+      2.578125,
+      2.296875
+    ],
+    [
+      2.1875,
+      1.0625,
+      1.796875,
+      1.4296875,
+      1.6328125,
+      0.98828125,
+      1.5,
+      1.4921875,
+      1.3984375,
+      3.015625,
+      1.0703125,
+      1.296875,
+      1.125,
+      1.2890625,
+      1.6875,
+      1.765625,
+      1.2421875,
+      1.1015625,
+      1.6796875,
+      3.015625,
+      1.2421875,
+      0.97265625,
+      2.1875,
+      1.2109375,
+      1.390625,
+      1.390625,
+      1.171875,
+      2.890625,
+      1.7421875,
+      1.578125,
+      1.625,
+      1.421875,
+      1.2109375,
+      1.3203125,
+      2.78125,
+      2.09375,
+      1.7890625,
+      1.4296875,
+      2.109375,
+      2.15625
+    ],
+    [
+      1.3828125,
+      1.796875,
+      2.03125,
+      2.28125,
+      1.09375,
+      1.203125,
+      1.1015625,
+      1.609375,
+      2.109375,
+      1.2421875,
+      1.7265625,
+      1.078125,
+      1.3515625,
+      1.265625,
+      1.1328125,
+      2.15625,
+      2.078125,
+      1.5546875,
+      1.6015625,
+      2.546875,
+      1.1328125,
+      1.171875,
+      1.3046875,
+      1.6484375,
+      1.15625,
+      1.609375,
+      1.0625,
+      0.88671875,
+      1.375,
+      1.6953125,
+      2.796875,
+      1.59375,
+      1.7109375,
+      0.99609375,
+      1.0,
+      1.75,
+      1.515625,
+      2.140625,
+      1.7890625,
+      1.40625
+    ],
+    [
+      1.9375,
+      2.46875,
+      1.6015625,
+      1.6015625,
+      0.8828125,
+      1.1875,
+      2.03125,
+      1.8515625,
+      1.40625,
+      3.3125,
+      1.2890625,
+      1.7578125,
+      2.59375,
+      1.9921875,
+      1.8984375,
+      1.5625,
+      1.9765625,
+      2.09375,
+      1.765625,
+      1.0703125,
+      2.484375,
+      2.140625,
+      2.0625,
+      1.546875,
+      1.4921875,
+      1.359375,
+      1.46875,
+      1.515625,
+      1.5390625,
+      1.40625,
+      1.6875,
+      1.53125,
+      1.921875,
+      1.921875,
+      2.140625,
+      1.140625,
+      1.53125,
+      1.4765625,
+      1.8984375,
+      1.6171875
+    ],
+    [
+      1.6953125,
+      2.296875,
+      1.6953125,
+      1.4140625,
+      1.484375,
+      1.984375,
+      1.4375,
+      2.15625,
+      1.6953125,
+      1.4921875,
+      1.46875,
+      2.625,
+      1.6484375,
+      1.609375,
+      1.421875,
+      1.640625,
+      1.109375,
+      1.609375,
+      1.8125,
+      2.1875,
+      1.8828125,
+      1.3203125,
+      2.46875,
+      1.71875,
+      1.9609375,
+      1.8984375,
+      1.6484375,
+      1.8671875,
+      2.3125,
+      2.21875,
+      1.7890625,
+      1.9921875,
+      1.2578125,
+      2.359375,
+      1.8359375,
+      1.0234375,
+      1.8828125,
+      2.125,
+      2.015625,
+      2.40625
+    ],
+    [
+      1.234375,
+      1.53125,
+      1.96875,
+      1.46875,
+      1.5703125,
+      1.421875,
+      1.9375,
+      1.8828125,
+      1.65625,
+      1.1796875,
+      1.8046875,
+      1.4453125,
+      1.59375,
+      2.765625,
+      1.453125,
+      2.03125,
+      1.4140625,
+      1.2734375,
+      1.65625,
+      1.734375,
+      1.28125,
+      1.59375,
+      1.8359375,
+      1.1875,
+      1.921875,
+      2.0,
+      1.4375,
+      2.03125,
+      2.046875,
+      1.515625,
+      1.859375,
+      1.6484375,
+      1.28125,
+      2.0625,
+      1.8515625,
+      1.359375,
+      1.4765625,
+      1.90625,
+      1.6015625,
+      1.1875
+    ],
+    [
+      1.390625,
+      2.015625,
+      1.890625,
+      1.515625,
+      2.0625,
+      1.359375,
+      1.6015625,
+      1.2578125,
+      1.2265625,
+      1.4375,
+      1.9296875,
+      2.09375,
+      1.7421875,
+      1.5,
+      1.296875,
+      1.9921875,
+      1.765625,
+      1.734375,
+      1.96875,
+      2.265625,
+      1.7109375,
+      1.2890625,
+      2.234375,
+      1.6796875,
+      1.203125,
+      1.7890625,
+      1.2109375,
+      1.421875,
+      1.8046875,
+      1.484375,
+      2.046875,
+      1.4765625,
+      1.9296875,
+      1.046875,
+      1.53125,
+      1.5,
+      1.265625,
+      1.109375,
+      1.5078125,
+      1.8203125
+    ],
+    [
+      2.25,
+      1.7578125,
+      2.15625,
+      2.203125,
+      1.1875,
+      2.28125,
+      1.0390625,
+      3.3125,
+      2.109375,
+      2.171875,
+      1.4765625,
+      2.09375,
+      1.6875,
+      1.453125,
+      1.6015625,
+      2.140625,
+      1.6953125,
+      1.7734375,
+      1.75,
+      1.6953125,
+      1.6015625,
+      1.5,
+      1.0390625,
+      1.4609375,
+      1.6640625,
+      1.9453125,
+      1.6796875,
+      2.125,
+      1.3046875,
+      1.5,
+      1.34375,
+      2.1875,
+      1.703125,
+      1.7109375,
+      1.765625,
+      1.4921875,
+      1.84375,
+      2.03125,
+      1.46875,
+      1.359375
+    ],
+    [
+      1.421875,
+      1.5234375,
+      1.84375,
+      1.9140625,
+      1.59375,
+      1.90625,
+      2.46875,
+      1.40625,
+      1.2734375,
+      1.453125,
+      2.03125,
+      1.6953125,
+      1.875,
+      1.203125,
+      1.5390625,
+      2.25,
+      1.84375,
+      2.140625,
+      1.0390625,
+      1.796875,
+      2.125,
+      2.140625,
+      1.828125,
+      1.90625,
+      2.03125,
+      1.7421875,
+      1.71875,
+      1.171875,
+      1.9765625,
+      1.5859375,
+      1.4921875,
+      2.15625,
+      2.953125,
+      2.3125,
+      1.1796875,
+      1.609375,
+      1.390625,
+      2.0,
+      1.4453125,
+      1.4375
+    ],
+    [
+      1.9609375,
+      1.703125,
+      2.171875,
+      1.3671875,
+      1.3125,
+      1.6875,
+      1.234375,
+      1.65625,
+      1.0859375,
+      1.328125,
+      1.3984375,
+      1.875,
+      2.0,
+      2.171875,
+      2.109375,
+      1.9765625,
+      2.140625,
+      1.2109375,
+      2.1875,
+      1.8359375,
+      3.609375,
+      1.0,
+      2.984375,
+      1.4296875,
+      2.4375,
+      1.859375,
+      2.078125,
+      1.5,
+      2.34375,
+      1.9140625,
+      1.609375,
+      2.28125,
+      1.875,
+      1.609375,
+      1.65625,
+      2.15625,
+      1.375,
+      1.9609375,
+      2.078125,
+      1.9765625
+    ],
+    [
+      3.21875,
+      2.03125,
+      1.5,
+      1.7890625,
+      1.8359375,
+      2.0,
+      2.6875,
+      2.390625,
+      2.9375,
+      2.1875,
+      2.0625,
+      2.5625,
+      1.40625,
+      1.8359375,
+      1.3203125,
+      2.15625,
+      2.640625,
+      2.125,
+      2.109375,
+      2.3125,
+      1.7578125,
+      2.875,
+      1.9296875,
+      1.3984375,
+      2.0,
+      1.9921875,
+      1.3203125,
+      2.25,
+      2.296875,
+      2.109375,
+      2.296875,
+      1.984375,
+      2.21875,
+      1.3671875,
+      1.6953125,
+      1.859375,
+      1.671875,
+      2.140625,
+      1.0546875,
+      1.234375
+    ],
+    [
+      1.515625,
+      1.6640625,
+      1.9375,
+      2.265625,
+      1.59375,
+      1.9765625,
+      2.09375,
+      2.0625,
+      1.6015625,
+      2.09375,
+      1.1640625,
+      1.96875,
+      1.5703125,
+      1.453125,
+      3.203125,
+      1.71875,
+      1.4375,
+      1.15625,
+      2.734375,
+      2.15625,
+      2.0,
+      3.25,
+      3.5,
+      2.046875,
+      2.3125,
+      1.625,
+      2.09375,
+      2.046875,
+      1.3046875,
+      1.515625,
+      1.9609375,
+      2.1875,
+      1.21875,
+      3.25,
+      1.6015625,
+      1.953125,
+      1.1640625,
+      2.421875,
+      1.796875,
+      2.09375
+    ],
+    [
+      3.328125,
+      1.3359375,
+      3.171875,
+      1.515625,
+      2.25,
+      1.828125,
+      1.734375,
+      2.390625,
+      1.125,
+      1.5234375,
+      2.421875,
+      3.03125,
+      1.953125,
+      3.484375,
+      2.375,
+      1.375,
+      2.046875,
+      1.8828125,
+      1.859375,
+      1.4296875,
+      1.7734375,
+      1.453125,
+      2.359375,
+      2.28125,
+      1.4921875,
+      1.984375,
+      1.2421875,
+      2.203125,
+      2.421875,
+      1.390625,
+      1.8984375,
+      2.71875,
+      1.6640625,
+      1.4765625,
+      1.609375,
+      1.5,
+      1.421875,
+      2.359375,
+      1.25,
+      1.5390625
+    ],
+    [
+      1.5859375,
+      1.2890625,
+      3.125,
+      1.921875,
+      1.3671875,
+      1.8984375,
+      1.4140625,
+      1.5,
+      1.765625,
+      1.3671875,
+      2.640625,
+      2.390625,
+      2.84375,
+      1.5078125,
+      1.5546875,
+      1.515625,
+      2.5625,
+      2.296875,
+      2.921875,
+      2.015625,
+      1.9296875,
+      2.21875,
+      1.53125,
+      1.6953125,
+      2.078125,
+      2.078125,
+      2.109375,
+      2.609375,
+      2.359375,
+      1.21875,
+      1.703125,
+      1.78125,
+      2.25,
+      1.4453125,
+      1.4140625,
+      2.3125,
+      2.125,
+      2.546875,
+      1.859375,
+      2.03125
+    ],
+    [
+      2.15625,
+      2.0625,
+      1.546875,
+      1.8046875,
+      2.4375,
+      2.21875,
+      1.3515625,
+      2.59375,
+      2.234375,
+      1.703125,
+      1.8125,
+      2.53125,
+      2.21875,
+      2.4375,
+      2.75,
+      2.171875,
+      2.578125,
+      2.84375,
+      3.109375,
+      2.578125,
+      1.34375,
+      2.046875,
+      2.078125,
+      1.578125,
+      2.375,
+      1.96875,
+      3.328125,
+      1.3828125,
+      2.75,
+      1.546875,
+      2.671875,
+      1.7109375,
+      2.953125,
+      2.5625,
+      2.0625,
+      1.6796875,
+      1.703125,
+      3.390625,
+      2.125,
+      2.28125
+    ],
+    [
+      2.203125,
+      1.5078125,
+      1.5390625,
+      1.390625,
+      1.7265625,
+      2.0,
+      2.09375,
+      1.875,
+      1.65625,
+      2.015625,
+      1.8671875,
+      2.3125,
+      1.6015625,
+      1.5703125,
+      1.7734375,
+      2.4375,
+      1.640625,
+      2.5,
+      1.609375,
+      2.078125,
+      1.65625,
+      1.8359375,
+      2.53125,
+      2.15625,
+      1.9765625,
+      1.828125,
+      1.921875,
+      2.328125,
+      2.734375,
+      2.078125,
+      2.3125,
+      2.46875,
+      2.34375,
+      2.78125,
+      1.5390625,
+      2.875,
+      2.328125,
+      1.46875,
+      1.453125,
+      2.28125
+    ],
+    [
+      2.84375,
+      2.328125,
+      1.5390625,
+      1.8828125,
+      2.6875,
+      1.7265625,
+      2.21875,
+      2.78125,
+      2.125,
+      2.28125,
+      1.6953125,
+      2.078125,
+      2.390625,
+      2.15625,
+      2.390625,
+      2.0625,
+      2.421875,
+      2.671875,
+      1.5234375,
+      2.046875,
+      1.9140625,
+      2.359375,
+      2.0625,
+      2.640625,
+      2.140625,
+      2.3125,
+      2.234375,
+      1.65625,
+      2.140625,
+      2.140625,
+      2.296875,
+      2.984375,
+      2.15625,
+      3.265625,
+      2.234375,
+      2.40625,
+      1.4609375,
+      1.734375,
+      1.8125,
+      1.6875
+    ],
+    [
+      1.8203125,
+      2.71875,
+      2.09375,
+      2.390625,
+      2.0625,
+      2.015625,
+      1.5859375,
+      1.734375,
+      3.0,
+      2.78125,
+      1.625,
+      2.40625,
+      2.265625,
+      2.171875,
+      3.359375,
+      2.703125,
+      2.03125,
+      3.203125,
+      1.671875,
+      2.859375,
+      1.96875,
+      2.328125,
+      1.5,
+      2.671875,
+      2.265625,
+      2.625,
+      2.421875,
+      2.6875,
+      2.546875,
+      2.3125,
+      2.265625,
+      1.4140625,
+      1.59375,
+      2.625,
+      1.5859375,
+      1.6875,
+      2.078125,
+      2.421875,
+      1.9921875,
+      1.6484375
+    ],
+    [
+      1.875,
+      2.34375,
+      1.5703125,
+      2.09375,
+      2.828125,
+      3.609375,
+      1.90625,
+      2.828125,
+      2.734375,
+      2.765625,
+      3.078125,
+      2.484375,
+      2.1875,
+      2.53125,
+      1.859375,
+      2.703125,
+      1.9375,
+      2.484375,
+      2.140625,
+      2.28125,
+      2.953125,
+      2.109375,
+      2.25,
+      2.3125,
+      2.109375,
+      1.53125,
+      2.015625,
+      1.9296875,
+      2.40625,
+      1.6640625,
+      2.359375,
+      1.8125,
+      1.859375,
+      2.671875,
+      1.75,
+      1.5546875,
+      1.921875,
+      2.34375,
+      2.296875,
+      2.359375
+    ],
+    [
+      2.21875,
+      1.5703125,
+      2.703125,
+      3.0,
+      2.84375,
+      1.7421875,
+      2.84375,
+      2.59375,
+      1.8046875,
+      2.234375,
+      2.515625,
+      2.328125,
+      1.9765625,
+      2.609375,
+      1.84375,
+      2.390625,
+      2.4375,
+      2.75,
+      2.625,
+      2.140625,
+      1.8046875,
+      2.0,
+      2.53125,
+      1.7578125,
+      2.09375,
+      1.640625,
+      2.53125,
+      2.28125,
+      2.296875,
+      2.109375,
+      2.140625,
+      2.390625,
+      2.0,
+      1.421875,
+      2.234375,
+      2.25,
+      1.8671875,
+      1.5859375,
+      2.359375,
+      1.8046875
+    ],
+    [
+      1.953125,
+      2.09375,
+      3.25,
+      2.09375,
+      2.46875,
+      2.515625,
+      1.9921875,
+      1.6328125,
+      2.34375,
+      2.484375,
+      2.21875,
+      2.078125,
+      2.546875,
+      2.578125,
+      2.265625,
+      2.265625,
+      2.0,
+      2.890625,
+      2.015625,
+      3.03125,
+      2.09375,
+      2.484375,
+      1.9765625,
+      1.5625,
+      2.703125,
+      1.40625,
+      1.765625,
+      1.890625,
+      2.328125,
+      1.6640625,
+      1.625,
+      2.46875,
+      2.515625,
+      2.109375,
+      2.5625,
+      1.8046875,
+      1.8515625,
+      2.546875,
+      1.8125,
+      2.3125
+    ],
+    [
+      2.65625,
+      2.375,
+      2.53125,
+      2.5,
+      2.25,
+      2.53125,
+      2.03125,
+      3.15625,
+      2.03125,
+      3.125,
+      2.3125,
+      2.375,
+      1.734375,
+      2.390625,
+      1.4140625,
+      2.390625,
+      2.65625,
+      2.328125,
+      2.5625,
+      2.1875,
+      2.859375,
+      2.140625,
+      2.671875,
+      1.921875,
+      2.265625,
+      2.515625,
+      2.875,
+      2.578125,
+      2.6875,
+      2.640625,
+      1.8203125,
+      3.203125,
+      2.265625,
+      2.71875,
+      3.296875,
+      2.875,
+      2.5,
+      1.953125,
+      2.71875,
+      2.03125
+    ],
+    [
+      2.390625,
+      2.234375,
+      2.5,
+      1.9296875,
+      3.5,
+      2.984375,
+      1.84375,
+      2.6875,
+      2.4375,
+      1.75,
+      2.859375,
+      2.265625,
+      2.75,
+      2.03125,
+      2.921875,
+      2.796875,
+      2.265625,
+      2.515625,
+      2.671875,
+      1.9375,
+      2.421875,
+      2.390625,
+      3.734375,
+      1.953125,
+      1.9140625,
+      1.609375,
+      2.546875,
+      1.3203125,
+      3.125,
+      2.671875,
+      2.234375,
+      1.8671875,
+      1.8046875,
+      2.34375,
+      3.390625,
+      1.953125,
+      2.0625,
+      2.03125,
+      1.34375,
+      1.6953125
+    ],
+    [
+      2.28125,
+      2.6875,
+      2.28125,
+      3.046875,
+      2.515625,
+      2.5625,
+      2.609375,
+      2.84375,
+      2.65625,
+      1.796875,
+      2.75,
+      2.484375,
+      2.890625,
+      1.296875,
+      2.859375,
+      3.34375,
+      1.71875,
+      2.6875,
+      2.234375,
+      1.984375,
+      2.0,
+      1.984375,
+      2.375,
+      2.96875,
+      1.796875,
+      2.796875,
+      3.171875,
+      2.4375,
+      1.9765625,
+      2.265625,
+      1.8359375,
+      2.40625,
+      2.5,
+      2.71875,
+      1.9765625,
+      2.59375,
+      2.21875,
+      2.640625,
+      2.53125,
+      3.078125
+    ],
+    [
+      2.453125,
+      2.203125,
+      2.9375,
+      1.40625,
+      2.953125,
+      2.828125,
+      2.5625,
+      3.03125,
+      2.40625,
+      3.015625,
+      2.0,
+      2.265625,
+      2.8125,
+      2.421875,
+      1.6953125,
+      1.96875,
+      2.203125,
+      2.5625,
+      2.171875,
+      2.59375,
+      1.140625,
+      2.515625,
+      2.46875,
+      1.78125,
+      2.3125,
+      2.921875,
+      2.90625,
+      1.8671875,
+      1.9609375,
+      3.15625,
+      3.28125,
+      2.609375,
+      3.03125,
+      3.3125,
+      1.9140625,
+      2.53125,
+      3.234375,
+      1.6015625,
+      2.34375,
+      2.1875
+    ],
+    [
+      1.875,
+      2.96875,
+      2.578125,
+      2.703125,
+      2.453125,
+      2.5,
+      3.03125,
+      2.71875,
+      2.84375,
+      2.25,
+      3.15625,
+      2.0,
+      2.53125,
+      2.234375,
+      1.6484375,
+      2.5,
+      1.6328125,
+      2.3125,
+      1.6796875,
+      2.578125,
+      2.234375,
+      2.5625,
+      1.84375,
+      3.03125,
+      1.59375,
+      2.734375,
+      3.09375,
+      1.984375,
+      2.5625,
+      2.359375,
+      2.15625,
+      2.546875,
+      2.84375,
+      2.125,
+      1.8671875,
+      2.203125,
+      1.828125,
+      2.125,
+      2.125,
+      2.234375
+    ],
+    [
+      2.640625,
+      1.9765625,
+      2.546875,
+      1.84375,
+      2.140625,
+      2.875,
+      2.15625,
+      2.296875,
+      2.265625,
+      1.84375,
+      3.0,
+      2.03125,
+      2.75,
+      2.328125,
+      2.9375,
+      2.484375,
+      2.078125,
+      2.21875,
+      2.484375,
+      2.265625,
+      2.1875,
+      1.5390625,
+      1.953125,
+      2.296875,
+      2.390625,
+      1.46875,
+      2.8125,
+      2.828125,
+      1.65625,
+      3.296875,
+      1.5703125,
+      2.40625,
+      2.015625,
+      1.765625,
+      2.234375,
+      2.40625,
+      2.5625,
+      2.21875,
+      2.078125,
+      3.125
+    ],
+    [
+      1.7265625,
+      2.3125,
+      1.9765625,
+      2.875,
+      2.171875,
+      2.34375,
+      2.046875,
+      2.3125,
+      1.53125,
+      2.046875,
+      2.6875,
+      1.7265625,
+      1.9453125,
+      1.8203125,
+      2.265625,
+      2.515625,
+      2.109375,
+      2.453125,
+      1.875,
+      2.078125,
+      2.03125,
+      2.40625,
+      2.46875,
+      2.0625,
+      3.0625,
+      1.8203125,
+      2.34375,
+      2.34375,
+      2.0625,
+      2.390625,
+      2.046875,
+      2.15625,
+      2.1875,
+      2.0625,
+      2.09375,
+      2.171875,
+      2.375,
+      2.171875,
+      1.875,
+      1.9921875
+    ],
+    [
+      1.9296875,
+      2.234375,
+      2.046875,
+      1.578125,
+      2.296875,
+      1.671875,
+      1.8671875,
+      2.546875,
+      2.359375,
+      2.40625,
+      2.25,
+      1.546875,
+      2.578125,
+      2.125,
+      2.4375,
+      2.03125,
+      2.34375,
+      1.765625,
+      2.171875,
+      1.734375,
+      2.4375,
+      1.4453125,
+      2.25,
+      2.578125,
+      1.9921875,
+      2.6875,
+      1.9609375,
+      1.6796875,
+      1.9375,
+      2.46875,
+      2.296875,
+      2.015625,
+      2.515625,
+      2.140625,
+      2.28125,
+      2.34375,
+      2.21875,
+      1.8984375,
+      2.0,
+      1.859375
+    ],
+    [
+      2.3125,
+      2.484375,
+      2.09375,
+      2.328125,
+      2.671875,
+      2.484375,
+      2.609375,
+      2.015625,
+      2.328125,
+      2.40625,
+      2.078125,
+      1.5859375,
+      1.6484375,
+      2.265625,
+      2.0,
+      2.671875,
+      1.9296875,
+      2.40625,
+      1.859375,
+      2.828125,
+      2.0,
+      1.9921875,
+      1.953125,
+      1.9921875,
+      2.421875,
+      2.359375,
+      2.4375,
+      2.078125,
+      1.515625,
+      2.234375,
+      2.0,
+      1.96875,
+      2.03125,
+      1.71875,
+      2.046875,
+      2.03125,
+      2.0,
+      2.4375,
+      2.375,
+      2.265625
+    ],
+    [
+      2.25,
+      2.21875,
+      2.71875,
+      2.359375,
+      2.671875,
+      1.8046875,
+      2.53125,
+      2.484375,
+      1.6796875,
+      2.046875,
+      2.28125,
+      1.8515625,
+      2.234375,
+      2.296875,
+      2.046875,
+      2.515625,
+      2.203125,
+      2.390625,
+      3.03125,
+      1.9453125,
+      2.453125,
+      2.234375,
+      1.734375,
+      2.5625,
+      2.359375,
+      2.140625,
+      2.34375,
+      2.1875,
+      2.453125,
+      1.9296875,
+      2.4375,
+      2.171875,
+      1.8828125,
+      2.3125,
+      2.234375,
+      2.21875,
+      1.9765625,
+      2.359375,
+      1.859375,
+      2.3125
+    ],
+    [
+      2.25,
+      2.21875,
+      1.90625,
+      2.21875,
+      2.296875,
+      2.40625,
+      1.84375,
+      2.109375,
+      2.421875,
+      2.21875,
+      2.5625,
+      1.6953125,
+      2.40625,
+      2.5,
+      2.28125,
+      2.484375,
+      2.875,
+      2.8125,
+      2.15625,
+      2.34375,
+      2.15625,
+      2.5,
+      2.359375,
+      1.8125,
+      2.453125,
+      1.75,
+      2.15625,
+      1.8984375,
+      2.375,
+      2.6875,
+      2.3125,
+      1.9453125,
+      1.9609375,
+      2.390625,
+      2.109375,
+      2.484375,
+      2.859375,
+      2.25,
+      2.21875,
+      1.890625
+    ],
+    [
+      2.390625,
+      2.296875,
+      1.7578125,
+      2.078125,
+      2.453125,
+      2.484375,
+      1.9921875,
+      2.34375,
+      2.09375,
+      2.6875,
+      2.296875,
+      2.3125,
+      2.421875,
+      1.90625,
+      2.09375,
+      2.421875,
+      2.578125,
+      1.8828125,
+      2.765625,
+      2.375,
+      2.796875,
+      2.328125,
+      2.8125,
+      2.546875,
+      2.640625,
+      2.375,
+      2.25,
+      2.75,
+      2.5,
+      2.171875,
+      2.203125,
+      2.34375,
+      2.234375,
+      2.5,
+      1.9296875,
+      1.7578125,
+      1.859375,
+      2.265625,
+      2.671875,
+      2.65625
+    ],
+    [
+      1.640625,
+      1.96875,
+      1.71875,
+      2.15625,
+      2.375,
+      1.890625,
+      2.0625,
+      2.1875,
+      2.75,
+      2.125,
+      1.9765625,
+      2.046875,
+      1.9140625,
+      2.84375,
+      1.8515625,
+      1.984375,
+      3.34375,
+      1.8125,
+      2.375,
+      2.046875,
+      1.765625,
+      1.7421875,
+      1.9921875,
+      2.75,
+      2.0625,
+      2.1875,
+      1.7734375,
+      1.765625,
+      1.8359375,
+      2.234375,
+      1.7265625,
+      2.34375,
+      2.546875,
+      2.1875,
+      1.859375,
+      1.4921875,
+      2.484375,
+      1.9453125,
+      1.65625,
+      2.03125
+    ],
+    [
+      1.6484375,
+      1.890625,
+      1.8125,
+      1.625,
+      1.640625,
+      2.109375,
+      1.875,
+      1.890625,
+      2.15625,
+      2.21875,
+      2.4375,
+      1.890625,
+      2.125,
+      1.796875,
+      2.0,
+      1.9921875,
+      1.59375,
+      2.078125,
+      1.9140625,
+      1.8515625,
+      1.921875,
+      1.796875,
+      1.59375,
+      1.7578125,
+      2.171875,
+      2.125,
+      1.5390625,
+      1.9921875,
+      1.921875,
+      3.09375,
+      2.4375,
+      1.6484375,
+      2.0,
+      2.359375,
+      1.984375,
+      2.53125,
+      1.8515625,
+      1.4921875,
+      1.7421875,
+      2.046875
+    ],
+    [
+      2.09375,
+      1.78125,
+      1.9765625,
+      1.7578125,
+      1.90625,
+      1.96875,
+      2.78125,
+      2.28125,
+      2.546875,
+      3.15625,
+      1.9609375,
+      1.796875,
+      1.8125,
+      1.6796875,
+      1.84375,
+      2.15625,
+      2.453125,
+      1.8671875,
+      1.671875,
+      1.734375,
+      1.8515625,
+      1.7734375,
+      2.515625,
+      1.796875,
+      1.8671875,
+      1.6640625,
+      2.84375,
+      2.109375,
+      2.828125,
+      2.1875,
+      2.125,
+      1.984375,
+      1.8359375,
+      1.5859375,
+      2.0,
+      2.15625,
+      2.078125,
+      2.515625,
+      1.6015625,
+      1.953125
+    ],
+    [
+      2.125,
+      1.9765625,
+      2.234375,
+      1.4453125,
+      2.5625,
+      1.8515625,
+      3.0,
+      3.0625,
+      2.5,
+      1.9453125,
+      1.7734375,
+      1.6953125,
+      1.9140625,
+      1.53125,
+      2.109375,
+      2.59375,
+      1.5234375,
+      1.671875,
+      1.453125,
+      1.8984375,
+      2.78125,
+      1.546875,
+      2.03125,
+      2.5,
+      1.5703125,
+      2.640625,
+      1.5859375,
+      1.359375,
+      1.6328125,
+      1.2734375,
+      2.0,
+      2.421875,
+      2.09375,
+      2.140625,
+      2.0625,
+      1.6484375,
+      1.703125,
+      1.9296875,
+      1.8515625,
+      2.09375
+    ],
+    [
+      1.5078125,
+      2.21875,
+      2.625,
+      2.25,
+      1.6484375,
+      1.734375,
+      1.84375,
+      1.5078125,
+      2.171875,
+      1.828125,
+      1.5625,
+      3.265625,
+      2.15625,
+      2.828125,
+      1.5703125,
+      1.890625,
+      1.234375,
+      2.328125,
+      3.328125,
+      1.5703125,
+      1.6875,
+      2.75,
+      1.9609375,
+      1.5390625,
+      2.125,
+      2.5,
+      1.8984375,
+      1.734375,
+      1.5703125,
+      1.390625,
+      2.078125,
+      1.3828125,
+      1.46875,
+      1.5234375,
+      1.859375,
+      1.5234375,
+      1.7734375,
+      2.078125,
+      1.515625,
+      1.7421875
+    ]
+  ],
+  "talkie_initial_rms_norm": true,
+  "talkie_mlp_gains": [
+    0.00023651123046875,
+    0.000370025634765625,
+    0.00018978118896484375,
+    0.000621795654296875,
+    0.00182342529296875,
+    0.0021209716796875,
+    0.0025177001953125,
+    0.0032196044921875,
+    0.005523681640625,
+    0.00958251953125,
+    0.01141357421875,
+    0.01495361328125,
+    0.01806640625,
+    0.0211181640625,
+    0.0238037109375,
+    0.0322265625,
+    0.04150390625,
+    0.057373046875,
+    0.0732421875,
+    0.0947265625,
+    0.12109375,
+    0.1533203125,
+    0.1875,
+    0.236328125,
+    0.296875,
+    0.36328125,
+    0.484375,
+    0.60546875,
+    0.78125,
+    0.99609375,
+    1.1953125,
+    1.4140625,
+    1.6015625,
+    1.7578125,
+    2.125,
+    2.59375,
+    3.0,
+    3.09375,
+    2.90625,
+    2.421875
+  ],
+  "talkie_qwen3_compat": true,
+  "talkie_rope_sign": -1,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 65540
+}

model-00001-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8512208b2067cfcda3c0275bf240e786fd7f81508dd16c993f3ee82ec56cda28
+size 1342269760

model-00002-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2ee15d1392357d7473dc57033a683c4ce8fb3569607f1fe7dd2e1b467854f72
+size 630478584

model-00003-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce4acc43c335214c0399a468b2d5a7aba692214c1cdd01499bb5b7d7d49379fe
+size 630478584

model-00004-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cc11db4dd8fb3a4221f3f153a9b62172d5e296e221ed5354e11121785672580
+size 630478584

model-00005-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb1f4c7109cd700c079b909dd108e38e575f1732806c10b91843cb27f00d7593
+size 630478584

model-00006-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d996914f4dc0581b27865e25a5589a2d7c38ed849a05e9a2ec875a2f51df4a71
+size 630478584

model-00007-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62b2e90ac9b4c8497d0348255599f337fdba10ddb4410f2ebbd8cbafbd2abf8a
+size 630478584

model-00008-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b1322be02140144a937ef3dd303e73d082fb62d018fc6902b03a50d0bd8d9a4
+size 630478584

model-00009-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13024cdbedc692771e64e85362767253f875bda77e71f0d058dc7cc5ac09ff69
+size 630478584

model-00010-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5637aa9412a0b4aed2f5d7f40b555bea9d440a6cd2071b64b4830d4e55ff6f82
+size 630478584

model-00011-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:423fc40ec8afa777bf0dfc232dbf1cfabf7e8006892896ba3ed77f8b17f57cc7
+size 630478584

model-00012-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2ef2ce597d324bde508d2627678eba2fbe33ee357131e2d6f2926901cd40f15
+size 630478592

model-00013-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2a09a7bfb05781d6193cafa63a354d1917fbd8e5b42ecebe17ad7e4aa1a7014
+size 630478592

model-00014-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78b69e25befa8cc80bea65910ac24fdfa6fdf27bce8147c02b4305ebab0614d0
+size 630478592

model-00015-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d0b486773c1911cdf71bd65bb908e81e311665511ea816af7cfbb72fcf1b47b
+size 630478592

model-00016-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4db095940b355888d2581cb4dfba96b6121d4f7a89a18ba39ab1175ca633f439
+size 630478592

model-00017-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c756394e75aeed87c9a53a006e2be332da0ef3b5a3ddd825d4af71ec61c2f6bb
+size 630478592

model-00018-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aca83c53bbcffaf68fa880eae340d87859c0596e3e6075c1712c60f95b6ee069
+size 630478592

model-00019-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd377b97d06d39fd704a903763d4b1d67e890153a8f158344fffc1b3f15f16f9
+size 630478592

model-00020-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2436675e1c36b443b04622ed31395932a532f14be6f3c7dd1a9de9f71e11e1f4
+size 630478592

model-00021-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c4922babafd0da845135cbf61a948fd0a2145e5dac09117963a1fb6e8a46b34
+size 630478592

model-00022-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28f82dc3115d416697b7511ad6eec32d6a665b25c1458fbc33b311ce5d6de894
+size 630478592

model-00023-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09a96f70d3d71dc6991cf95c3a1970d25bbc74d4a3d99730fe3c31030a02678e
+size 630478592

model-00024-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed3761646fd806ec1cdbbc1f9dcbfd0fa53c1d809fe7f9cefca54c68615c8018
+size 630478592

model-00025-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7505959615283aefc2d4c00d59256efb3a0590630ceaec3455b6f7c03e33725
+size 630478592

model-00026-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b48a5c7d44014039e3a0d5e312d96308384c3a8b861e84d32853b51a58dc20d
+size 630478592

model-00027-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c9b7ae41f8647c214ef00e190d1bc88402e242b8166b0747bc63a4f9c55386
+size 630478592

model-00028-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4441050cd41c96df8f90ff58679a8f9b9fc2c6f57efb3160b32c0b542c0b1d33
+size 630478592

model-00029-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2ce37acc59120e0cf0cb999650c8995f1acf46a429cab9a902baf9457741b0d
+size 630478592

model-00030-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6331e6fd2152a95c822aaa6e187b4617294c60496822d975a539bc3354631b56
+size 630478592

model-00031-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8f6842baa83cc134b8cff9a753d2d5c90c3a544576332f20cd576e5310f84d5
+size 630478592

model-00032-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:603b2a08c9aa8dbb005cb703b32dcc703800f3b059d1b7ae001f60b7bb4f324a
+size 630478592

model-00033-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d53f8868278b7c64aac116a67a3cac592bbd1ef303bf34a335950994b6fba250
+size 630478592

model-00034-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d3703f2cee07b0c0eb1bd92eb1e0a44f7d6476dca0c91ddbc7965a9bad9f0c8
+size 630478592

model-00035-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a5ba5e75005455a8f72fe498c331bda72984f10094d10cf2a0690457e3ef927
+size 630478592

model-00036-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff9758f19f4cdcd1e99cc8c45a49d9051ce1812eef5ae2043e0a3e84191da3ec
+size 630478592

model-00037-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3176dabddbc910658d524141df1fda174b3d7fd66ecd8feb0a3351922de8c640
+size 630478592

model-00038-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6866c3e9f4cb0bcef8e774569218acf818324558c3ce9e01c4ab667b6a6884bf
+size 630478592

model-00039-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56622fdd98c7b6d1b9062c5c5e71fb2db90a23e737819a75f134967c27ee37cc
+size 630478592

model-00040-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c224b47639b64817f4fc51e23331cbf8f7541d6724965e46f613b56c72f3fbcb
+size 630478592

model-00041-of-00041.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9be3296315fd70d6833f4b2c1459737afd317cb27451ebd56dcc0ee06497a63b
+size 630478592

model.safetensors.index.json

@@ -0,0 +1,450 @@
+{
+  "metadata": {
+    "total_size": 26561361920
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00001-of-00041.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00041.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00002-of-00041.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00002-of-00041.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00003-of-00041.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00003-of-00041.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00012-of-00041.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00012-of-00041.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00013-of-00041.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00013-of-00041.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00014-of-00041.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00014-of-00041.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00015-of-00041.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00015-of-00041.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00016-of-00041.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00016-of-00041.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00017-of-00041.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00017-of-00041.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00018-of-00041.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00018-of-00041.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00019-of-00041.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00019-of-00041.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00020-of-00041.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00020-of-00041.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00021-of-00041.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00021-of-00041.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00004-of-00041.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00004-of-00041.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00022-of-00041.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00022-of-00041.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00023-of-00041.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00023-of-00041.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00024-of-00041.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00024-of-00041.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00025-of-00041.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00025-of-00041.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00026-of-00041.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00026-of-00041.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00027-of-00041.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00027-of-00041.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00028-of-00041.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00028-of-00041.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00029-of-00041.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00029-of-00041.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00030-of-00041.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00030-of-00041.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00031-of-00041.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00031-of-00041.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00005-of-00041.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00005-of-00041.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00032-of-00041.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00032-of-00041.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00033-of-00041.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00033-of-00041.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00034-of-00041.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00034-of-00041.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00035-of-00041.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00035-of-00041.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00036-of-00041.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00036-of-00041.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00037-of-00041.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00037-of-00041.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.self_attn.k_norm.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.self_attn.q_norm.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00038-of-00041.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00038-of-00041.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.self_attn.k_norm.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.self_attn.q_norm.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00039-of-00041.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00039-of-00041.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.self_attn.k_norm.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.self_attn.q_norm.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00040-of-00041.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00040-of-00041.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.self_attn.k_norm.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.self_attn.q_norm.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00041-of-00041.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00041-of-00041.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00006-of-00041.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00006-of-00041.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00007-of-00041.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00007-of-00041.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00008-of-00041.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00008-of-00041.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00009-of-00041.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00009-of-00041.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00010-of-00041.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00010-of-00041.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00011-of-00041.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00011-of-00041.safetensors",
+    "model.norm.weight": "model-00001-of-00041.safetensors"
+  }
+}

tokenization_talkie.py

@@ -0,0 +1,68 @@
+
+from __future__ import annotations
+
+from pathlib import Path
+import torch
+from transformers import PreTrainedTokenizer
+from talkie.chat import Message, format_chat
+from talkie.tokenizer import build_tokenizer
+
+
+class TalkieTokenizer(PreTrainedTokenizer):
+    model_input_names = ["input_ids", "attention_mask"]
+    vocab_files_names = {"vocab_file": "vocab.txt"}
+
+    def __init__(self, vocab_file, style="it", **kwargs):
+        self.vocab_file = str(vocab_file)
+        self.style = style
+        self.encoding = build_tokenizer(self.vocab_file, style=style)
+        eos = "<|end|>" if style == "it" else "<|endoftext|>"
+        kwargs.setdefault("eos_token", eos)
+        kwargs.setdefault("pad_token", eos)
+        super().__init__(**kwargs)
+
+    @property
+    def vocab_size(self):
+        return 65540 if self.style == "it" else 65536
+
+    def get_vocab(self):
+        return {str(i): i for i in range(self.vocab_size)}
+
+    def _tokenize(self, text, **kwargs):
+        return [str(i) for i in self.encoding.encode(text, allowed_special="all")]
+
+    def _convert_token_to_id(self, token):
+        return int(token)
+
+    def _convert_id_to_token(self, index):
+        return str(index)
+
+    def convert_tokens_to_string(self, tokens):
+        return self.encoding.decode([int(t) for t in tokens])
+
+    def encode(self, text, add_special_tokens=False, **kwargs):
+        return self.encoding.encode(text, allowed_special="all")
+
+    def decode(self, token_ids, skip_special_tokens=False, **kwargs):
+        if isinstance(token_ids, torch.Tensor):
+            token_ids = token_ids.tolist()
+        if skip_special_tokens:
+            specials = {self.encoding.encode_single_token(t) for t in self.encoding.special_tokens_set}
+            token_ids = [int(i) for i in token_ids if int(i) not in specials]
+        return self.encoding.decode([int(i) for i in token_ids])
+
+    def apply_chat_template(self, conversation, tokenize=False, add_generation_prompt=True, return_tensors=None, **kwargs):
+        messages = [m if isinstance(m, Message) else Message(m["role"], m["content"]) for m in conversation]
+        text = format_chat(messages)
+        if not add_generation_prompt and text.endswith("<|assistant|>"):
+            text = text[:-len("<|assistant|>")]
+        if not tokenize:
+            return text
+        ids = self.encode(text)
+        return torch.tensor([ids], dtype=torch.long) if return_tensors == "pt" else ids
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        out = Path(save_directory) / ((filename_prefix or "") + "vocab.txt")
+        if Path(self.vocab_file).resolve() != out.resolve():
+            out.write_bytes(Path(self.vocab_file).read_bytes())
+        return (str(out),)

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "tokenizer_class": "TalkieTokenizer",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_talkie.TalkieTokenizer",
+      null
+    ]
+  },
+  "vocab_file": "vocab.txt",
+  "style": "it",
+  "eos_token": "<|end|>",
+  "pad_token": "<|end|>"
+}