Upload folder using huggingface_hub

chat_template.jinja

@@ -0,0 +1,87 @@
+{%- if messages[0]["role"] == "system" %}
+    {%- set system_message = messages[0]["content"] %}
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set loop_messages = messages %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
+
+{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}
+{%- set ns = namespace() %}
+{%- set ns.index = 0 %}
+{%- for message in loop_messages %}
+    {%- if not (message.role == "tool" or message.role == "tool_results" or (message.tool_calls is defined and message.tool_calls is not none)) %}
+        {%- if (message["role"] == "user") != (ns.index % 2 == 0) %}
+            {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
+        {%- endif %}
+        {%- set ns.index = ns.index + 1 %}
+    {%- endif %}
+{%- endfor %}
+
+{{- bos_token }}
+{%- for message in loop_messages %}
+    {%- if message["role"] == "user" %}
+        {%- if tools is not none and (message == user_messages[-1]) %}
+            {{- "[AVAILABLE_TOOLS] [" }}
+            {%- for tool in tools %}
+                {%- set tool = tool.function %}
+                {{- '{"type": "function", "function": {' }}
+                {%- for key, val in tool.items() if key != "return" %}
+                    {%- if val is string %}
+                        {{- '"' + key + '": "' + val + '"' }}
+                    {%- else %}
+                        {{- '"' + key + '": ' + val|tojson }}
+                    {%- endif %}
+                    {%- if not loop.last %}
+                        {{- ", " }}
+                    {%- endif %}
+                {%- endfor %}
+                {{- "}}" }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- else %}
+                    {{- "]" }}
+                {%- endif %}
+            {%- endfor %}
+            {{- "[/AVAILABLE_TOOLS]" }}
+            {%- endif %}
+        {%- if loop.last and system_message is defined %}
+            {{- "[INST] " + system_message + "\n\n" + message["content"] + "[/INST]" }}
+        {%- else %}
+            {{- "[INST] " + message["content"] + "[/INST]" }}
+        {%- endif %}
+    {%- elif message.tool_calls is defined and message.tool_calls is not none %}
+        {{- "[TOOL_CALLS] [" }}
+        {%- for tool_call in message.tool_calls %}
+            {%- set out = tool_call.function|tojson %}
+            {{- out[:-1] }}
+            {%- if not tool_call.id is defined or tool_call.id|length != 9 %}
+                {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+            {%- endif %}
+            {{- ', "id": "' + tool_call.id + '"}' }}
+            {%- if not loop.last %}
+                {{- ", " }}
+            {%- else %}
+                {{- "]" + eos_token }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif message["role"] == "assistant" %}
+        {{- " " + message["content"]|trim + eos_token}}
+    {%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
+        {%- if message.content is defined and message.content.content is defined %}
+            {%- set content = message.content.content %}
+        {%- else %}
+            {%- set content = message.content %}
+        {%- endif %}
+        {{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }}
+        {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}
+            {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+        {%- endif %}
+        {{- '"call_id": "' + message.tool_call_id + '"}[/TOOL_RESULTS]' }}
+    {%- else %}
+        {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
+    {%- endif %}
+{%- endfor %}

config.json

@@ -0,0 +1,74 @@
+{
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "format": "gptq",
+    "group_size": 128,
+    "lm_head": false,
+    "meta": {
+      "act_group_aware": true,
+      "auto_forward_data_parallel": true,
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.05,
+      "fallback": {
+        "smooth": null,
+        "strategy": "rtn",
+        "threshold": "0.5%"
+      },
+      "foem": null,
+      "gc_mode": "interval",
+      "gptaq": null,
+      "hessian": {
+        "chunk_bytes": null,
+        "chunk_size": null,
+        "staging_dtype": "float32"
+      },
+      "mock_quantization": false,
+      "mse": 0.0,
+      "offload_to_disk": true,
+      "offload_to_disk_path": "./gptqmodel_offload/orcmjeui-tnqhqrfy/",
+      "pack_impl": "cpu",
+      "quantizer": [
+        "gptqmodel:6.0.3"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "vram_strategy": "exclusive",
+      "wait_for_submodule_finalizers": false
+    },
+    "method": "gptq",
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.3",
+  "use_cache": true,
+  "vocab_size": 32768
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "transformers_version": "5.5.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfcf75467cadbe45ff417cf8051d93164dd3f6545955085ad623b95c81adc9b6
+size 4168468336

quant_log.csv

@@ -0,0 +1,225 @@
+layer,module,loss,samples,damp,time
+0,self_attn.k_proj,0.0000025754,0.05000,4.550
+0,self_attn.v_proj,0.0000002206,0.05000,4.562
+0,self_attn.q_proj,0.0000087561,0.05000,4.573
+0,self_attn.o_proj,0.0000000012,0.05000,1.502
+0,mlp.up_proj,0.0000081505,0.05000,2.709
+0,mlp.gate_proj,0.0000094237,0.05000,2.716
+0,mlp.down_proj,0.0000000070,0.05000,5.802
+1,self_attn.q_proj,0.0000285129,0.05000,5.844
+1,self_attn.v_proj,0.0000020318,0.05000,5.856
+1,self_attn.k_proj,0.0000127451,0.05000,5.870
+1,self_attn.o_proj,0.0000000052,0.05000,1.503
+1,mlp.gate_proj,0.0000322149,0.05000,2.735
+1,mlp.up_proj,0.0000282160,0.05000,2.741
+1,mlp.down_proj,0.0000052935,0.05000,5.818
+2,self_attn.v_proj,0.0000137185,0.05000,5.586
+2,self_attn.k_proj,0.0001054883,0.05000,5.603
+2,self_attn.q_proj,0.0002171249,0.05000,5.632
+2,self_attn.o_proj,0.0000000075,0.05000,1.523
+2,mlp.gate_proj,0.0000672672,0.05000,2.829
+2,mlp.up_proj,0.0000587216,0.05000,2.835
+2,mlp.down_proj,0.0000000224,0.05000,5.795
+3,self_attn.v_proj,0.0000093818,0.05000,5.672
+3,self_attn.q_proj,0.0001175257,0.05000,5.684
+3,self_attn.k_proj,0.0000570549,0.05000,5.696
+3,self_attn.o_proj,0.0000000112,0.05000,1.535
+3,mlp.gate_proj,0.0001121420,0.05000,2.703
+3,mlp.up_proj,0.0000968189,0.05000,2.706
+3,mlp.down_proj,0.0000000410,0.05000,5.772
+4,self_attn.q_proj,0.0001813112,0.05000,5.543
+4,self_attn.v_proj,0.0000157679,0.05000,5.555
+4,self_attn.k_proj,0.0000802006,0.05000,5.555
+4,self_attn.o_proj,0.0000000139,0.05000,1.527
+4,mlp.gate_proj,0.0001625613,0.05000,2.747
+4,mlp.up_proj,0.0001317042,0.05000,2.761
+4,mlp.down_proj,0.0000000744,0.05000,5.809
+5,self_attn.k_proj,0.0001039211,0.05000,5.706
+5,self_attn.q_proj,0.0002341090,0.05000,5.739
+5,self_attn.v_proj,0.0000174460,0.05000,5.786
+5,self_attn.o_proj,0.0000000322,0.05000,1.503
+5,mlp.gate_proj,0.0002247521,0.05000,2.734
+5,mlp.up_proj,0.0001709394,0.05000,2.734
+5,mlp.down_proj,0.0000001268,0.05000,5.818
+6,self_attn.k_proj,0.0001032290,0.05000,5.563
+6,self_attn.v_proj,0.0000177316,0.05000,5.576
+6,self_attn.q_proj,0.0002197504,0.05000,5.594
+6,self_attn.o_proj,0.0000000426,0.05000,1.519
+6,mlp.gate_proj,0.0002620231,0.05000,2.695
+6,mlp.up_proj,0.0002037956,0.05000,2.700
+6,mlp.down_proj,0.0000001768,0.05000,5.722
+7,self_attn.k_proj,0.0001301391,0.05000,5.649
+7,self_attn.q_proj,0.0002715318,0.05000,5.686
+7,self_attn.v_proj,0.0000233400,0.05000,5.700
+7,self_attn.o_proj,0.0000000710,0.05000,1.489
+7,mlp.gate_proj,0.0003091105,0.05000,2.726
+7,mlp.up_proj,0.0002347886,0.05000,2.734
+7,mlp.down_proj,0.0000002321,0.05000,5.731
+8,self_attn.k_proj,0.0001017436,0.05000,5.731
+8,self_attn.q_proj,0.0002240664,0.05000,5.757
+8,self_attn.v_proj,0.0000217979,0.05000,5.766
+8,self_attn.o_proj,0.0000000915,0.05000,1.519
+8,mlp.up_proj,0.0002580485,0.05000,2.713
+8,mlp.gate_proj,0.0003330376,0.05000,2.718
+8,mlp.down_proj,0.0000002755,0.05000,5.747
+9,self_attn.v_proj,0.0000258198,0.05000,5.596
+9,self_attn.q_proj,0.0003062698,0.05000,5.632
+9,self_attn.k_proj,0.0001433366,0.05000,5.638
+9,self_attn.o_proj,0.0000001034,0.05000,1.490
+9,mlp.up_proj,0.0002804786,0.05000,2.756
+9,mlp.gate_proj,0.0003499140,0.05000,2.766
+9,mlp.down_proj,0.0000003341,0.05000,5.813
+10,self_attn.k_proj,0.0001297817,0.05000,5.668
+10,self_attn.v_proj,0.0000226510,0.05000,5.703
+10,self_attn.q_proj,0.0002727139,0.05000,5.738
+10,self_attn.o_proj,0.0000001711,0.05000,1.504
+10,mlp.gate_proj,0.0003655863,0.05000,2.747
+10,mlp.up_proj,0.0003002079,0.05000,2.752
+10,mlp.down_proj,0.0000003908,0.05000,5.818
+11,self_attn.v_proj,0.0000333907,0.05000,5.675
+11,self_attn.k_proj,0.0001514282,0.05000,5.690
+11,self_attn.q_proj,0.0003284788,0.05000,5.704
+11,self_attn.o_proj,0.0000002205,0.05000,1.515
+11,mlp.gate_proj,0.0003984610,0.05000,2.731
+11,mlp.up_proj,0.0003320518,0.05000,2.740
+11,mlp.down_proj,0.0000004510,0.05000,5.806
+12,self_attn.k_proj,0.0001964771,0.05000,5.543
+12,self_attn.q_proj,0.0004358755,0.05000,5.666
+12,self_attn.v_proj,0.0000379014,0.05000,5.708
+12,self_attn.o_proj,0.0000002516,0.05000,1.495
+12,mlp.up_proj,0.0003744479,0.05000,2.767
+12,mlp.gate_proj,0.0004391452,0.05000,2.776
+12,mlp.down_proj,0.0000005534,0.05000,5.815
+13,self_attn.v_proj,0.0000360716,0.05000,5.625
+13,self_attn.q_proj,0.0003422806,0.05000,5.741
+13,self_attn.k_proj,0.0001674168,0.05000,5.749
+13,self_attn.o_proj,0.0000003109,0.05000,1.493
+13,mlp.gate_proj,0.0004911288,0.05000,2.685
+13,mlp.up_proj,0.0004301303,0.05000,2.689
+13,mlp.down_proj,0.0000006914,0.05000,5.771
+14,self_attn.k_proj,0.0001664226,0.05000,5.596
+14,self_attn.v_proj,0.0000576001,0.05000,5.624
+14,self_attn.q_proj,0.0003889395,0.05000,5.668
+14,self_attn.o_proj,0.0000004098,0.05000,1.493
+14,mlp.up_proj,0.0004783811,0.05000,2.743
+14,mlp.gate_proj,0.0005488174,0.05000,2.747
+14,mlp.down_proj,0.0000008860,0.05000,5.796
+15,self_attn.q_proj,0.0004867710,0.05000,5.634
+15,self_attn.k_proj,0.0002165216,0.05000,5.752
+15,self_attn.v_proj,0.0000653710,0.05000,5.760
+15,self_attn.o_proj,0.0000004345,0.05000,1.515
+15,mlp.gate_proj,0.0006246117,0.05000,2.696
+15,mlp.up_proj,0.0005259726,0.05000,2.704
+15,mlp.down_proj,0.0000011260,0.05000,5.745
+16,self_attn.k_proj,0.0001991889,0.05000,5.492
+16,self_attn.v_proj,0.0000615639,0.05000,5.508
+16,self_attn.q_proj,0.0004318745,0.05000,5.523
+16,self_attn.o_proj,0.0000005076,0.05000,1.503
+16,mlp.up_proj,0.0006129682,0.05000,2.751
+16,mlp.gate_proj,0.0007694462,0.05000,2.759
+16,mlp.down_proj,0.0000015809,0.05000,5.768
+17,self_attn.v_proj,0.0000590238,0.05000,5.789
+17,self_attn.k_proj,0.0001672252,0.05000,5.818
+17,self_attn.q_proj,0.0003993788,0.05000,5.828
+17,self_attn.o_proj,0.0000006334,0.05000,1.505
+17,mlp.up_proj,0.0007126845,0.05000,2.701
+17,mlp.gate_proj,0.0008941703,0.05000,2.715
+17,mlp.down_proj,0.0000021782,0.05000,5.786
+18,self_attn.v_proj,0.0000725807,0.05000,5.491
+18,self_attn.k_proj,0.0001988027,0.05000,5.544
+18,self_attn.q_proj,0.0005069916,0.05000,5.570
+18,self_attn.o_proj,0.0000005907,0.05000,1.536
+18,mlp.gate_proj,0.0010087889,0.05000,2.764
+18,mlp.up_proj,0.0008049888,0.05000,2.769
+18,mlp.down_proj,0.0000031469,0.05000,5.804
+19,self_attn.q_proj,0.0004640531,0.05000,5.718
+19,self_attn.k_proj,0.0001974665,0.05000,5.751
+19,self_attn.v_proj,0.0000839271,0.05000,5.758
+19,self_attn.o_proj,0.0000007363,0.05000,1.516
+19,mlp.up_proj,0.0008893232,0.05000,2.702
+19,mlp.gate_proj,0.0011344106,0.05000,2.704
+19,mlp.down_proj,0.0000041928,0.05000,5.776
+20,self_attn.v_proj,0.0000917941,0.05000,5.457
+20,self_attn.k_proj,0.0002024085,0.05000,5.464
+20,self_attn.q_proj,0.0004972993,0.05000,5.482
+20,self_attn.o_proj,0.0000006737,0.05000,1.525
+20,mlp.gate_proj,0.0012861970,0.05000,2.697
+20,mlp.up_proj,0.0009820292,0.05000,2.696
+20,mlp.down_proj,0.0000043654,0.05000,5.801
+21,self_attn.k_proj,0.0001981790,0.05000,5.710
+21,self_attn.v_proj,0.0000964703,0.05000,5.747
+21,self_attn.q_proj,0.0004897626,0.05000,5.770
+21,self_attn.o_proj,0.0000006938,0.05000,1.536
+21,mlp.gate_proj,0.0015099732,0.05000,2.692
+21,mlp.up_proj,0.0010752812,0.05000,2.708
+21,mlp.down_proj,0.0000044608,0.05000,5.819
+22,self_attn.v_proj,0.0000996507,0.05000,5.592
+22,self_attn.q_proj,0.0004715399,0.05000,5.642
+22,self_attn.k_proj,0.0001876980,0.05000,5.657
+22,self_attn.o_proj,0.0000004057,0.05000,1.512
+22,mlp.gate_proj,0.0016046927,0.05000,2.720
+22,mlp.up_proj,0.0011544167,0.05000,2.726
+22,mlp.down_proj,0.0000046066,0.05000,5.792
+23,self_attn.q_proj,0.0004739835,0.05000,5.550
+23,self_attn.k_proj,0.0001863427,0.05000,5.639
+23,self_attn.v_proj,0.0001045092,0.05000,5.656
+23,self_attn.o_proj,0.0000006813,0.05000,1.508
+23,mlp.up_proj,0.0012700862,0.05000,2.755
+23,mlp.gate_proj,0.0017711886,0.05000,2.766
+23,mlp.down_proj,0.0000051867,0.05000,5.805
+24,self_attn.q_proj,0.0005410001,0.05000,5.842
+24,self_attn.v_proj,0.0001214173,0.05000,5.883
+24,self_attn.k_proj,0.0002158198,0.05000,5.946
+24,self_attn.o_proj,0.0000006280,0.05000,1.501
+24,mlp.gate_proj,0.0019970989,0.05000,2.721
+24,mlp.up_proj,0.0014057605,0.05000,2.721
+24,mlp.down_proj,0.0000056286,0.05000,5.838
+25,self_attn.k_proj,0.0002077088,0.05000,5.659
+25,self_attn.v_proj,0.0001386532,0.05000,5.680
+25,self_attn.q_proj,0.0005421010,0.05000,5.690
+25,self_attn.o_proj,0.0000006926,0.05000,1.508
+25,mlp.gate_proj,0.0021648855,0.05000,2.806
+25,mlp.up_proj,0.0015368462,0.05000,2.807
+25,mlp.down_proj,0.0000063590,0.05000,5.777
+26,self_attn.q_proj,0.0005148287,0.05000,5.733
+26,self_attn.v_proj,0.0001467281,0.05000,5.761
+26,self_attn.k_proj,0.0001950796,0.05000,5.790
+26,self_attn.o_proj,0.0000009540,0.05000,1.512
+26,mlp.up_proj,0.0016700355,0.05000,2.757
+26,mlp.gate_proj,0.0022701313,0.05000,2.764
+26,mlp.down_proj,0.0000073036,0.05000,5.829
+27,self_attn.q_proj,0.0005313924,0.05000,5.597
+27,self_attn.k_proj,0.0001970788,0.05000,5.617
+27,self_attn.v_proj,0.0001329189,0.05000,5.631
+27,self_attn.o_proj,0.0000010856,0.05000,1.539
+27,mlp.gate_proj,0.0024865442,0.05000,2.738
+27,mlp.up_proj,0.0018403968,0.05000,2.737
+27,mlp.down_proj,0.0000087551,0.05000,5.844
+28,self_attn.k_proj,0.0001981637,0.05000,5.458
+28,self_attn.v_proj,0.0001949648,0.05000,5.500
+28,self_attn.q_proj,0.0005183223,0.05000,5.508
+28,self_attn.o_proj,0.0000016372,0.05000,1.530
+28,mlp.gate_proj,0.0026638827,0.05000,2.748
+28,mlp.up_proj,0.0020455004,0.05000,2.762
+28,mlp.down_proj,0.0000119281,0.05000,5.836
+29,self_attn.q_proj,0.0005931044,0.05000,5.728
+29,self_attn.k_proj,0.0001997706,0.05000,5.741
+29,self_attn.v_proj,0.0002828346,0.05000,5.767
+29,self_attn.o_proj,0.0000034553,0.05000,1.534
+29,mlp.up_proj,0.0020760434,0.05000,2.725
+29,mlp.gate_proj,0.0025936320,0.05000,2.729
+29,mlp.down_proj,0.0000151650,0.05000,5.807
+30,self_attn.v_proj,0.0003089608,0.05000,5.433
+30,self_attn.k_proj,0.0001842356,0.05000,5.482
+30,self_attn.q_proj,0.0005517652,0.05000,5.508
+30,self_attn.o_proj,0.0000030842,0.05000,1.513
+30,mlp.up_proj,0.0021368292,0.05000,2.743
+30,mlp.gate_proj,0.0026252946,0.05000,2.751
+30,mlp.down_proj,0.0000199158,0.05000,5.768
+31,self_attn.k_proj,0.0001757457,0.05000,5.804
+31,self_attn.v_proj,0.0003163291,0.05000,5.898
+31,self_attn.q_proj,0.0005140052,0.05000,5.915
+31,self_attn.o_proj,0.0000041636,0.05000,1.517
+31,mlp.gate_proj,0.0022865063,0.05000,2.720
+31,mlp.up_proj,0.0018066571,0.05000,2.729
+31,mlp.down_proj,0.0000333210,0.05000,5.901

quantize_config.json

@@ -0,0 +1,44 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "lm_head": false,
+  "method": "gptq",
+  "quant_method": "gptq",
+  "format": "gptq",
+  "checkpoint_format": "gptq",
+  "pack_dtype": "int32",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:6.0.3"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.05,
+    "damp_auto_increment": 0.01,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.0,
+    "gptaq": null,
+    "foem": null,
+    "act_group_aware": true,
+    "fallback": {
+      "strategy": "rtn",
+      "threshold": "0.5%",
+      "smooth": null
+    },
+    "offload_to_disk": true,
+    "offload_to_disk_path": "./gptqmodel_offload/orcmjeui-tnqhqrfy/",
+    "pack_impl": "cpu",
+    "gc_mode": "interval",
+    "wait_for_submodule_finalizers": false,
+    "auto_forward_data_parallel": true,
+    "vram_strategy": "exclusive",
+    "mock_quantization": false,
+    "hessian": {
+      "chunk_size": null,
+      "chunk_bytes": null,
+      "staging_dtype": "float32"
+    }
+  },
+  "sym": true
+}

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}