Upload GPTQ quantized model

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,86 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "format": "gptq",
+    "group_size": 128,
+    "lm_head": false,
+    "meta": {
+      "act_group_aware": true,
+      "auto_forward_data_parallel": true,
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.05,
+      "failsafe": {
+        "smooth": {
+          "group_size_threshold": 128,
+          "k": 2.75,
+          "type": "mad"
+        },
+        "strategy": "rtn",
+        "threshold": "0.5%"
+      },
+      "gc_mode": "interval",
+      "gptaq": null,
+      "hessian": {
+        "chunk_bytes": null,
+        "chunk_size": null,
+        "staging_dtype": "float32"
+      },
+      "mock_quantization": false,
+      "mse": 0.0,
+      "offload_to_disk": true,
+      "offload_to_disk_path": "./gptqmodel_offload/znzwasrn-gqcllywg/",
+      "pack_impl": "cpu",
+      "quantizer": [
+        "gptqmodel:5.7.0"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "vram_strategy": "exclusive",
+      "wait_for_submodule_finalizers": false
+    },
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.3.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecdcaefe73ea1bc089b009a972240e8f514dfb7d13e077d803575e6627d5c213
+size 2255767876

quant_log.csv

@@ -0,0 +1,197 @@
+layer,module,loss,samples,damp,time
+0,self_attn.k_proj,0.0000833145,0.05000,5.504
+0,self_attn.v_proj,0.0000048262,0.05000,5.610
+0,self_attn.q_proj,0.0001627265,0.05000,5.606
+0,self_attn.o_proj,0.0000002214,0.05000,1.191
+0,mlp.gate_proj,0.0000997544,0.05000,3.078
+0,mlp.up_proj,0.0000880482,0.05000,3.099
+0,mlp.down_proj,0.0000010819,0.05000,3.332
+1,self_attn.q_proj,0.0002339984,0.05000,5.468
+1,self_attn.k_proj,0.0001366074,0.05000,5.615
+1,self_attn.v_proj,0.0000161811,0.05000,5.641
+1,self_attn.o_proj,0.0000006392,0.05000,1.165
+1,mlp.gate_proj,0.0001622974,0.05000,2.888
+1,mlp.up_proj,0.0001428511,0.05000,2.918
+1,mlp.down_proj,0.0003240824,0.05000,3.307
+2,self_attn.q_proj,0.0009462966,0.05000,5.619
+2,self_attn.k_proj,0.0005598217,0.05000,5.643
+2,self_attn.v_proj,0.0000587609,0.05000,5.671
+2,self_attn.o_proj,0.0000006137,0.05000,1.139
+2,mlp.up_proj,0.0002262556,0.05000,2.879
+2,mlp.gate_proj,0.0002659620,0.05000,2.937
+2,mlp.down_proj,0.0000028477,0.05000,2.979
+3,self_attn.q_proj,0.0006456479,0.05000,5.110
+3,self_attn.v_proj,0.0000623240,0.05000,5.181
+3,self_attn.k_proj,0.0003549168,0.05000,5.208
+3,self_attn.o_proj,0.0000012870,0.05000,1.070
+3,mlp.up_proj,0.0002929948,0.05000,2.290
+3,mlp.gate_proj,0.0003876079,0.05000,2.322
+3,mlp.down_proj,0.0000042261,0.05000,3.058
+4,self_attn.v_proj,0.0000641730,0.05000,5.203
+4,self_attn.q_proj,0.0006326817,0.05000,5.248
+4,self_attn.k_proj,0.0003315510,0.05000,5.274
+4,self_attn.o_proj,0.0000023387,0.05000,1.078
+4,mlp.gate_proj,0.0005043794,0.05000,2.391
+4,mlp.up_proj,0.0003418379,0.05000,2.413
+4,mlp.down_proj,0.0000061632,0.05000,3.072
+5,self_attn.k_proj,0.0005265745,0.05000,5.141
+5,self_attn.v_proj,0.0000638105,0.05000,5.204
+5,self_attn.q_proj,0.0008782418,0.05000,5.240
+5,self_attn.o_proj,0.0000025126,0.05000,1.060
+5,mlp.up_proj,0.0003865409,0.05000,2.458
+5,mlp.gate_proj,0.0005401021,0.05000,2.486
+5,mlp.down_proj,0.0000082887,0.05000,3.048
+6,self_attn.v_proj,0.0000612041,0.05000,5.136
+6,self_attn.k_proj,0.0003914881,0.05000,5.177
+6,self_attn.q_proj,0.0007378610,0.05000,5.211
+6,self_attn.o_proj,0.0000043394,0.05000,1.034
+6,mlp.gate_proj,0.0005627311,0.05000,2.457
+6,mlp.up_proj,0.0004075962,0.05000,2.475
+6,mlp.down_proj,0.0000099089,0.05000,2.958
+7,self_attn.k_proj,0.0003767969,0.05000,5.226
+7,self_attn.v_proj,0.0000546954,0.05000,5.279
+7,self_attn.q_proj,0.0006362990,0.05000,5.295
+7,self_attn.o_proj,0.0000061320,0.05000,1.056
+7,mlp.up_proj,0.0004317052,0.05000,2.434
+7,mlp.gate_proj,0.0005475797,0.05000,2.443
+7,mlp.down_proj,0.0000111637,0.05000,3.088
+8,self_attn.k_proj,0.0004685118,0.05000,5.168
+8,self_attn.v_proj,0.0000670339,0.05000,5.199
+8,self_attn.q_proj,0.0007847853,0.05000,5.229
+8,self_attn.o_proj,0.0000080611,0.05000,1.108
+8,mlp.up_proj,0.0004457112,0.05000,2.385
+8,mlp.gate_proj,0.0005756153,0.05000,2.414
+8,mlp.down_proj,0.0000119709,0.05000,3.063
+9,self_attn.q_proj,0.0007281643,0.05000,5.169
+9,self_attn.v_proj,0.0000788263,0.05000,5.243
+9,self_attn.k_proj,0.0004247850,0.05000,5.248
+9,self_attn.o_proj,0.0000077604,0.05000,1.016
+9,mlp.gate_proj,0.0005563830,0.05000,2.452
+9,mlp.up_proj,0.0004468315,0.05000,2.474
+9,mlp.down_proj,0.0000119611,0.05000,2.982
+10,self_attn.v_proj,0.0000621950,0.05000,5.226
+10,self_attn.k_proj,0.0004504600,0.05000,5.274
+10,self_attn.q_proj,0.0007390235,0.05000,5.337
+10,self_attn.o_proj,0.0000074142,0.05000,1.079
+10,mlp.gate_proj,0.0005601441,0.05000,2.303
+10,mlp.up_proj,0.0004797459,0.05000,2.322
+10,mlp.down_proj,0.0000133788,0.05000,3.071
+11,self_attn.v_proj,0.0000770151,0.05000,5.048
+11,self_attn.k_proj,0.0003612847,0.05000,5.134
+11,self_attn.q_proj,0.0006474613,0.05000,5.161
+11,self_attn.o_proj,0.0000095606,0.05000,1.116
+11,mlp.up_proj,0.0005172904,0.05000,2.391
+11,mlp.gate_proj,0.0005825480,0.05000,2.425
+11,mlp.down_proj,0.0000147894,0.05000,3.155
+12,self_attn.k_proj,0.0005030657,0.05000,5.040
+12,self_attn.q_proj,0.0008600825,0.05000,5.172
+12,self_attn.v_proj,0.0000796377,0.05000,5.186
+12,self_attn.o_proj,0.0000104103,0.05000,1.071
+12,mlp.up_proj,0.0005417673,0.05000,2.423
+12,mlp.gate_proj,0.0006048538,0.05000,2.449
+12,mlp.down_proj,0.0000159788,0.05000,3.090
+13,self_attn.k_proj,0.0005679255,0.05000,5.098
+13,self_attn.q_proj,0.0009026564,0.05000,5.188
+13,self_attn.v_proj,0.0000920754,0.05000,5.205
+13,self_attn.o_proj,0.0000111667,0.05000,1.057
+13,mlp.up_proj,0.0005947397,0.05000,2.364
+13,mlp.gate_proj,0.0007042694,0.05000,2.385
+13,mlp.down_proj,0.0000207118,0.05000,3.023
+14,self_attn.v_proj,0.0001053973,0.05000,5.222
+14,self_attn.k_proj,0.0004498990,0.05000,5.278
+14,self_attn.q_proj,0.0009584807,0.05000,5.306
+14,self_attn.o_proj,0.0000123116,0.05000,1.090
+14,mlp.up_proj,0.0006279950,0.05000,2.357
+14,mlp.gate_proj,0.0007503058,0.05000,2.383
+14,mlp.down_proj,0.0000250014,0.05000,3.074
+15,self_attn.k_proj,0.0004797868,0.05000,4.990
+15,self_attn.v_proj,0.0001005991,0.05000,5.166
+15,self_attn.q_proj,0.0009365002,0.05000,5.185
+15,self_attn.o_proj,0.0000076168,0.05000,1.067
+15,mlp.up_proj,0.0006121564,0.05000,2.409
+15,mlp.gate_proj,0.0007879107,0.05000,2.440
+15,mlp.down_proj,0.0000245487,0.05000,3.107
+16,self_attn.v_proj,0.0001098983,0.05000,5.208
+16,self_attn.q_proj,0.0009610252,0.05000,5.322
+16,self_attn.k_proj,0.0005322082,0.05000,5.361
+16,self_attn.o_proj,0.0000051471,0.05000,1.175
+16,mlp.up_proj,0.0005994866,0.05000,2.891
+16,mlp.gate_proj,0.0007955586,0.05000,2.940
+16,mlp.down_proj,0.0000232563,0.05000,3.234
+17,self_attn.v_proj,0.0001056392,0.05000,5.383
+17,self_attn.k_proj,0.0004881229,0.05000,5.419
+17,self_attn.q_proj,0.0009143897,0.05000,5.493
+17,self_attn.o_proj,0.0000044561,0.05000,1.173
+17,mlp.gate_proj,0.0008321888,0.05000,2.826
+17,mlp.up_proj,0.0006171612,0.05000,2.839
+17,mlp.down_proj,0.0000246102,0.05000,3.241
+18,self_attn.v_proj,0.0001294144,0.05000,5.402
+18,self_attn.q_proj,0.0010036130,0.05000,5.491
+18,self_attn.k_proj,0.0005490085,0.05000,5.539
+18,self_attn.o_proj,0.0000047686,0.05000,1.202
+18,mlp.up_proj,0.0006625156,0.05000,2.816
+18,mlp.gate_proj,0.0008736592,0.05000,2.858
+18,mlp.down_proj,0.0000259963,0.05000,3.274
+19,self_attn.q_proj,0.0009206464,0.05000,5.306
+19,self_attn.v_proj,0.0001300041,0.05000,5.346
+19,self_attn.k_proj,0.0005263201,0.05000,5.421
+19,self_attn.o_proj,0.0000081053,0.05000,1.206
+19,mlp.up_proj,0.0007115615,0.05000,2.476
+19,mlp.gate_proj,0.0009267995,0.05000,2.501
+19,mlp.down_proj,0.0000313765,0.05000,3.005
+20,self_attn.k_proj,0.0005501762,0.05000,5.269
+20,self_attn.q_proj,0.0009316968,0.05000,5.310
+20,self_attn.v_proj,0.0001533229,0.05000,5.339
+20,self_attn.o_proj,0.0000053971,0.05000,1.074
+20,mlp.gate_proj,0.0008958201,0.05000,2.386
+20,mlp.up_proj,0.0007214489,0.05000,2.403
+20,mlp.down_proj,0.0000298560,0.05000,3.048
+21,self_attn.k_proj,0.0005322746,0.05000,5.115
+21,self_attn.v_proj,0.0001965262,0.05000,5.151
+21,self_attn.q_proj,0.0009163225,0.05000,5.171
+21,self_attn.o_proj,0.0000058034,0.05000,1.071
+21,mlp.up_proj,0.0007544485,0.05000,2.394
+21,mlp.gate_proj,0.0009466076,0.05000,2.413
+21,mlp.down_proj,0.0000325444,0.05000,2.984
+22,self_attn.v_proj,0.0001977625,0.05000,5.113
+22,self_attn.q_proj,0.0009122455,0.05000,5.193
+22,self_attn.k_proj,0.0005100145,0.05000,5.229
+22,self_attn.o_proj,0.0000058460,0.05000,1.057
+22,mlp.gate_proj,0.0010337803,0.05000,2.355
+22,mlp.up_proj,0.0008206790,0.05000,2.376
+22,mlp.down_proj,0.0000370742,0.05000,2.997
+23,self_attn.v_proj,0.0001945850,0.05000,5.120
+23,self_attn.k_proj,0.0005421275,0.05000,5.187
+23,self_attn.q_proj,0.0009014102,0.05000,5.207
+23,self_attn.o_proj,0.0000118679,0.05000,1.071
+23,mlp.gate_proj,0.0011874684,0.05000,2.391
+23,mlp.up_proj,0.0009013988,0.05000,2.412
+23,mlp.down_proj,0.0000456437,0.05000,3.112
+24,self_attn.k_proj,0.0006013965,0.05000,5.117
+24,self_attn.q_proj,0.0009966169,0.05000,5.178
+24,self_attn.v_proj,0.0002814000,0.05000,5.192
+24,self_attn.o_proj,0.0000167604,0.05000,1.067
+24,mlp.up_proj,0.0009939129,0.05000,2.419
+24,mlp.gate_proj,0.0013279702,0.05000,2.446
+24,mlp.down_proj,0.0000607813,0.05000,3.056
+25,self_attn.k_proj,0.0004979556,0.05000,5.164
+25,self_attn.q_proj,0.0009840556,0.05000,5.252
+25,self_attn.v_proj,0.0002535410,0.05000,5.264
+25,self_attn.o_proj,0.0000184899,0.05000,1.043
+25,mlp.gate_proj,0.0014792334,0.05000,2.438
+25,mlp.up_proj,0.0011075135,0.05000,2.447
+25,mlp.down_proj,0.0000877130,0.05000,3.067
+26,self_attn.v_proj,0.0003436589,0.05000,5.014
+26,self_attn.q_proj,0.0008804976,0.05000,5.161
+26,self_attn.k_proj,0.0005433848,0.05000,5.189
+26,self_attn.o_proj,0.0000359726,0.05000,1.044
+26,mlp.up_proj,0.0011477621,0.05000,2.349
+26,mlp.gate_proj,0.0015504545,0.05000,2.368
+26,mlp.down_proj,0.0001262205,0.05000,3.025
+27,self_attn.v_proj,0.0002239405,0.05000,5.051
+27,self_attn.k_proj,0.0003883892,0.05000,5.105
+27,self_attn.q_proj,0.0007127493,0.05000,5.130
+27,self_attn.o_proj,0.0000817960,0.05000,1.054
+27,mlp.up_proj,0.0012877010,0.05000,2.463
+27,mlp.gate_proj,0.0015451877,0.05000,2.483
+27,mlp.down_proj,0.0004653256,0.05000,3.056

quantize_config.json

@@ -0,0 +1,46 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "lm_head": false,
+  "quant_method": "gptq",
+  "checkpoint_format": "gptq",
+  "pack_dtype": "int32",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:5.7.0"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.05,
+    "damp_auto_increment": 0.01,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.0,
+    "gptaq": null,
+    "act_group_aware": true,
+    "failsafe": {
+      "strategy": "rtn",
+      "threshold": "0.5%",
+      "smooth": {
+        "type": "mad",
+        "group_size_threshold": 128,
+        "k": 2.75
+      }
+    },
+    "offload_to_disk": true,
+    "offload_to_disk_path": "./gptqmodel_offload/znzwasrn-gqcllywg/",
+    "pack_impl": "cpu",
+    "mock_quantization": false,
+    "gc_mode": "interval",
+    "wait_for_submodule_finalizers": false,
+    "auto_forward_data_parallel": true,
+    "hessian": {
+      "chunk_size": null,
+      "chunk_bytes": null,
+      "staging_dtype": "float32"
+    },
+    "vram_strategy": "exclusive"
+  },
+  "sym": true,
+  "format": "gptq"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c85066e7642934ed09b44155e6566b0b5dab2637fb9433439ba5c9c7f8b50d3
+size 17210018

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "tokenizer_class": "TokenizersBackend"
+}