Upload GPTQ quantized model

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1 @@
+{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}

config.json

@@ -0,0 +1,193 @@
+{
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM",
+    "AutoTokenizer": "Xenova/gpt-4o"
+  },
+  "bos_token_id": 199999,
+  "dtype": "bfloat16",
+  "embd_pdrop": 0.0,
+  "eos_token_id": 199999,
+  "full_attn_mod": 1,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "interpolate_factor": 1,
+  "lm_head_bias": false,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "phi3",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 199999,
+  "partial_rotary_factor": 0.75,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "format": "gptq",
+    "group_size": 128,
+    "lm_head": false,
+    "meta": {
+      "act_group_aware": true,
+      "auto_forward_data_parallel": true,
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.05,
+      "dense_vram_strategy": "exclusive",
+      "dense_vram_strategy_devices": null,
+      "fallback": {
+        "smooth": null,
+        "strategy": "rtn",
+        "threshold": "0.5%"
+      },
+      "foem": null,
+      "gc_mode": "interval",
+      "gptaq": null,
+      "hessian": {
+        "chunk_bytes": null,
+        "chunk_size": null,
+        "staging_dtype": "float32"
+      },
+      "mock_quantization": false,
+      "moe_vram_strategy": "exclusive",
+      "moe_vram_strategy_devices": null,
+      "mse": 0.0,
+      "offload_to_disk": true,
+      "offload_to_disk_path": "/tmp/gptqmodel_l3mykc7c",
+      "pack_impl": "cpu",
+      "quantizer": [
+        "gptqmodel:7.0.0"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "wait_for_submodule_finalizers": false
+    },
+    "method": "gptq",
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "long_factor": [
+      1,
+      1.118320672,
+      1.250641126,
+      1.398617824,
+      1.564103225,
+      1.74916897,
+      1.956131817,
+      2.187582649,
+      2.446418898,
+      2.735880826,
+      3.059592084,
+      3.421605075,
+      3.826451687,
+      4.279200023,
+      4.785517845,
+      5.351743533,
+      5.984965424,
+      6.693110555,
+      7.485043894,
+      8.370679318,
+      9.36110372,
+      10.4687158,
+      11.70738129,
+      13.09260651,
+      14.64173252,
+      16.37415215,
+      18.31155283,
+      20.47818807,
+      22.90118105,
+      25.61086418,
+      28.64115884,
+      32.03,
+      32.1,
+      32.13,
+      32.23,
+      32.6,
+      32.61,
+      32.64,
+      32.66,
+      32.7,
+      32.71,
+      32.93,
+      32.97,
+      33.28,
+      33.49,
+      33.5,
+      44.16,
+      47.77
+    ],
+    "original_max_position_embeddings": 4096,
+    "partial_rotary_factor": 0.75,
+    "rope_theta": 10000.0,
+    "rope_type": "longrope",
+    "short_factor": [
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0
+    ],
+    "type": "longrope"
+  },
+  "sliding_window": 262144,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.9.0",
+  "use_cache": true,
+  "vocab_size": 200064
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 199999,
+  "do_sample": true,
+  "eos_token_id": [
+    200020,
+    199999
+  ],
+  "pad_token_id": 199999,
+  "transformers_version": "5.9.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e4959d65f52543658f0a1ccdd01ec423952432edb204c9f47b700b79bae89d2
+size 2905820840

quant_log.csv

@@ -0,0 +1,161 @@
+layer,module,loss,samples,damp,time
+0,self_attn.qkv_proj,0.0000238235,0.05000,1.526
+0,self_attn.o_proj,0.0000012382,0.05000,1.191
+0,mlp.gate_proj,0.0000777876,0.05000,2.541
+0,mlp.up_proj,0.0000749441,0.05000,2.631
+0,mlp.down_proj,0.0000051147,0.05000,3.721
+1,self_attn.qkv_proj,0.0001717266,0.05000,1.136
+1,self_attn.o_proj,0.0000035207,0.05000,1.105
+1,mlp.up_proj,0.0001190371,0.05000,2.573
+1,mlp.gate_proj,0.0001262916,0.05000,2.590
+1,mlp.down_proj,0.0000209985,0.05000,3.567
+2,self_attn.qkv_proj,0.0004606717,0.05000,1.153
+2,self_attn.o_proj,0.0000034317,0.05000,1.204
+2,mlp.gate_proj,0.0001890496,0.05000,2.425
+2,mlp.up_proj,0.0001766404,0.05000,2.478
+2,mlp.down_proj,0.0000112479,0.05000,3.655
+3,self_attn.qkv_proj,0.0006576798,0.05000,1.162
+3,self_attn.o_proj,0.0000072551,0.05000,1.165
+3,mlp.up_proj,0.0002476366,0.05000,2.459
+3,mlp.gate_proj,0.0002709125,0.05000,2.520
+3,mlp.down_proj,0.0259172664,0.05000,3.695
+4,self_attn.qkv_proj,0.0006655008,0.05000,1.215
+4,self_attn.o_proj,0.0000091820,0.05000,1.136
+4,mlp.gate_proj,0.0003341130,0.05000,2.537
+4,mlp.up_proj,0.0002933624,0.05000,2.573
+4,mlp.down_proj,0.0000299716,0.05000,3.575
+5,self_attn.qkv_proj,0.0007712968,0.05000,1.190
+5,self_attn.o_proj,0.0000108736,0.05000,1.193
+5,mlp.up_proj,0.0003416692,0.05000,2.371
+5,mlp.gate_proj,0.0003962075,0.05000,2.424
+5,mlp.down_proj,0.0000411710,0.05000,3.709
+6,self_attn.qkv_proj,0.0006552734,0.05000,1.180
+6,self_attn.o_proj,0.0000143339,0.05000,1.128
+6,mlp.gate_proj,0.0003783428,0.05000,2.483
+6,mlp.up_proj,0.0003409256,0.05000,2.500
+6,mlp.down_proj,0.0000462377,0.05000,3.630
+7,self_attn.qkv_proj,0.0009352742,0.05000,1.217
+7,self_attn.o_proj,0.0000168276,0.05000,1.170
+7,mlp.up_proj,0.0003627903,0.05000,2.417
+7,mlp.gate_proj,0.0003876315,0.05000,2.451
+7,mlp.down_proj,0.0000598990,0.05000,3.605
+8,self_attn.qkv_proj,0.0008719424,0.05000,1.185
+8,self_attn.o_proj,0.0000235102,0.05000,1.168
+8,mlp.gate_proj,0.0003903917,0.05000,2.488
+8,mlp.up_proj,0.0003862102,0.05000,2.522
+8,mlp.down_proj,0.0000738566,0.05000,3.713
+9,self_attn.qkv_proj,0.0008102300,0.05000,1.182
+9,self_attn.o_proj,0.0000252474,0.05000,1.166
+9,mlp.up_proj,0.0004095033,0.05000,2.477
+9,mlp.gate_proj,0.0004039039,0.05000,2.545
+9,mlp.down_proj,0.0000623042,0.05000,3.594
+10,self_attn.qkv_proj,0.0008590046,0.05000,1.223
+10,self_attn.o_proj,0.0000361045,0.05000,1.164
+10,mlp.gate_proj,0.0004138994,0.05000,2.371
+10,mlp.up_proj,0.0004185723,0.05000,2.445
+10,mlp.down_proj,0.0000664324,0.05000,3.691
+11,self_attn.qkv_proj,0.0008356616,0.05000,1.182
+11,self_attn.o_proj,0.0000273909,0.05000,1.169
+11,mlp.up_proj,0.0004351710,0.05000,2.454
+11,mlp.gate_proj,0.0004140602,0.05000,2.499
+11,mlp.down_proj,0.0000679036,0.05000,3.662
+12,self_attn.qkv_proj,0.0008949989,0.05000,1.214
+12,self_attn.o_proj,0.0000258759,0.05000,1.159
+12,mlp.gate_proj,0.0003999382,0.05000,2.366
+12,mlp.up_proj,0.0004371861,0.05000,2.444
+12,mlp.down_proj,0.0000790061,0.05000,3.681
+13,self_attn.qkv_proj,0.0010007589,0.05000,1.201
+13,self_attn.o_proj,0.0000284145,0.05000,1.141
+13,mlp.up_proj,0.0004744596,0.05000,2.411
+13,mlp.gate_proj,0.0004218430,0.05000,2.477
+13,mlp.down_proj,0.0000882345,0.05000,3.668
+14,self_attn.qkv_proj,0.0009347553,0.05000,1.254
+14,self_attn.o_proj,0.0000278657,0.05000,1.132
+14,mlp.gate_proj,0.0004590117,0.05000,2.410
+14,mlp.up_proj,0.0005133247,0.05000,2.452
+14,mlp.down_proj,0.0000937629,0.05000,3.722
+15,self_attn.qkv_proj,0.0010369427,0.05000,1.178
+15,self_attn.o_proj,0.0000249127,0.05000,1.143
+15,mlp.up_proj,0.0005674770,0.05000,2.445
+15,mlp.gate_proj,0.0005078213,0.05000,2.458
+15,mlp.down_proj,0.0001074678,0.05000,3.565
+16,self_attn.qkv_proj,0.0009804815,0.05000,1.219
+16,self_attn.o_proj,0.0000504799,0.05000,1.134
+16,mlp.gate_proj,0.0005253795,0.05000,2.448
+16,mlp.up_proj,0.0005916055,0.05000,2.500
+16,mlp.down_proj,0.0001349178,0.05000,3.665
+17,self_attn.qkv_proj,0.0010292538,0.05000,1.187
+17,self_attn.o_proj,0.0000403605,0.05000,1.153
+17,mlp.up_proj,0.0006312746,0.05000,2.306
+17,mlp.gate_proj,0.0005545107,0.05000,2.378
+17,mlp.down_proj,0.0001600333,0.05000,3.641
+18,self_attn.qkv_proj,0.0010037017,0.05000,1.182
+18,self_attn.o_proj,0.0000637205,0.05000,1.175
+18,mlp.gate_proj,0.0005921678,0.05000,2.351
+18,mlp.up_proj,0.0006664776,0.05000,2.437
+18,mlp.down_proj,0.0002355822,0.05000,3.763
+19,self_attn.qkv_proj,0.0010758908,0.05000,1.198
+19,self_attn.o_proj,0.0000702092,0.05000,1.154
+19,mlp.up_proj,0.0007063535,0.05000,2.483
+19,mlp.gate_proj,0.0006349712,0.05000,2.577
+19,mlp.down_proj,0.0002522909,0.05000,3.706
+20,self_attn.qkv_proj,0.0010339192,0.05000,1.196
+20,self_attn.o_proj,0.0000883794,0.05000,1.164
+20,mlp.gate_proj,0.0007679065,0.05000,2.510
+20,mlp.up_proj,0.0008546022,0.05000,2.524
+20,mlp.down_proj,0.0003165363,0.05000,3.687
+21,self_attn.qkv_proj,0.0012044958,0.05000,1.212
+21,self_attn.o_proj,0.0000640508,0.05000,1.162
+21,mlp.up_proj,0.0009123604,0.05000,2.478
+21,mlp.gate_proj,0.0008116388,0.05000,2.527
+21,mlp.down_proj,0.0004007476,0.05000,3.808
+22,self_attn.qkv_proj,0.0011504276,0.05000,1.175
+22,self_attn.o_proj,0.0000911345,0.05000,1.149
+22,mlp.gate_proj,0.0008588976,0.05000,2.403
+22,mlp.up_proj,0.0009739880,0.05000,2.458
+22,mlp.down_proj,0.0004551394,0.05000,3.702
+23,self_attn.qkv_proj,0.0013398652,0.05000,1.189
+23,self_attn.o_proj,0.0000587729,0.05000,1.188
+23,mlp.up_proj,0.0010498388,0.05000,2.402
+23,mlp.gate_proj,0.0009337669,0.05000,2.449
+23,mlp.down_proj,0.0004714720,0.05000,3.573
+24,self_attn.qkv_proj,0.0014503187,0.05000,1.312
+24,self_attn.o_proj,0.0000586708,0.05000,1.156
+24,mlp.gate_proj,0.0010084476,0.05000,2.444
+24,mlp.up_proj,0.0011423412,0.05000,2.492
+24,mlp.down_proj,0.0005463777,0.05000,3.822
+25,self_attn.qkv_proj,0.0015214244,0.05000,1.204
+25,self_attn.o_proj,0.0000830141,0.05000,1.149
+25,mlp.up_proj,0.0012727813,0.05000,2.471
+25,mlp.gate_proj,0.0011201203,0.05000,2.545
+25,mlp.down_proj,0.0006316668,0.05000,3.679
+26,self_attn.qkv_proj,0.0013453441,0.05000,1.193
+26,self_attn.o_proj,0.0001492357,0.05000,1.147
+26,mlp.up_proj,0.0013456151,0.05000,2.563
+26,mlp.gate_proj,0.0011857533,0.05000,2.593
+26,mlp.down_proj,0.0007685589,0.05000,3.712
+27,self_attn.qkv_proj,0.0014054913,0.05000,1.197
+27,self_attn.o_proj,0.0001597724,0.05000,1.177
+27,mlp.up_proj,0.0014349269,0.05000,2.501
+27,mlp.gate_proj,0.0012792713,0.05000,2.572
+27,mlp.down_proj,0.0009183092,0.05000,3.721
+28,self_attn.qkv_proj,0.0016723366,0.05000,1.208
+28,self_attn.o_proj,0.0001700563,0.05000,1.215
+28,mlp.gate_proj,0.0013890771,0.05000,2.471
+28,mlp.up_proj,0.0015182733,0.05000,2.542
+28,mlp.down_proj,0.0011522665,0.05000,3.806
+29,self_attn.qkv_proj,0.0015151091,0.05000,1.153
+29,self_attn.o_proj,0.0002396083,0.05000,1.168
+29,mlp.up_proj,0.0015197603,0.05000,2.468
+29,mlp.gate_proj,0.0015041680,0.05000,2.508
+29,mlp.down_proj,0.0018855077,0.05000,3.632
+30,self_attn.qkv_proj,0.0010489146,0.05000,1.232
+30,self_attn.o_proj,0.0002350578,0.05000,1.128
+30,mlp.gate_proj,0.0015210857,0.05000,2.606
+30,mlp.up_proj,0.0013783333,0.05000,2.638
+30,mlp.down_proj,0.0171129781,0.05000,3.743
+31,self_attn.qkv_proj,0.0000227647,0.05000,1.255
+31,self_attn.o_proj,0.0000320150,0.05000,1.175
+31,mlp.up_proj,0.0003993228,0.05000,2.532
+31,mlp.gate_proj,0.0006483612,0.05000,2.584
+31,mlp.down_proj,0.0030651457,0.05000,3.721

quantize_config.json

@@ -0,0 +1,47 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "lm_head": false,
+  "method": "gptq",
+  "quant_method": "gptq",
+  "format": "gptq",
+  "checkpoint_format": "gptq",
+  "pack_dtype": "int32",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:7.0.0"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.05,
+    "damp_auto_increment": 0.01,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.0,
+    "gptaq": null,
+    "foem": null,
+    "act_group_aware": true,
+    "fallback": {
+      "strategy": "rtn",
+      "threshold": "0.5%",
+      "smooth": null
+    },
+    "offload_to_disk": true,
+    "offload_to_disk_path": "/tmp/gptqmodel_l3mykc7c",
+    "pack_impl": "cpu",
+    "gc_mode": "interval",
+    "wait_for_submodule_finalizers": false,
+    "auto_forward_data_parallel": true,
+    "dense_vram_strategy": "exclusive",
+    "dense_vram_strategy_devices": null,
+    "moe_vram_strategy": "exclusive",
+    "moe_vram_strategy_devices": null,
+    "mock_quantization": false,
+    "hessian": {
+      "chunk_size": null,
+      "chunk_bytes": null,
+      "staging_dtype": "float32"
+    }
+  },
+  "sym": true
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:439894569b1646b4096ff6f0c104c2e078931c0581850932f8a576d05b755e5a
+size 15524574

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}