Self-quantized phi-2-gptq 4-bit (group_size=128, seed=42)

config.json

@@ -0,0 +1,75 @@
+{
+  "architectures": [
+    "PhiForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 50256,
+  "dtype": "float16",
+  "embd_pdrop": 0.0,
+  "eos_token_id": 50256,
+  "hidden_act": "gelu_new",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 10240,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "phi",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 50256,
+  "partial_rotary_factor": 0.4,
+  "qk_layernorm": false,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "format": "gptq",
+    "group_size": 128,
+    "lm_head": false,
+    "meta": {
+      "act_group_aware": true,
+      "auto_forward_data_parallel": true,
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.05,
+      "failsafe": {
+        "smooth": null,
+        "strategy": "rtn",
+        "threshold": "0.5%"
+      },
+      "gc_mode": "interval",
+      "gptaq": null,
+      "hessian": {
+        "chunk_bytes": null,
+        "chunk_size": null,
+        "staging_dtype": "float32"
+      },
+      "mock_quantization": false,
+      "mse": 0.0,
+      "offload_to_disk": true,
+      "offload_to_disk_path": "./gptqmodel_offload/bxitqbrw-vnczleuk/",
+      "pack_impl": "cpu",
+      "quantizer": [
+        "gptqmodel:5.8.0"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "vram_strategy": "exclusive",
+      "wait_for_submodule_finalizers": false
+    },
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "resid_pdrop": 0.1,
+  "rope_parameters": {
+    "partial_rotary_factor": 0.4,
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.4.0",
+  "use_cache": true,
+  "vocab_size": 51200
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "do_sample": true,
+  "eos_token_id": 50256,
+  "transformers_version": "5.4.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad5a14048f4bc42ebe924f4c13355d96ca51fdf7c680e0ff2792a4d746af1c42
+size 1836707608

quant_log.csv

@@ -0,0 +1,193 @@
+layer,module,loss,samples,damp,time
+0,self_attn.q_proj,0.0000858512,0.05000,1.097
+0,self_attn.k_proj,0.0001029116,0.05000,1.129
+0,self_attn.v_proj,0.0000302286,0.05000,0.869
+0,self_attn.dense,0.0000123729,0.05000,0.857
+0,mlp.fc1,0.0001053048,0.05000,0.953
+0,mlp.fc2,0.0000069301,0.05000,4.047
+1,self_attn.q_proj,0.0000778551,0.05000,0.913
+1,self_attn.k_proj,0.0000812629,0.05000,0.925
+1,self_attn.v_proj,0.0000198182,0.05000,0.865
+1,self_attn.dense,0.0000046095,0.05000,0.893
+1,mlp.fc1,0.0002500907,0.05000,0.915
+1,mlp.fc2,0.0000168595,0.05000,3.982
+2,self_attn.q_proj,0.0001165650,0.05000,0.969
+2,self_attn.k_proj,0.0001173934,0.05000,0.950
+2,self_attn.v_proj,0.0000463347,0.05000,0.882
+2,self_attn.dense,0.0000052613,0.05000,0.881
+2,mlp.fc1,0.0004108166,0.05000,1.135
+2,mlp.fc2,0.0000199922,0.05000,4.276
+3,self_attn.q_proj,0.0001364429,0.05000,0.991
+3,self_attn.k_proj,0.0001284486,0.05000,0.984
+3,self_attn.v_proj,0.0000970313,0.05000,0.888
+3,self_attn.dense,0.0000050054,0.05000,0.920
+3,mlp.fc1,0.0006537699,0.05000,0.965
+3,mlp.fc2,0.0000279560,0.05000,4.165
+4,self_attn.q_proj,0.0001806877,0.05000,1.029
+4,self_attn.k_proj,0.0001703575,0.05000,0.915
+4,self_attn.v_proj,0.0001177177,0.05000,0.927
+4,self_attn.dense,0.0000071838,0.05000,0.914
+4,mlp.fc1,0.0006948134,0.05000,0.997
+4,mlp.fc2,0.0000382566,0.05000,4.367
+5,self_attn.q_proj,0.0001677401,0.05000,1.003
+5,self_attn.k_proj,0.0001583099,0.05000,0.936
+5,self_attn.v_proj,0.0001066596,0.05000,0.892
+5,self_attn.dense,0.0000078510,0.05000,0.975
+5,mlp.fc1,0.0006093672,0.05000,1.055
+5,mlp.fc2,0.0000421267,0.05000,4.136
+6,self_attn.q_proj,0.0002005054,0.05000,1.016
+6,self_attn.k_proj,0.0001911783,0.05000,0.982
+6,self_attn.v_proj,0.0001320928,0.05000,0.967
+6,self_attn.dense,0.0000092071,0.05000,0.945
+6,mlp.fc1,0.0006585670,0.05000,0.935
+6,mlp.fc2,0.0000462386,0.05000,4.468
+7,self_attn.q_proj,0.0002163916,0.05000,1.070
+7,self_attn.k_proj,0.0002072118,0.05000,0.937
+7,self_attn.v_proj,0.0001352158,0.05000,0.978
+7,self_attn.dense,0.0000117110,0.05000,0.970
+7,mlp.fc1,0.0006694943,0.05000,0.943
+7,mlp.fc2,0.0000498323,0.05000,4.157
+8,self_attn.q_proj,0.0002109012,0.05000,0.949
+8,self_attn.k_proj,0.0002039318,0.05000,1.241
+8,self_attn.v_proj,0.0001560394,0.05000,0.996
+8,self_attn.dense,0.0000082801,0.05000,0.990
+8,mlp.fc1,0.0006576708,0.05000,1.030
+8,mlp.fc2,0.0000524794,0.05000,4.458
+9,self_attn.q_proj,0.0002133507,0.05000,1.027
+9,self_attn.k_proj,0.0002043295,0.05000,0.955
+9,self_attn.v_proj,0.0001553389,0.05000,0.979
+9,self_attn.dense,0.0000089070,0.05000,0.967
+9,mlp.fc1,0.0006463157,0.05000,1.011
+9,mlp.fc2,0.0000528918,0.05000,4.181
+10,self_attn.q_proj,0.0002181723,0.05000,0.950
+10,self_attn.k_proj,0.0002149413,0.05000,0.952
+10,self_attn.v_proj,0.0001367607,0.05000,0.928
+10,self_attn.dense,0.0000134973,0.05000,0.896
+10,mlp.fc1,0.0006295212,0.05000,0.923
+10,mlp.fc2,0.0000522401,0.05000,4.083
+11,self_attn.q_proj,0.0002261469,0.05000,0.958
+11,self_attn.k_proj,0.0002225379,0.05000,0.977
+11,self_attn.v_proj,0.0001492103,0.05000,0.945
+11,self_attn.dense,0.0000167664,0.05000,0.914
+11,mlp.fc1,0.0006092769,0.05000,0.930
+11,mlp.fc2,0.0000528684,0.05000,5.138
+12,self_attn.q_proj,0.0002321886,0.05000,0.974
+12,self_attn.k_proj,0.0002303848,0.05000,0.891
+12,self_attn.v_proj,0.0001530323,0.05000,0.908
+12,self_attn.dense,0.0000169518,0.05000,0.920
+12,mlp.fc1,0.0006060201,0.05000,0.973
+12,mlp.fc2,0.0000549961,0.05000,4.304
+13,self_attn.q_proj,0.0002252652,0.05000,0.972
+13,self_attn.k_proj,0.0002252355,0.05000,1.009
+13,self_attn.v_proj,0.0001491018,0.05000,0.952
+13,self_attn.dense,0.0000157061,0.05000,0.996
+13,mlp.fc1,0.0006169498,0.05000,1.020
+13,mlp.fc2,0.0000539558,0.05000,4.684
+14,self_attn.q_proj,0.0002650832,0.05000,1.065
+14,self_attn.k_proj,0.0002238111,0.05000,0.999
+14,self_attn.v_proj,0.0001467716,0.05000,0.949
+14,self_attn.dense,0.0000165920,0.05000,0.983
+14,mlp.fc1,0.0006014784,0.05000,0.989
+14,mlp.fc2,0.0000545418,0.05000,4.357
+15,self_attn.q_proj,0.0002222408,0.05000,0.952
+15,self_attn.k_proj,0.0002214731,0.05000,0.935
+15,self_attn.v_proj,0.0001403913,0.05000,0.971
+15,self_attn.dense,0.0000169734,0.05000,0.912
+15,mlp.fc1,0.0005749508,0.05000,0.944
+15,mlp.fc2,0.0000612357,0.05000,4.166
+16,self_attn.q_proj,0.0002301284,0.05000,1.015
+16,self_attn.k_proj,0.0002283591,0.05000,0.881
+16,self_attn.v_proj,0.0001480706,0.05000,0.896
+16,self_attn.dense,0.0000156473,0.05000,0.876
+16,mlp.fc1,0.0005911736,0.05000,0.936
+16,mlp.fc2,0.0000615094,0.05000,4.146
+17,self_attn.q_proj,0.0002169144,0.05000,0.949
+17,self_attn.k_proj,0.0002164072,0.05000,0.980
+17,self_attn.v_proj,0.0001373301,0.05000,0.942
+17,self_attn.dense,0.0000168942,0.05000,0.929
+17,mlp.fc1,0.0005453525,0.05000,0.907
+17,mlp.fc2,0.0000619073,0.05000,4.035
+18,self_attn.q_proj,0.0002519506,0.05000,0.997
+18,self_attn.k_proj,0.0002533842,0.05000,0.974
+18,self_attn.v_proj,0.0001425683,0.05000,0.953
+18,self_attn.dense,0.0000179922,0.05000,0.886
+18,mlp.fc1,0.0005474294,0.05000,0.933
+18,mlp.fc2,0.0000634956,0.05000,4.298
+19,self_attn.q_proj,0.0002300704,0.05000,1.044
+19,self_attn.k_proj,0.0002330887,0.05000,0.926
+19,self_attn.v_proj,0.0001444852,0.05000,0.995
+19,self_attn.dense,0.0000164025,0.05000,1.168
+19,mlp.fc1,0.0005811269,0.05000,1.058
+19,mlp.fc2,0.0000723687,0.05000,4.503
+20,self_attn.q_proj,0.0002827060,0.05000,1.040
+20,self_attn.k_proj,0.0002414499,0.05000,0.943
+20,self_attn.v_proj,0.0001363139,0.05000,0.986
+20,self_attn.dense,0.0000187169,0.05000,1.034
+20,mlp.fc1,0.0005760693,0.05000,1.011
+20,mlp.fc2,0.0000753935,0.05000,4.585
+21,self_attn.q_proj,0.0002291035,0.05000,1.085
+21,self_attn.k_proj,0.0002217587,0.05000,0.947
+21,self_attn.v_proj,0.0001625256,0.05000,0.968
+21,self_attn.dense,0.0000147600,0.05000,0.959
+21,mlp.fc1,0.0006096665,0.05000,1.005
+21,mlp.fc2,0.0000837082,0.05000,4.373
+22,self_attn.q_proj,0.0002641166,0.05000,1.019
+22,self_attn.k_proj,0.0002266009,0.05000,1.127
+22,self_attn.v_proj,0.0001701314,0.05000,0.935
+22,self_attn.dense,0.0000122861,0.05000,0.906
+22,mlp.fc1,0.0006319354,0.05000,0.929
+22,mlp.fc2,0.0000845768,0.05000,4.132
+23,self_attn.q_proj,0.0002803879,0.05000,0.939
+23,self_attn.k_proj,0.0002535842,0.05000,0.996
+23,self_attn.v_proj,0.0001640246,0.05000,0.914
+23,self_attn.dense,0.0000198861,0.05000,0.889
+23,mlp.fc1,0.0006627535,0.05000,0.919
+23,mlp.fc2,0.0000945106,0.05000,4.222
+24,self_attn.q_proj,0.0002384906,0.05000,0.962
+24,self_attn.k_proj,0.0002369107,0.05000,0.904
+24,self_attn.v_proj,0.0001989545,0.05000,0.933
+24,self_attn.dense,0.0000162029,0.05000,0.887
+24,mlp.fc1,0.0007166684,0.05000,0.989
+24,mlp.fc2,0.0001029097,0.05000,4.025
+25,self_attn.q_proj,0.0003113051,0.05000,1.047
+25,self_attn.k_proj,0.0002572589,0.05000,1.003
+25,self_attn.v_proj,0.0002068744,0.05000,0.917
+25,self_attn.dense,0.0000187007,0.05000,0.979
+25,mlp.fc1,0.0007801538,0.05000,0.941
+25,mlp.fc2,0.0001016458,0.05000,4.126
+26,self_attn.q_proj,0.0003590267,0.05000,0.980
+26,self_attn.k_proj,0.0002574304,0.05000,0.954
+26,self_attn.v_proj,0.0002369789,0.05000,0.952
+26,self_attn.dense,0.0000299120,0.05000,0.937
+26,mlp.fc1,0.0007834066,0.05000,0.999
+26,mlp.fc2,0.0001124531,0.05000,4.201
+27,self_attn.q_proj,0.0003025899,0.05000,0.952
+27,self_attn.k_proj,0.0003528239,0.05000,0.908
+27,self_attn.v_proj,0.0002532654,0.05000,0.892
+27,self_attn.dense,0.0000275395,0.05000,0.961
+27,mlp.fc1,0.0008518580,0.05000,0.971
+27,mlp.fc2,0.0001470365,0.05000,4.424
+28,self_attn.q_proj,0.0002637209,0.05000,1.063
+28,self_attn.k_proj,0.0002721585,0.05000,0.953
+28,self_attn.v_proj,0.0002665452,0.05000,0.952
+28,self_attn.dense,0.0000327637,0.05000,0.881
+28,mlp.fc1,0.0008877072,0.05000,0.929
+28,mlp.fc2,0.0001765151,0.05000,4.167
+29,self_attn.q_proj,0.0037034005,0.05000,0.975
+29,self_attn.k_proj,0.0030506286,0.05000,1.012
+29,self_attn.v_proj,0.0001793814,0.05000,0.972
+29,self_attn.dense,0.0000784153,0.05000,0.954
+29,mlp.fc1,0.0012136550,0.05000,1.026
+29,mlp.fc2,0.0001923476,0.05000,4.545
+30,self_attn.q_proj,0.0124896431,0.05000,1.096
+30,self_attn.k_proj,0.0038405387,0.05000,1.110
+30,self_attn.v_proj,0.0001383956,0.05000,1.107
+30,self_attn.dense,0.0000560131,0.05000,1.004
+30,mlp.fc1,0.0013959989,0.05000,1.031
+30,mlp.fc2,0.0001954762,0.05000,4.354
+31,self_attn.q_proj,0.0038123704,0.05000,0.988
+31,self_attn.k_proj,0.0006134240,0.05000,0.890
+31,self_attn.v_proj,0.0000717409,0.05000,0.939
+31,self_attn.dense,0.0000127410,0.05000,0.895
+31,mlp.fc1,0.0007779660,0.05000,0.936
+31,mlp.fc2,0.0001298984,0.05000,4.108

quantize_config.json

@@ -0,0 +1,7 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "sym": true,
+  "quant_method": "gptq"
+}

quantize_manifest.json

@@ -0,0 +1,12 @@
+{
+  "model_name": "phi-2",
+  "hf_id": "microsoft/phi-2",
+  "method": "gptq",
+  "bits": 4,
+  "group_size": 128,
+  "calibration_dataset": "allenai/c4",
+  "calibration_samples": 128,
+  "seed": 42,
+  "elapsed_s": 690.9595537185669,
+  "tool": "gptqmodel"
+}

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CodeGenTokenizerFast",
+  "unk_token": "<|endoftext|>",
+  "_commit_hash": null
+}