quantization_config.json

{
  "quant_method": "QUiP",
  "rescale_WH": false,
  "use_rand": true,
  "codebook": "E8P12",
  "codesz": 8,
  "idx_dtype": "torch.int16",
  "modules_to_not_convert": [
    "gate"
  ]
}