quantization_config.json

{
  "quantization_method": "bitsandbytes",
  "bits": 8,
  "load_in_8bit": true,
  "llm_int8_threshold": 6.0
}