File size: 393 Bytes
6438a35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | {
"recipe": "bitsandbytes LLM.int8",
"load_in_8bit": true,
"llm_int8_threshold": 6.0,
"llm_int8_has_fp16_weight": false,
"llm_int8_skip_modules": ["lm_head"],
"floating_compute_dtype": "bfloat16",
"pointer_head_dtype": "float32",
"adapter_merged": false,
"serialization": "safetensors",
"max_shard_size": "4GB",
"device": "single CUDA GPU",
"automatic_upload": false
}
|