| { | |
| "recipe": "bitsandbytes LLM.int8", | |
| "load_in_8bit": true, | |
| "llm_int8_threshold": 6.0, | |
| "llm_int8_has_fp16_weight": false, | |
| "llm_int8_skip_modules": ["lm_head"], | |
| "floating_compute_dtype": "bfloat16", | |
| "pointer_head_dtype": "float32", | |
| "adapter_merged": false, | |
| "serialization": "safetensors", | |
| "max_shard_size": "4GB", | |
| "device": "single CUDA GPU", | |
| "automatic_upload": false | |
| } | |