File size: 1,214 Bytes
626cbe5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
{
"bf16": {
"enabled": false
},
"fp16": {
"enabled": true
},
"zero_optimization": {
"stage": 3,
"contiguous_gradients": true,
"stage3_gather_16bit_weights_on_model_save": true,
"stage3_max_live_parameters": 1e9,
"stage3_max_reuse_distance": 1e9,
"stage3_prefetch_bucket_size": 1e7,
"stage3_param_persistence_threshold": 1e5,
"reduce_bucket_size": 1e7,
"sub_group_size": 1e9,
"offload_optimizer": {
"device": "none",
"nvme_path": "/scratch",
"pin_memory": true
},
"offload_param": {
"device": "none",
"nvme_path": "/scratch",
"buffer_size": 4e9,
"pin_memory": true
}
},
"aio": {
"block_size": 26214400,
"queue_depth": 32,
"thread_count": 1,
"single_submit": false,
"overlap_events": true
},
"gradient_accumulation_steps": 1,
"gradient_clipping": 1.0,
"steps_per_print": 20000,
"train_batch_size": 128,
"train_micro_batch_size_per_gpu": 16,
"wall_clock_breakdown": false,
"zero_allow_untested_optimizer": true
}
|