| { | |
| "train_batch_size":"auto", | |
| "train_micro_batch_size_per_gpu":"auto", | |
| "gradient_accumulation_steps":"auto", | |
| "gradient_clipping":"auto", | |
| "zero_allow_untested_optimizer":true, | |
| "fp16":{ | |
| "enabled":"auto", | |
| "loss_scale":0, | |
| "initial_scale_power":16, | |
| "loss_scale_window":1000, | |
| "hysteresis":2, | |
| "min_loss_scale":1e-32 | |
| }, | |
| "zero_optimization":{ | |
| "stage":2, | |
| "allgather_partitions":true, | |
| "allgather_bucket_size":2e8, | |
| "reduce_scatter":true, | |
| "reduce_bucket_size":2e8, | |
| "overlap_comm":false, | |
| "contiguous_gradients":true | |
| } | |
| } |