{
    "train_batch_size":"auto",
    "train_micro_batch_size_per_gpu":"auto",
    "gradient_accumulation_steps":"auto",
    "gradient_clipping":"auto",
    "zero_allow_untested_optimizer":true,
    "fp16":{
        "enabled":"auto",
        "loss_scale":0,
        "initial_scale_power":16,
        "loss_scale_window":1000,
        "hysteresis":2,
        "min_loss_scale":1e-32
    },
    "zero_optimization":{
        "stage":2,
        "allgather_partitions":true,
        "allgather_bucket_size":2e8,
        "reduce_scatter":true,
        "reduce_bucket_size":2e8,
        "overlap_comm":false,
        "contiguous_gradients":true
    }
}