| {"fp16": {"enabled": true}, "optimizer": {"type": "AdamW", "params": {"lr": "auto", "weight_decay": "auto"}}, "scheduler": {"type": "WarmupDecayLR", "params": {"warmup_min_lr": "auto", "warmup_max_lr": "auto", "warmup_num_steps": "auto", "total_num_steps": "auto"}}, "zero_optimization": {"stage": 3, "offload_optimizer": {"device": "cpu", "pin_memory": true}, "offload_param": {"device": "cpu", "pin_memory": true}, "overlap_comm": true, "contiguous_gradients": true, "reduce_bucket_size": "auto", "stage3_prefetch_bucket_size": "auto", "stage3_param_persistence_threshold": "auto", "sub_group_size": 1000000000.0, "stage3_max_live_parameters": 1000000000.0, "stage3_max_reuse_distance": 1000000000.0, "stage3_gather_16bit_weights_on_model_save": "auto"}, "gradient_accumulation_steps": 1, "gradient_clipping": "auto", "steps_per_print": 2000, "train_batch_size": 256, "train_micro_batch_size_per_gpu": 32, "wall_clock_breakdown": false, "bf16": {"enabled": false}} |