File size: 515 Bytes
2ee4cd6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 | distributed_type: DEEPSPEED
deepspeed_config:
deepspeed_multinode_launcher: standard
gradient_clipping: 0.0
zero_stage: 3 #2
offload_optimizer_device: cpu # Moves optimizer states to CPU RAM
offload_param_device: cpu # Moves model parameters to CPU RAM
zero3_init_flag: true # Initializes the model directly across GPUs to save CPU RAM
zero3_save_16bit_model: true # Consolidates weights into a single file when saving checkpoints
num_machines: 1
num_processes: 8
machine_rank: 0
|