| { | |
| "_NOTE": "ZeRO Stage 2 for full-parameter SFT on single H200 SXM. Stage 2 shards gradients (not params) and offloads optimizer states to CPU RAM. No optimizer block here — Adafactor is configured in the training script directly, because DeepSpeed's optimizer config conflicts with custom optimizers passed to the Trainer.", | |
| "bf16": { | |
| "enabled": true | |
| }, | |
| "zero_optimization": { | |
| "stage": 2, | |
| "offload_optimizer": { | |
| "device": "cpu", | |
| "pin_memory": true | |
| }, | |
| "offload_param": { | |
| "device": "none" | |
| }, | |
| "overlap_comm": true, | |
| "contiguous_gradients": true, | |
| "reduce_scatter": true | |
| }, | |
| "gradient_clipping": 1.0, | |
| "gradient_accumulation_steps": 8, | |
| "train_micro_batch_size_per_gpu": 1, | |
| "steps_per_print": 10 | |
| } | |