Add training template: ds_config_zero2.json

training-template/ds_config_zero2.json

@@ -0,0 +1,29 @@
+{
+  "_NOTE": "ZeRO Stage 2 for full-parameter SFT on single H200 SXM. Stage 2 shards gradients (not params) and offloads optimizer states to CPU RAM. No optimizer block here — Adafactor is configured in the training script directly, because DeepSpeed's optimizer config conflicts with custom optimizers passed to the Trainer.",
+
+  "bf16": {
+    "enabled": true
+  },
+
+  "zero_optimization": {
+    "stage": 2,
+
+    "offload_optimizer": {
+      "device": "cpu",
+      "pin_memory": true
+    },
+
+    "offload_param": {
+      "device": "none"
+    },
+
+    "overlap_comm": true,
+    "contiguous_gradients": true,
+    "reduce_scatter": true
+  },
+
+  "gradient_clipping": 1.0,
+  "gradient_accumulation_steps": 8,
+  "train_micro_batch_size_per_gpu": 1,
+  "steps_per_print": 10
+}