Upload ds_config.json with huggingface_hub

ds_config.json

@@ -0,0 +1,44 @@
+{
+    "bf16": {
+        "enabled": true
+    },
+    "fp16": {
+        "enabled": false
+    },
+    "optimizer": {
+        "type": "AdamW",
+        "params": {
+            "lr": 0,
+            "weight_decay": 0.05,
+            "adam_w_mode": true,
+            "betas": [
+                    0.9,
+                    0.95
+                  ]
+        }
+    },
+    "scheduler": {
+        "type": "WarmupDecayLR",
+        "params": {
+            "warmup_min_lr": 0,
+            "warmup_max_lr": 5e-4,
+            "warmup_num_steps": 500,
+            "total_num_steps": 50000
+        }
+    },
+    "zero_optimization": {
+        "stage": 2,
+        "stage3_gather_16bit_weights_on_model_save": true,
+        "allgather_partitions": true,
+        "allgather_bucket_size": 5e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 5e8,
+        "contiguous_gradients": true
+    },
+    "gradient_accumulation_steps": 3,
+    "gradient_clipping": 1.0,
+    "steps_per_print": 100,
+    "train_micro_batch_size_per_gpu": 3,
+    "wall_clock_breakdown": false
+}