pretrain core 4

scripts/pretrain_core_model_4.yaml

@@ -60,10 +60,10 @@ train:
   # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
   # global_batch_size: 512
   # global_batch_size: 256
-  global_batch_size: 32
+  global_batch_size: 16
 
   # Number of samples per data-parallel rank (type: int, default: 4)
-  micro_batch_size: 4
+  micro_batch_size: 1
 
   # Number of iterations with learning rate warmup active (type: int, default: 2000)
   lr_warmup_steps: 0