global_batch_size: 512; micro_batch_size: 1

scripts/pretrain_core_model.yaml

@@ -58,10 +58,10 @@ train:
   log_interval: 1
 
   # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
-  global_batch_size: 256
+  global_batch_size: 512
 
   # Number of samples per data-parallel rank (type: int, default: 4)
-  micro_batch_size: 2
+  micro_batch_size: 1
 
   # Number of iterations with learning rate warmup active (type: int, default: 2000)
   lr_warmup_steps: 500