pretrain core 4

scripts/pretrain_core_model_4.yaml

@@ -136,10 +136,10 @@ eval:
 
 optimizer:
   # class_path: torch.optim.AdamW
-  class_path: torchao.prototype.low_bit_optim.AdamW8bit
+  # class_path: torchao.prototype.low_bit_optim.AdamW8bit
   # class_path: torchao.prototype.low_bit_optim.AdamW4bit
   # class_path: bitsandbytes.optim.AdamW8bit
-  # class_path: bitsandbytes.optim.PagedAdamW8bit
+  class_path: bitsandbytes.optim.PagedAdamW8bit
   init_args:
     # (type: float, default: 0.001)
     lr: 1e-4