pretrain checkpoint step 8200

training-runs/pretrain-750m/args.json

@@ -8,7 +8,7 @@
   "batch_size": 1,
   "grad_accum": 64,
   "target_tokens": 750000000,
-  "max_steps": 11445,
+  "max_steps": 0,
   "lr": 0.00015,
   "min_lr": 1.5e-05,
   "warmup_steps": 1000,
@@ -17,14 +17,14 @@
   "dropout": 0.1,
   "eval_interval": 1000,
   "eval_batches": 10,
-  "save_interval": 1000,
+  "save_interval": 200,
   "log_interval": 20,
   "pcs_a": 0.8309193524478643,
   "pcs_b": 0.0,
   "out_dir": "runs/pretrain",
   "hub_repo_id": "Daisuke675/scugnizz-1b",
   "hub_path": "training-runs/pretrain-750m",
-  "resume_hub_file": "training-runs/pretrain-500m/checkpoint_last.pt",
+  "resume_hub_file": "training-runs/pretrain-750m/checkpoint_last.pt",
   "push": true,
   "push_every_save": true,
   "seed": 42,

training-runs/pretrain-750m/checkpoint_last.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6df04129b8d7bac06904d0e34f40c460ea83ca815254960e6779cd8375de7fe2
-size 15758461011
+oid sha256:25c474f14d3604ea6853a1a69fe35cdf0e437bd08849fd0b62e30b82a2bfe88c
+size 15758460435