pretrain checkpoint step 8000

training-runs/pretrain-750m/args.json

@@ -0,0 +1,33 @@
+{
+  "model_size": "1b",
+  "dataset": "HuggingFaceFW/fineweb",
+  "dataset_config": "sample-10BT",
+  "split": "train",
+  "text_field": "text",
+  "block_size": 1024,
+  "batch_size": 1,
+  "grad_accum": 64,
+  "target_tokens": 750000000,
+  "max_steps": 11445,
+  "lr": 0.00015,
+  "min_lr": 1.5e-05,
+  "warmup_steps": 1000,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "dropout": 0.1,
+  "eval_interval": 1000,
+  "eval_batches": 10,
+  "save_interval": 1000,
+  "log_interval": 20,
+  "pcs_a": 0.8309193524478643,
+  "pcs_b": 0.0,
+  "out_dir": "runs/pretrain",
+  "hub_repo_id": "Daisuke675/scugnizz-1b",
+  "hub_path": "training-runs/pretrain-750m",
+  "resume_hub_file": "training-runs/pretrain-500m/checkpoint_last.pt",
+  "push": true,
+  "push_every_save": true,
+  "seed": 42,
+  "dtype": "auto",
+  "no_resume": false
+}

training-runs/pretrain-750m/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80ad049f531a45b5bc261f70a6bc2e152150a5ffd576a6e96630d2ade25a4e2b
+size 15758461011

training-runs/pretrain-750m/checkpoint_last.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e4a9ee1cade79a47a8e6e54c27e5525c1e50db7b784928770c9667bef024ce7
+size 15758461011