pretrain checkpoint step 250

training-runs/pretrain-test-50m/args.json

@@ -0,0 +1,32 @@
+{
+  "model_size": "1b",
+  "dataset": "HuggingFaceFW/fineweb",
+  "dataset_config": "sample-10BT",
+  "split": "train",
+  "text_field": "text",
+  "block_size": 1024,
+  "batch_size": 1,
+  "grad_accum": 64,
+  "target_tokens": 50000000,
+  "max_steps": 763,
+  "lr": 0.00015,
+  "min_lr": 1.5e-05,
+  "warmup_steps": 1000,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "dropout": 0.1,
+  "eval_interval": 250,
+  "eval_batches": 10,
+  "save_interval": 250,
+  "log_interval": 10,
+  "pcs_a": 0.8309193524478643,
+  "pcs_b": 0.0,
+  "out_dir": "runs/pretrain",
+  "hub_repo_id": "Daisuke675/scugnizz-1b",
+  "hub_path": "training-runs/pretrain-test-50m",
+  "push": true,
+  "push_every_save": true,
+  "seed": 42,
+  "dtype": "auto",
+  "no_resume": false
+}

training-runs/pretrain-test-50m/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fce0f5dea0150d8dcc00671241869f744c8ba36974c48807f1e1c29dea1302d9
+size 15758460307

training-runs/pretrain-test-50m/checkpoint_last.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c672a480b42a4f73460063b33798a315595a1c3ef9ab61b59210271be108b1b0
+size 15758460307