SFT PRO MIX checkpoint step 250

training-runs/sft-rag-api-1b-from-pretrain-20k/args.json

@@ -0,0 +1,41 @@
+{
+  "base_repo_id": "ProjectScugnizz/scugnizz-1b",
+  "base_ckpt": "training-runs/pretrain-1b/checkpoint_best.pt",
+  "resume": "",
+  "datasets": [
+    "Daisuke675/scugnizz-instruct-pro-50k",
+    "Daisuke675/synthetic-rag-api-json-response",
+    "iamtarun/python_code_instructions_18k_alpaca"
+  ],
+  "dataset_weights": [
+    0.45,
+    0.4,
+    0.15
+  ],
+  "split": "train",
+  "max_samples_per_dataset": 50000,
+  "val_size": 1000,
+  "packing": true,
+  "block_size": 1024,
+  "batch_size": 2,
+  "grad_accum": 16,
+  "epochs": 1,
+  "max_steps": 20000,
+  "lr": 4e-06,
+  "min_lr": 8e-07,
+  "warmup_steps": 100,
+  "weight_decay": 0.05,
+  "grad_clip": 1.0,
+  "dropout": 0.05,
+  "eval_interval": 250,
+  "save_interval": 250,
+  "log_interval": 10,
+  "eval_batches": 30,
+  "seed": 42,
+  "dtype": "auto",
+  "out_dir": "runs/sft-rag-api-1b-from-pretrain-20k",
+  "hub_repo_id": "ProjectScugnizz/scugnizz-1b",
+  "hub_path": "training-runs/sft-rag-api-1b-from-pretrain-20k",
+  "push": true,
+  "push_every_save": true
+}

training-runs/sft-rag-api-1b-from-pretrain-20k/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6292c29c22f8ad95b482c518422410282440e104bc7077e5a9c88bdb12d29dd
+size 15758460627

training-runs/sft-rag-api-1b-from-pretrain-20k/checkpoint_last.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:000e864bba0307088cba35c18d800c06df5ac3745d7d1cbab0593a10b80e800c
+size 15758460627