Upload 6 files

config_pretrain.json

@@ -0,0 +1,40 @@
+{
+    "use_lora": false,
+    "lora_rank": 8,
+    "lora_alpha": 16,
+    "lora_dropout": 0.0,
+
+    "from_checkpoint": "/home/bd4sur/ai/Nano/checkpoint/checkpoint_20241122_232939_step_196000.pt",
+    "save_checkpoint_to": "/home/bd4sur/ai/Nano/checkpoint",
+    "dataset_path": [
+        ["/home/bd4sur/ai/Nano/dataset_preprocessed/pt_train_0.base64", "/home/bd4sur/ai/Nano/dataset_preprocessed/pt_val_0.base64"]
+    ],
+    "tokenizer_path": "/home/bd4sur/ai/Nano/tokenizer/tokenizer_16384.json",
+
+    "random_seed": 39,
+    "batch_size": 80,
+    "gradient_accumulation_steps": 1,
+    "grad_clip": 1.0,
+
+    "dropout": 0.0,
+
+    "learning_rate": 5e-4,
+    "weight_decay": 1e-1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+
+    "decay_lr": true,
+    "warmup_iters": 500,
+    "lr_decay_iters": 1e9,
+    "min_lr": 6e-5,
+
+    "eval_interval": 500,
+    "log_interval": 10,
+    "eval_iters": 2,
+
+    "backend": "nccl",
+    "device": "cuda",
+    "sdp_kernel": "flash",
+    "dtype": "bfloat16",
+    "use_amp": true
+}

config_sft.json

@@ -0,0 +1,40 @@
+{
+    "use_lora": false,
+    "lora_rank": 8,
+    "lora_alpha": 16,
+    "lora_dropout": 0.0,
+
+    "from_checkpoint": "/root/autodl-tmp/checkpoint_20241130_005942_step_307000.pt",
+    "save_checkpoint_to": "/root/autodl-tmp/checkpoint",
+    "dataset_path": [
+        ["/root/autodl-tmp/sft_train.base64", "/root/autodl-tmp/sft_val.base64"]
+    ],
+    "tokenizer_path": "/root/Nano/tokenizer/tokenizer_16384.json",
+
+    "random_seed": 39,
+    "batch_size": 120,
+    "gradient_accumulation_steps": 2,
+    "grad_clip": 1.0,
+
+    "dropout": 0.1,
+
+    "learning_rate": 1e-6,
+    "weight_decay": 1e-1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+
+    "decay_lr": false,
+    "warmup_iters": 10000,
+    "lr_decay_iters": 1e9,
+    "min_lr": 1e-8,
+
+    "eval_interval": 500,
+    "log_interval": 10,
+    "eval_iters": 5,
+
+    "backend": "nccl",
+    "device": "cuda",
+    "sdp_kernel": "flash",
+    "dtype": "bfloat16",
+    "use_amp": true
+}

nano_168m_307000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac4e4469d8346cb01195bf804115a2693b548610729ecb1742de51ef81ffaa29
+size 2045994798

nano_168m_307000_sft_379000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c184f0373682a4f2715163464358b0a8882e392a7fed6dd9f24563aafe45747
+size 2045994734

nano_168m_320000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:162bc16899a55900f044c656146e3687b3260e9c295a9e4b79e5cebd6e4d960d
+size 2045994798