Upload config_nano_168m_625000_sft_875000_amateur_radio_890000.json

config_nano_168m_625000_sft_875000_amateur_radio_890000.json

@@ -0,0 +1,40 @@
+{
+    "use_lora": false,
+    "lora_rank": 8,
+    "lora_alpha": 16,
+    "lora_dropout": 0.0,
+
+    "from_checkpoint": "/home/bd4sur/ai/Nano/checkpoint/checkpoint_20250104_120620_step_875000.pt",
+    "save_checkpoint_to": "/home/bd4sur/ai/Nano/checkpoint",
+    "dataset_path": [
+        ["/home/bd4sur/ai/Nano/dataset_preprocessed/sft_train.base64", "/home/bd4sur/ai/Nano/dataset_preprocessed/sft_val.base64"]
+    ],
+    "tokenizer_path": "/home/bd4sur/ai/Nano/tokenizer/tokenizer_16384.json",
+
+    "random_seed": 39,
+    "batch_size": 10,
+    "gradient_accumulation_steps": 1,
+    "grad_clip": 1.0,
+
+    "dropout": 0.1,
+
+    "learning_rate": 5e-6,
+    "weight_decay": 1e-1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+
+    "decay_lr": false,
+    "warmup_iters": 10000,
+    "lr_decay_iters": 1e9,
+    "min_lr": 1e-8,
+
+    "eval_interval": 500,
+    "log_interval": 10,
+    "eval_iters": 5,
+
+    "backend": "nccl",
+    "device": "cuda",
+    "sdp_kernel": "flash",
+    "dtype": "bfloat16",
+    "use_amp": true
+}