Upload 5 files
Browse files- psycho_230k.bin +3 -0
- psycho_230k.pt +3 -0
- psycho_230k_config_pretrain.json +40 -0
- psycho_230k_model_config.json +13 -0
- psycho_230k_q80.bin +3 -0
psycho_230k.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7394d60248493f8ad7e6d10fc69fd12c411767de0848bd5c369ac88eadfabe91
|
| 3 |
+
size 986052
|
psycho_230k.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9eef36a0799e67315706929870872cb93054deae05e776e0f2d291d020897c8e
|
| 3 |
+
size 11371891
|
psycho_230k_config_pretrain.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"use_lora": false,
|
| 3 |
+
"lora_rank": 8,
|
| 4 |
+
"lora_alpha": 16,
|
| 5 |
+
"lora_dropout": 0.0,
|
| 6 |
+
|
| 7 |
+
"from_checkpoint": "",
|
| 8 |
+
"save_checkpoint_to": "/home/bd4sur/ai/Nano/checkpoint",
|
| 9 |
+
"dataset_path": [
|
| 10 |
+
["/home/bd4sur/ai/Nano/dataset_preprocessed/pt_train_0.base64", "/home/bd4sur/ai/Nano/dataset_preprocessed/pt_val_0.base64"]
|
| 11 |
+
],
|
| 12 |
+
"tokenizer_path": "/home/bd4sur/ai/Nano/tokenizer/tokenizer_4096.json",
|
| 13 |
+
|
| 14 |
+
"random_seed": 39,
|
| 15 |
+
"batch_size": 256,
|
| 16 |
+
"gradient_accumulation_steps": 1,
|
| 17 |
+
"grad_clip": 1.0,
|
| 18 |
+
|
| 19 |
+
"dropout": 0.0,
|
| 20 |
+
|
| 21 |
+
"learning_rate": 5e-4,
|
| 22 |
+
"weight_decay": 1e-1,
|
| 23 |
+
"beta1": 0.9,
|
| 24 |
+
"beta2": 0.95,
|
| 25 |
+
|
| 26 |
+
"decay_lr": true,
|
| 27 |
+
"warmup_iters": 500,
|
| 28 |
+
"lr_decay_iters": 1e9,
|
| 29 |
+
"min_lr": 6e-5,
|
| 30 |
+
|
| 31 |
+
"eval_interval": 100,
|
| 32 |
+
"log_interval": 10,
|
| 33 |
+
"eval_iters": 5,
|
| 34 |
+
|
| 35 |
+
"backend": "nccl",
|
| 36 |
+
"device": "cuda",
|
| 37 |
+
"sdp_kernel": "flash",
|
| 38 |
+
"dtype": "bfloat16",
|
| 39 |
+
"use_amp": true
|
| 40 |
+
}
|
psycho_230k_model_config.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"block_size": 512,
|
| 3 |
+
"vocab_size": 4096,
|
| 4 |
+
"n_layer": 8,
|
| 5 |
+
"n_embd": 32,
|
| 6 |
+
"n_head": 4,
|
| 7 |
+
"n_kv_head": 2,
|
| 8 |
+
"n_hidden": 96,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"use_rope": true,
|
| 11 |
+
"norm_eps": 1e-5,
|
| 12 |
+
"is_causal": true
|
| 13 |
+
}
|
psycho_230k_q80.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0f354e3fb1975670915cf9bf18a88d0cf80ef5bbf63fa64d87d3267cf58a4f2f
|
| 3 |
+
size 326596
|