Hugging Face's logo

vxnuaj
/
SLM-35.7M

Model card Files
xet
 Community
vxnuaj commited on
Commit
30e3c4f
·
verified ·
1 Parent(s): 009a8e5

Upload folder using huggingface_hub

Browse files
Files changed (35) hide show
  1. .gitattributes +33 -0
  2. Run_3/checkpoint_epoch_0_step_12499_global_step_12500 +3 -0
  3. Run_3/checkpoint_epoch_0_step_14999_global_step_15000 +3 -0
  4. Run_3/checkpoint_epoch_0_step_17499_global_step_17500 +3 -0
  5. Run_3/checkpoint_epoch_0_step_19999_global_step_20000 +3 -0
  6. Run_3/checkpoint_epoch_0_step_2499_global_step_2500 +3 -0
  7. Run_3/checkpoint_epoch_0_step_4999_global_step_5000 +3 -0
  8. Run_3/checkpoint_epoch_0_step_7499_global_step_7500 +3 -0
  9. Run_3/checkpoint_epoch_0_step_9999_global_step_10000 +3 -0
  10. Run_3/checkpoint_epoch_1_step_11136_global_step_32500 +3 -0
  11. Run_3/checkpoint_epoch_1_step_1136_global_step_22500 +3 -0
  12. Run_3/checkpoint_epoch_1_step_13636_global_step_35000 +3 -0
  13. Run_3/checkpoint_epoch_1_step_16136_global_step_37500 +3 -0
  14. Run_3/checkpoint_epoch_1_step_18636_global_step_40000 +3 -0
  15. Run_3/checkpoint_epoch_1_step_21136_global_step_42500 +3 -0
  16. Run_3/checkpoint_epoch_1_step_3636_global_step_25000 +3 -0
  17. Run_3/checkpoint_epoch_1_step_6136_global_step_27500 +3 -0
  18. Run_3/checkpoint_epoch_1_step_8636_global_step_30000 +3 -0
  19. Run_3/checkpoint_epoch_2_step_12273_global_step_55000 +3 -0
  20. Run_3/checkpoint_epoch_2_step_14773_global_step_57500 +3 -0
  21. Run_3/checkpoint_epoch_2_step_17273_global_step_60000 +3 -0
  22. Run_3/checkpoint_epoch_2_step_19773_global_step_62500 +3 -0
  23. Run_3/checkpoint_epoch_2_step_2273_global_step_45000 +3 -0
  24. Run_3/checkpoint_epoch_2_step_4773_global_step_47500 +3 -0
  25. Run_3/checkpoint_epoch_2_step_7273_global_step_50000 +3 -0
  26. Run_3/checkpoint_epoch_2_step_9773_global_step_52500 +3 -0
  27. Run_3/checkpoint_epoch_3_step_10910_global_step_75000 +3 -0
  28. Run_3/checkpoint_epoch_3_step_13410_global_step_77500 +3 -0
  29. Run_3/checkpoint_epoch_3_step_15910_global_step_80000 +3 -0
  30. Run_3/checkpoint_epoch_3_step_18410_global_step_82500 +3 -0
  31. Run_3/checkpoint_epoch_3_step_3410_global_step_67500 +3 -0
  32. Run_3/checkpoint_epoch_3_step_5910_global_step_70000 +3 -0
  33. Run_3/checkpoint_epoch_3_step_8410_global_step_72500 +3 -0
  34. Run_3/checkpoint_epoch_3_step_910_global_step_65000 +3 -0
  35. Run_3/run_3_config/configs.txt +90 -0
.gitattributes CHANGED
@@ -33,3 +33,36 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ Run_3/checkpoint_epoch_0_step_12499_global_step_12500 filter=lfs diff=lfs merge=lfs -text
37
+ Run_3/checkpoint_epoch_0_step_14999_global_step_15000 filter=lfs diff=lfs merge=lfs -text
38
+ Run_3/checkpoint_epoch_0_step_17499_global_step_17500 filter=lfs diff=lfs merge=lfs -text
39
+ Run_3/checkpoint_epoch_0_step_19999_global_step_20000 filter=lfs diff=lfs merge=lfs -text
40
+ Run_3/checkpoint_epoch_0_step_2499_global_step_2500 filter=lfs diff=lfs merge=lfs -text
41
+ Run_3/checkpoint_epoch_0_step_4999_global_step_5000 filter=lfs diff=lfs merge=lfs -text
42
+ Run_3/checkpoint_epoch_0_step_7499_global_step_7500 filter=lfs diff=lfs merge=lfs -text
43
+ Run_3/checkpoint_epoch_0_step_9999_global_step_10000 filter=lfs diff=lfs merge=lfs -text
44
+ Run_3/checkpoint_epoch_1_step_11136_global_step_32500 filter=lfs diff=lfs merge=lfs -text
45
+ Run_3/checkpoint_epoch_1_step_1136_global_step_22500 filter=lfs diff=lfs merge=lfs -text
46
+ Run_3/checkpoint_epoch_1_step_13636_global_step_35000 filter=lfs diff=lfs merge=lfs -text
47
+ Run_3/checkpoint_epoch_1_step_16136_global_step_37500 filter=lfs diff=lfs merge=lfs -text
48
+ Run_3/checkpoint_epoch_1_step_18636_global_step_40000 filter=lfs diff=lfs merge=lfs -text
49
+ Run_3/checkpoint_epoch_1_step_21136_global_step_42500 filter=lfs diff=lfs merge=lfs -text
50
+ Run_3/checkpoint_epoch_1_step_3636_global_step_25000 filter=lfs diff=lfs merge=lfs -text
51
+ Run_3/checkpoint_epoch_1_step_6136_global_step_27500 filter=lfs diff=lfs merge=lfs -text
52
+ Run_3/checkpoint_epoch_1_step_8636_global_step_30000 filter=lfs diff=lfs merge=lfs -text
53
+ Run_3/checkpoint_epoch_2_step_12273_global_step_55000 filter=lfs diff=lfs merge=lfs -text
54
+ Run_3/checkpoint_epoch_2_step_14773_global_step_57500 filter=lfs diff=lfs merge=lfs -text
55
+ Run_3/checkpoint_epoch_2_step_17273_global_step_60000 filter=lfs diff=lfs merge=lfs -text
56
+ Run_3/checkpoint_epoch_2_step_19773_global_step_62500 filter=lfs diff=lfs merge=lfs -text
57
+ Run_3/checkpoint_epoch_2_step_2273_global_step_45000 filter=lfs diff=lfs merge=lfs -text
58
+ Run_3/checkpoint_epoch_2_step_4773_global_step_47500 filter=lfs diff=lfs merge=lfs -text
59
+ Run_3/checkpoint_epoch_2_step_7273_global_step_50000 filter=lfs diff=lfs merge=lfs -text
60
+ Run_3/checkpoint_epoch_2_step_9773_global_step_52500 filter=lfs diff=lfs merge=lfs -text
61
+ Run_3/checkpoint_epoch_3_step_10910_global_step_75000 filter=lfs diff=lfs merge=lfs -text
62
+ Run_3/checkpoint_epoch_3_step_13410_global_step_77500 filter=lfs diff=lfs merge=lfs -text
63
+ Run_3/checkpoint_epoch_3_step_15910_global_step_80000 filter=lfs diff=lfs merge=lfs -text
64
+ Run_3/checkpoint_epoch_3_step_18410_global_step_82500 filter=lfs diff=lfs merge=lfs -text
65
+ Run_3/checkpoint_epoch_3_step_3410_global_step_67500 filter=lfs diff=lfs merge=lfs -text
66
+ Run_3/checkpoint_epoch_3_step_5910_global_step_70000 filter=lfs diff=lfs merge=lfs -text
67
+ Run_3/checkpoint_epoch_3_step_8410_global_step_72500 filter=lfs diff=lfs merge=lfs -text
68
+ Run_3/checkpoint_epoch_3_step_910_global_step_65000 filter=lfs diff=lfs merge=lfs -text
Run_3/checkpoint_epoch_0_step_12499_global_step_12500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:088d68613d6845085d8444f5d7800e8b0069e9021b014e0a59f5b2bf7533d0a0
3
+ size 234482304
Run_3/checkpoint_epoch_0_step_14999_global_step_15000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2187dfad79e4d51809535518934fe817bfefd472c322440d2b664e1c643a31b7
3
+ size 234482304
Run_3/checkpoint_epoch_0_step_17499_global_step_17500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ece0d9ab451a88e95e2ce06be6c45f39e42e346e8c7b02bc0f4794dbc3975ea
3
+ size 234482304
Run_3/checkpoint_epoch_0_step_19999_global_step_20000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b17e8fecb90c86cb290182be4925dd11ce8643e89aed869f45d9383330c7df93
3
+ size 234482304
Run_3/checkpoint_epoch_0_step_2499_global_step_2500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:249cbcbf1bb6fed35030c16aa7fc9dcac2cd4d3e62d572e7b2671b4cd0dcfabb
3
+ size 234481886
Run_3/checkpoint_epoch_0_step_4999_global_step_5000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d8f07df4268f9bd226ba12f5f5e0b0013b98a002a2413c2bb7e736a2b054459
3
+ size 234481886
Run_3/checkpoint_epoch_0_step_7499_global_step_7500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5fb317611d71c5bdfd66d0c9b58c625a7a2a841f9920a23498309e713c276e8
3
+ size 234481886
Run_3/checkpoint_epoch_0_step_9999_global_step_10000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c95cb0291cc15ed3936a8949879facfa6a4d074efbca25d3e82e60d679a20355
3
+ size 234482095
Run_3/checkpoint_epoch_1_step_11136_global_step_32500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a35c087817954b1969d35e07d4caafdf8a14b540d2d492417b554eefdddf7116
3
+ size 234482304
Run_3/checkpoint_epoch_1_step_1136_global_step_22500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50460e5b8bc5101098d8581d66a7cc7cd6aa3770b5fa185638fd57f1218c10ca
3
+ size 234482095
Run_3/checkpoint_epoch_1_step_13636_global_step_35000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b82040cb17ded2d6aaac3126a8c483fd4c3bde6fe0eaa1932603bd70bc061ca
3
+ size 234482304
Run_3/checkpoint_epoch_1_step_16136_global_step_37500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00756fc379c4ab110f0eaa5d22c9f0d5d89d20e0d998747395b9cec65b0d08d7
3
+ size 234482304
Run_3/checkpoint_epoch_1_step_18636_global_step_40000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58b47b6d6afbdb5c241f1e64524f012b3d4a872892407491ab96954ffdc2f42c
3
+ size 234482304
Run_3/checkpoint_epoch_1_step_21136_global_step_42500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5aedc28b34811e0017573726460fa02e8d80915c7cb291c1f655e6d11f31bab6
3
+ size 234482304
Run_3/checkpoint_epoch_1_step_3636_global_step_25000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:db0f1e512dc7b16c43599f90fb4ca0f8e615c3d55a49624e10fbd0f9862ab70c
3
+ size 234482095
Run_3/checkpoint_epoch_1_step_6136_global_step_27500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:499f65d2413b3f6aee76c683f70f1e4a1a0657588693b2726a94de5a757a4ee3
3
+ size 234482095
Run_3/checkpoint_epoch_1_step_8636_global_step_30000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bba74599dd844f1388278b4ee1f61e96c551c541c3a4d634ca70d9e7e0892848
3
+ size 234482095
Run_3/checkpoint_epoch_2_step_12273_global_step_55000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33249f791cadd0e9a50832a2533dbfa3353fc94ce0a6fce7bb6a19827dedb224
3
+ size 234482304
Run_3/checkpoint_epoch_2_step_14773_global_step_57500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b772c086b38e61ed5a4f9477c9103900dd1441547b3da5a6e5bf19ed846bcc76
3
+ size 234482304
Run_3/checkpoint_epoch_2_step_17273_global_step_60000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30e0418d2b2a04f7a3434322d6d045c5b3ce98318b6f6804a6edca08958b8430
3
+ size 234482304
Run_3/checkpoint_epoch_2_step_19773_global_step_62500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab5bc7e11f048453590da23466474f2c87a013b16771d941ac166fecf940bd32
3
+ size 234482304
Run_3/checkpoint_epoch_2_step_2273_global_step_45000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da2a25f700845dc0cd3984f8a7f657e904b490f1783bc9d4f2b6e3b0e11f03fa
3
+ size 234482095
Run_3/checkpoint_epoch_2_step_4773_global_step_47500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b57a9fec88267eeccb4a83b0953a4d42d4518f7ef6a184f7b421b90af3a49321
3
+ size 234482095
Run_3/checkpoint_epoch_2_step_7273_global_step_50000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d567da7f4cc51af1ba48405d9d5bda9b332dc94409709b703b321d1729c5aacb
3
+ size 234482095
Run_3/checkpoint_epoch_2_step_9773_global_step_52500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae2cdb2b2be6ba51ffedc0934771c76e26b2de608d6bfebfdd6cf9ccbf43bccb
3
+ size 234482095
Run_3/checkpoint_epoch_3_step_10910_global_step_75000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ed0557b22dacf2bf9a00b530bcecd9df52cd97366468771c86db13031521ab8
3
+ size 234482304
Run_3/checkpoint_epoch_3_step_13410_global_step_77500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0eadb92478b3db504609dc11414afe2a5dfec5c9bd99a376e2d36aa0a0e7c09
3
+ size 234482304
Run_3/checkpoint_epoch_3_step_15910_global_step_80000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:afb4fbdc322fde7f44ca8c40406c2d224be72537c79e4c27022d5cd5ec2f973b
3
+ size 234482304
Run_3/checkpoint_epoch_3_step_18410_global_step_82500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3ccb5296549b46240d80ffa05d973121eb513d18d432dbf02f23a6ec53ac3f9
3
+ size 234482304
Run_3/checkpoint_epoch_3_step_3410_global_step_67500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2341deaba97f77eba15660fac775de1dd880c6f43a518df394cdd23830e77c95
3
+ size 234482095
Run_3/checkpoint_epoch_3_step_5910_global_step_70000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f635db26a059f3e7e4c642ba5fe29e6c29267f20569dcc049abce3da53db887
3
+ size 234482095
Run_3/checkpoint_epoch_3_step_8410_global_step_72500 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:318782634f0a200831d6fe6b8a725708b69e1f9c9c2e543754c7137acdc7043f
3
+ size 234482095
Run_3/checkpoint_epoch_3_step_910_global_step_65000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df799ad3332d075862e71868c3506ef3aeea8c5d711a329aea51c7698bb9cffc
3
+ size 234481886
Run_3/run_3_config/configs.txt ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ --- wandb_config.json ---
2
+ {
3
+ "wandb_": true,
4
+ "api_key": "adb8e27fcf9ff0f8d92bfd422ade4d77b7e5232e",
5
+ "project": "SLM-43.7M",
6
+ "dir": "wandb_cache/",
7
+ "id": "Official-Run-3",
8
+ "name": "Official-Run-3"
9
+ }
10
+
11
+ --- train_config.json ---
12
+ {
13
+ "total_epochs": 10,
14
+ "mixed_precision": false,
15
+ "save_checkpoint_path": "checkpoints/saved_cache/",
16
+ "load_checkpoint_path": false,
17
+ "save_checkpoint_steps": 2500,
18
+ "clip_grad_norm": true,
19
+ "max_grad_norm": 1,
20
+ "track_grad_norm": true,
21
+ "val_data_path": "data/tensors/val/",
22
+ "val_steps": 5000,
23
+ "val_size": 500,
24
+ "parallel": true,
25
+ "seed": 0,
26
+ "batch_size": 16,
27
+ "pad_token_id": 1,
28
+ "run": 3
29
+ }
30
+
31
+ --- parallel_config.json ---
32
+ {
33
+ "parallelism_type": "fsdp",
34
+ "backend": "nccl",
35
+ "min_num_params": "1_000_000",
36
+ "type_": "transformer"
37
+ }
38
+
39
+ --- opt_config.json ---
40
+ {
41
+ "lr": 0.0005,
42
+ "betas": [
43
+ 0.9,
44
+ 0.999
45
+ ],
46
+ "eps": 1e-08,
47
+ "weight_decay": 0.01
48
+ }
49
+
50
+ --- model_config.json ---
51
+ {
52
+ "d_model": 512,
53
+ "embed_dim": 512,
54
+ "max_seq_len": 512,
55
+ "dropout_p": 0.1,
56
+ "n_heads": 8,
57
+ "n_blocks": 4,
58
+ "context_len": 512,
59
+ "vocab_size": 10000,
60
+ "pretrained_embeddings": false,
61
+ "sliding_window": 256,
62
+ "learned_pe": false,
63
+ "freeze_embeddings": false
64
+ }
65
+
66
+ --- lr_config.json ---
67
+ {
68
+ "warmup_steps": 750,
69
+ "constant_steps": 0,
70
+ "decay_steps": 409870,
71
+ "max_lr": 0.0001,
72
+ "min_lr": 1e-05
73
+ }
74
+
75
+ --- loss_config.json ---
76
+ {
77
+ "reduction": "none",
78
+ "ignore_index": 1
79
+ }
80
+
81
+ --- dataloader_config.json ---
82
+ {
83
+ "list_": true,
84
+ "shuffle": false,
85
+ "train_tensor_path": "data/tensors/train",
86
+ "verbose": true,
87
+ "pin_memory": true,
88
+ "num_workers": 24
89
+ }
90
+