Hugging Face's logo

shresht8
/
nanochat_experiments

Model card Files
xet
 Community
shresht8 commited on
Commit
2edfcba
·
verified ·
1 Parent(s): a4ee1c3

Upload folder using huggingface_hub

Browse files
Files changed (10) hide show
  1. meta_005568.json +58 -0
  2. model_005568.pt +3 -0
  3. optim_005568_rank0.pt +3 -0
  4. optim_005568_rank1.pt +3 -0
  5. optim_005568_rank2.pt +3 -0
  6. optim_005568_rank3.pt +3 -0
  7. optim_005568_rank4.pt +3 -0
  8. optim_005568_rank5.pt +3 -0
  9. optim_005568_rank6.pt +3 -0
  10. optim_005568_rank7.pt +3 -0
meta_005568.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 5568,
3
+ "val_bpb": 0.7164733549290998,
4
+ "model_config": {
5
+ "sequence_len": 2048,
6
+ "vocab_size": 32768,
7
+ "n_layer": 24,
8
+ "n_head": 12,
9
+ "n_kv_head": 12,
10
+ "n_embd": 1536,
11
+ "window_pattern": "SSSL"
12
+ },
13
+ "user_config": {
14
+ "run": "2",
15
+ "device_type": "",
16
+ "fp8": false,
17
+ "fp8_recipe": "tensorwise",
18
+ "depth": 24,
19
+ "aspect_ratio": 64,
20
+ "head_dim": 128,
21
+ "max_seq_len": 2048,
22
+ "window_pattern": "SSSL",
23
+ "num_iterations": -1,
24
+ "target_flops": -1.0,
25
+ "target_param_data_ratio": 8.0,
26
+ "device_batch_size": 8,
27
+ "total_batch_size": -1,
28
+ "embedding_lr": 0.3,
29
+ "unembedding_lr": 0.008,
30
+ "weight_decay": 0.28,
31
+ "matrix_lr": 0.02,
32
+ "scalar_lr": 0.5,
33
+ "warmup_steps": 40,
34
+ "warmdown_ratio": 0.65,
35
+ "final_lr_frac": 0.05,
36
+ "resume_from_step": -1,
37
+ "eval_every": 250,
38
+ "eval_tokens": 41943040,
39
+ "core_metric_every": 2000,
40
+ "core_metric_max_per_task": 500,
41
+ "sample_every": 2000,
42
+ "save_every": -1,
43
+ "model_tag": null
44
+ },
45
+ "device_batch_size": 8,
46
+ "max_seq_len": 2048,
47
+ "total_batch_size": 1048576,
48
+ "dataloader_state_dict": {
49
+ "pq_idx": 122,
50
+ "rg_idx": 80,
51
+ "epoch": 1
52
+ },
53
+ "loop_state": {
54
+ "min_val_bpb": 0.7164733549290998,
55
+ "smooth_train_loss": 2.3555553397426117,
56
+ "total_training_time": 26996.598171949387
57
+ }
58
+ }
model_005568.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ca4b1957a9a53f3106164c48f223ce392ab27f3070551f94ad13dfd1cd99637
3
+ size 4227935530
optim_005568_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65c39c7c641b26e397cae4dd0cea6548bfa8826c01fe5a694f5b317452633fcf
3
+ size 717465109
optim_005568_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4119a7f380c6b414d72a5fc2db04c4ffae6223c0530e3fc8575ea6dde587b75a
3
+ size 717465109
optim_005568_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01e14fe360fce7dffc86fdbc1f84b902e27b079e2cac0cf4d2f4c9f49b1632cc
3
+ size 717465109
optim_005568_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:937894faeb36c58718f998c86db87a52d09494a7f565965dc8fc9ace5a44ec5d
3
+ size 717465109
optim_005568_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3227c58550f0efa7885d2014255324b61e3636eca3ea359ecb2e163f7fd4df48
3
+ size 717465109
optim_005568_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e3f6a797d0e5c798010e71ce58a556046ac46c1735fc5b85b5de2d4337c91d8
3
+ size 717465109
optim_005568_rank6.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f074d7fc62f7d84354cf66aa8a47d8022bf1100ffaf6fbd0debade5cd47a441
3
+ size 717465109
optim_005568_rank7.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c8a1b3b1fe547f8118eeb4f1e4dd27a01adf0ef719883f32a249b7f9b3292f6
3
+ size 717465109