Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
c59faac
·
verified ·
1 Parent(s): be6e336

Upload paper checkpoint l8k1p2b_dense_seed71 checkpoint-50462

Browse files
Files changed (15) hide show
  1. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/config.json +37 -0
  2. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank0.pt +3 -0
  3. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank1.pt +3 -0
  4. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank2.pt +3 -0
  5. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank3.pt +3 -0
  6. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank4.pt +3 -0
  7. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank5.pt +3 -0
  8. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/model.pt +3 -0
  9. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank0.pt +3 -0
  10. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank1.pt +3 -0
  11. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank2.pt +3 -0
  12. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank3.pt +3 -0
  13. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank4.pt +3 -0
  14. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank5.pt +3 -0
  15. checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/training_state.pt +3 -0
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "rope_train_position_scale_set": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151643,
22
+ "checkpoint_activations": false,
23
+ "attention_type": "sparse_stochastic",
24
+ "sparse_block_size": 128,
25
+ "sparse_num_random_blocks": 31,
26
+ "sparse_mask_pool_size": 1,
27
+ "sparse_max_seq_length": 8192,
28
+ "sparse_layer_indices": null,
29
+ "sparse_mask_cache_dir": null,
30
+ "sparse_mask_seed": 0,
31
+ "sparse_mixed_nrb_set": null,
32
+ "moba_block_size": 128,
33
+ "moba_top_k": 32,
34
+ "asentmax_alpha": 1.5,
35
+ "asentmax_learn_temp": true,
36
+ "asentmax_init_temp": 1.0
37
+ }
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a52ec8b470b53d99a4266717af513dc1335821eeccae71d7b1c741ea30fe7ecf
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4541d1270159558df57f6c4e0010eba014b22528730f001f281418c58c04068a
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cf7e766b6e265a6ca4408e0b3f07da4b27afda571643a5ed4bb29203e9ed62e
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63240e883e17d9a5d38fb7fd9fad2e0cc50b4e2a0e13b5d39232b6c19795a4f1
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfbfec0835c96b36ef9ba70883464f2f6d43f18c86b9fcd7ec1edf04672cb51e
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/loader_gen_state_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:998aa5e419a1db81568bcb5727cd5ed68729ea9492d8f6f1dedd6517fc295b01
3
+ size 6802
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:95c122886a4d972d0eb50fe7989de994b4bde8ba65cb424780b2d3152dffa764
3
+ size 2404819483
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ccee5b4e7fd58a12a990fd9538c1aed245a45bd5556b01d9c06bf0535cdba35
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f4515e21c45da4604daf9d38399e2467cbf918534d4efb755984c8709258647
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51c79c0d97b5e77fbc0ab241c9db6f5131bcad2ceb4a5e579c21cce4380dd6bc
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b8624de07d8164f12d11113519e8be8e9be334336d1a12ac562cace82d7d0b0
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9276907d9e449ef84e3cfe61d989b3d6b0350d4686d993281442e142e6ebde15
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/rng_state_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59f711fe1d5cc92c88db3e5a29b6fe6dbbff5f774a0b9b05163466b4d009f439
3
+ size 14821
checkpoints/scale_1p2b_current/l8k1p2b_dense_seed71/checkpoint-50462/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8431de880e79c90a904c23eb755325fd1491a662f6aba37f150c7ffc2b2750da
3
+ size 4809681879