Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
f16ddfc
·
verified ·
1 Parent(s): 20aef72

Upload paper checkpoint l8k3b_mixed_60b checkpoint-100000

Browse files
Files changed (15) hide show
  1. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/config.json +42 -0
  2. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank0.pt +3 -0
  3. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank1.pt +3 -0
  4. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank2.pt +3 -0
  5. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank3.pt +3 -0
  6. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank4.pt +3 -0
  7. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank5.pt +3 -0
  8. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/model.pt +3 -0
  9. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank0.pt +3 -0
  10. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank1.pt +3 -0
  11. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank2.pt +3 -0
  12. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank3.pt +3 -0
  13. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank4.pt +3 -0
  14. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank5.pt +3 -0
  15. checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/training_state.pt +3 -0
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 2304,
4
+ "intermediate_size": 7936,
5
+ "num_hidden_layers": 36,
6
+ "num_attention_heads": 24,
7
+ "num_key_value_heads": 8,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": false,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 31,
25
+ "sparse_mask_pool_size": 10000,
26
+ "sparse_max_seq_length": 8192,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
29
+ "sparse_mask_seed": 0,
30
+ "sparse_mixed_nrb_set": [
31
+ 63,
32
+ 47,
33
+ 31,
34
+ 21,
35
+ 11
36
+ ],
37
+ "moba_block_size": 128,
38
+ "moba_top_k": 32,
39
+ "asentmax_alpha": 1.5,
40
+ "asentmax_learn_temp": true,
41
+ "asentmax_init_temp": 1.0
42
+ }
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f59a1416dd062038a7e2dbac8c330163f45326660faeb92a199f62460447539
3
+ size 6802
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9907050ee7aa7455a32aae231218aac0666dd0bc7b78294eeb2cb3adc77e63c8
3
+ size 6802
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3adc40f4c25919716a49dee06d626601e0abca351145ddf1ed45e0bbfca3ea0c
3
+ size 6802
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e32dfbaa9aad434214e9087e9315fae33fa445ebde26c320765a117f68a6465
3
+ size 6802
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24107bc0f04d004ded30a79ee27372f06e83f1e0fe18048bf08f1f6c5cb844ae
3
+ size 6802
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/loader_gen_state_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38c7483ce8bdfde8534cd1d57902ee58030c706d74379a18f700c21898be27af
3
+ size 6802
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86e2b13ec1503caa51bcab2b14ead8e8418129436bb6a93c3bf09bf9f6c7eac0
3
+ size 6009048131
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99e82cb3a42e7902d66dcca15bf4f6624eb88a673a19ae0df71b1a4dd973b008
3
+ size 14821
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8449db24b7909dfa22329f2b4523c57cf3918cf9beffaa318bbec8efbc0aaf04
3
+ size 14885
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:80e4f31ebfd17adcb2db816833140da8d2281ea44d76e87ce8f0f932af8d058a
3
+ size 14821
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a01e7dc0c293797968253eef33cb98350b380a04881970dc9349981e1dfae1a7
3
+ size 14757
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:209c2e32577eea6708f56b95646eee51ec497797a279b79d7ef8ac4aa1af1951
3
+ size 14885
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/rng_state_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7eb1abc40a1a45a5b3548442b3035dede68e1d0dc1816305d76ff3e53ed06e16
3
+ size 14821
checkpoints/scale_3b_boundary/l8k3b_mixed_60b/checkpoint-100000/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:532b5aedc7640905e0334561b7e031b271253fb264e0e94363fae3c55ac0db95
3
+ size 12018140941