Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
fdfbf05
·
verified ·
1 Parent(s): d851b02

Upload paper checkpoint l30k1_2b_resampled checkpoint-37465

Browse files
Files changed (11) hide show
  1. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/config.json +30 -0
  2. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/loader_gen_state_rank0.pt +3 -0
  3. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/loader_gen_state_rank1.pt +3 -0
  4. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/loader_gen_state_rank2.pt +3 -0
  5. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/loader_gen_state_rank3.pt +3 -0
  6. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/model.pt +3 -0
  7. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/rng_state_rank0.pt +3 -0
  8. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/rng_state_rank1.pt +3 -0
  9. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/rng_state_rank2.pt +3 -0
  10. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/rng_state_rank3.pt +3 -0
  11. checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/training_state.pt +3 -0
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "attention_bias": false,
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151643,
21
+ "checkpoint_activations": true,
22
+ "attention_type": "sparse_stochastic",
23
+ "sparse_block_size": 128,
24
+ "sparse_num_random_blocks": 120,
25
+ "sparse_mask_pool_size": 1000,
26
+ "sparse_max_seq_length": 30720,
27
+ "sparse_layer_indices": null,
28
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120",
29
+ "sparse_mask_seed": 0
30
+ }
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f59a1416dd062038a7e2dbac8c330163f45326660faeb92a199f62460447539
3
+ size 6802
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9907050ee7aa7455a32aae231218aac0666dd0bc7b78294eeb2cb3adc77e63c8
3
+ size 6802
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3adc40f4c25919716a49dee06d626601e0abca351145ddf1ed45e0bbfca3ea0c
3
+ size 6802
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e32dfbaa9aad434214e9087e9315fae33fa445ebde26c320765a117f68a6465
3
+ size 6802
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93add5c1a6d279835d1fe44ae9378844173c7ab78932e4f2c24b295f72178d0b
3
+ size 2404819483
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f0adc019e458997c98bf4873b0b20294a4bd94a0e976c6320c4ada461144590
3
+ size 14821
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e97b18945786b1878a203dd8121dfa44a4a8d60fe3fd56bf95be6300773da02
3
+ size 14821
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a6b0c63045ac40369a1a231b9070d899f79195311f0319ab5abc22cd97ec535
3
+ size 14821
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:533ec1cebfad604c2a504b6b6e09616f9b97ecf4c01d10444688ef2007299f9a
3
+ size 14885
checkpoints/scale_1p2b_factorial/l30k1_2b_resampled/checkpoint-37465/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5fa5c053e10e55cf1945fba271872cb71b7b113edd671807695b3b866c130638
3
+ size 4809681879