Upload paper checkpoint l8k365m_dense_ropectl_a1p0_seed44 checkpoint-44000

checkpoints/control_rope_365m/l8k365m_dense_ropectl_a1p0_seed44/checkpoint-44000/config.json

@@ -0,0 +1,43 @@
+{
+  "vocab_size": 151936,
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_hidden_layers": 28,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "max_position_embeddings": 32768,
+  "initializer_range": 0.02,
+  "rms_norm_eps": 1e-06,
+  "use_cache": true,
+  "tie_word_embeddings": true,
+  "rope_theta": 1000000,
+  "rope_scaling": null,
+  "rope_train_position_scale_set": [
+    1.0,
+    0.9896339488176222,
+    0.9692159692159692,
+    0.9420289855072463,
+    0.8730158730158729
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "checkpoint_activations": false,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_max_seq_length": 8192,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}

checkpoints/control_rope_365m/l8k365m_dense_ropectl_a1p0_seed44/checkpoint-44000/loader_gen_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4050e1c23285b5ab02806005252fb5c5d57cc84365951a8d1f8caf328614ee38
+size 6802

checkpoints/control_rope_365m/l8k365m_dense_ropectl_a1p0_seed44/checkpoint-44000/loader_gen_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:741d13a2496875c23c79af48aa5ac707c8afbf56f135bf998d00b7373ad25ea9
+size 6802

checkpoints/control_rope_365m/l8k365m_dense_ropectl_a1p0_seed44/checkpoint-44000/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef7b4b4e8d501cabba3b681db531d0af2fd7bcb236060a0313f05e63c137522e
+size 718031899

checkpoints/control_rope_365m/l8k365m_dense_ropectl_a1p0_seed44/checkpoint-44000/rng_state_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc3d0e8e55e0a0e43312c684dc3b31403c0343063c977db9b943a47c0f1504cb
+size 14821

checkpoints/control_rope_365m/l8k365m_dense_ropectl_a1p0_seed44/checkpoint-44000/rng_state_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fd589f7a24f24bd89effb957ca599772ce85cd8cc5726f7b88604be3c410270
+size 14821

checkpoints/control_rope_365m/l8k365m_dense_ropectl_a1p0_seed44/checkpoint-44000/training_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0172ea73f73b7252c9f04f5f046da95eb47209a00b4ce9eeb4ce7c62e934067b
+size 1436105299