Upload folder using huggingface_hub

checkpoint-1000/chat_template.jinja

@@ -0,0 +1,9 @@
+{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'user' %}{{ '<|user|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
+'  + message['content'] + eos_token + '
+' }}{% else %}{{ '<|assistant|>
+'  + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
+' }}{% endif %}{% endfor %}

checkpoint-1000/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.56.1",
+  "use_cache": false,
+  "vocab_size": 100352
+}

checkpoint-1000/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.56.1"
+}

checkpoint-1000/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05ae91e1171f1f1753b23df3a5a6740d89da377932b33a1661bb7485d6ec530e
+size 4970591184

checkpoint-1000/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d41b81fe0d7e4c1d673a77beded5ee77080e39d6a6940b523e82f61eaa1dcbd
+size 4981161496

checkpoint-1000/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66448f6324b6e5dca7ef3623185f2f94dc402e34f8862c196a850e0e8119ed05
+size 4645523448

checkpoint-1000/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e831a92af07981d35b9c2c3681d1a979275ca6f41306b6f1551c72613d87619
+size 3263319914

checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdf49fbe089c54734b0cb49eebe2b139e2eb7e2ba668acc6d856b30fcc0384d7
+size 14645

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f23267b381a42784999e08fdda207042bbe3b9bc2b38cecbe7bd208dc572c4ab
+size 1465

checkpoint-1000/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,189 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-1000/trainer_state.json

@@ -0,0 +1,1474 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.8,
+  "eval_steps": 100,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.6430254079401493,
+      "epoch": 0.008,
+      "grad_norm": 4.53125,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 1.8435,
+      "mean_token_accuracy": 0.6683766476809978,
+      "num_tokens": 133939.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.6358832061290741,
+      "epoch": 0.016,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 1.8392,
+      "mean_token_accuracy": 0.6651193369179964,
+      "num_tokens": 273600.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.6754261270165444,
+      "epoch": 0.024,
+      "grad_norm": 4.9375,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 1.9101,
+      "mean_token_accuracy": 0.6521823685616255,
+      "num_tokens": 408034.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.6558344909921289,
+      "epoch": 0.032,
+      "grad_norm": 4.28125,
+      "learning_rate": 2.6e-06,
+      "loss": 1.8472,
+      "mean_token_accuracy": 0.6629668604582548,
+      "num_tokens": 542836.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.6494104014709592,
+      "epoch": 0.04,
+      "grad_norm": 3.8125,
+      "learning_rate": 3.266666666666667e-06,
+      "loss": 1.8027,
+      "mean_token_accuracy": 0.6676773574203253,
+      "num_tokens": 678419.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.662838158942759,
+      "epoch": 0.048,
+      "grad_norm": 4.03125,
+      "learning_rate": 3.9333333333333335e-06,
+      "loss": 1.8213,
+      "mean_token_accuracy": 0.6647731238976121,
+      "num_tokens": 813510.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.6506876813247799,
+      "epoch": 0.056,
+      "grad_norm": 3.71875,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 1.7955,
+      "mean_token_accuracy": 0.6665123915299773,
+      "num_tokens": 954682.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.6252049600705505,
+      "epoch": 0.064,
+      "grad_norm": 3.65625,
+      "learning_rate": 5.2666666666666665e-06,
+      "loss": 1.7861,
+      "mean_token_accuracy": 0.6728611093014478,
+      "num_tokens": 1096071.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.6236795110628008,
+      "epoch": 0.072,
+      "grad_norm": 3.171875,
+      "learning_rate": 5.933333333333335e-06,
+      "loss": 1.6458,
+      "mean_token_accuracy": 0.6831523377448321,
+      "num_tokens": 1239910.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.6565364092588425,
+      "epoch": 0.08,
+      "grad_norm": 3.125,
+      "learning_rate": 6.600000000000001e-06,
+      "loss": 1.6948,
+      "mean_token_accuracy": 0.6728412076830864,
+      "num_tokens": 1376045.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_biology_entropy": 1.821127786874771,
+      "eval_biology_loss": 4.8697123527526855,
+      "eval_biology_mean_token_accuracy": 0.3407035154700279,
+      "eval_biology_num_tokens": 1376045.0,
+      "eval_biology_runtime": 26.9612,
+      "eval_biology_samples_per_second": 18.545,
+      "eval_biology_steps_per_second": 9.273,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_chemistry_entropy": 1.1214371861219405,
+      "eval_chemistry_loss": 2.9583740234375,
+      "eval_chemistry_mean_token_accuracy": 0.5230503298044205,
+      "eval_chemistry_num_tokens": 1376045.0,
+      "eval_chemistry_runtime": 34.4743,
+      "eval_chemistry_samples_per_second": 14.504,
+      "eval_chemistry_steps_per_second": 7.252,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_math_entropy": 0.6455623766183853,
+      "eval_math_loss": 1.6610009670257568,
+      "eval_math_mean_token_accuracy": 0.6816041300296783,
+      "eval_math_num_tokens": 1376045.0,
+      "eval_math_runtime": 34.9863,
+      "eval_math_samples_per_second": 14.291,
+      "eval_math_steps_per_second": 7.146,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_physics_entropy": 0.8894976880550385,
+      "eval_physics_loss": 2.340852975845337,
+      "eval_physics_mean_token_accuracy": 0.5970065190792083,
+      "eval_physics_num_tokens": 1376045.0,
+      "eval_physics_runtime": 40.5871,
+      "eval_physics_samples_per_second": 12.319,
+      "eval_physics_steps_per_second": 6.16,
+      "step": 100
+    },
+    {
+      "entropy": 0.6108825646340847,
+      "epoch": 0.088,
+      "grad_norm": 3.0625,
+      "learning_rate": 7.266666666666668e-06,
+      "loss": 1.534,
+      "mean_token_accuracy": 0.6979989364743233,
+      "num_tokens": 1518257.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.6626681972295045,
+      "epoch": 0.096,
+      "grad_norm": 3.0,
+      "learning_rate": 7.933333333333334e-06,
+      "loss": 1.4858,
+      "mean_token_accuracy": 0.6921830836683511,
+      "num_tokens": 1656215.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.6807309044525027,
+      "epoch": 0.104,
+      "grad_norm": 2.4375,
+      "learning_rate": 8.6e-06,
+      "loss": 1.3897,
+      "mean_token_accuracy": 0.7023331839591265,
+      "num_tokens": 1791507.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.7359636599197984,
+      "epoch": 0.112,
+      "grad_norm": 1.9765625,
+      "learning_rate": 9.266666666666667e-06,
+      "loss": 1.3356,
+      "mean_token_accuracy": 0.7036307867616415,
+      "num_tokens": 1928153.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.8070579996332526,
+      "epoch": 0.12,
+      "grad_norm": 1.7265625,
+      "learning_rate": 9.933333333333334e-06,
+      "loss": 1.2682,
+      "mean_token_accuracy": 0.712115453183651,
+      "num_tokens": 2071008.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.874446533434093,
+      "epoch": 0.128,
+      "grad_norm": 1.2734375,
+      "learning_rate": 1.0600000000000002e-05,
+      "loss": 1.1443,
+      "mean_token_accuracy": 0.7246435590088367,
+      "num_tokens": 2216042.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.9639016635715961,
+      "epoch": 0.136,
+      "grad_norm": 1.125,
+      "learning_rate": 1.1266666666666668e-05,
+      "loss": 1.1005,
+      "mean_token_accuracy": 0.730505557358265,
+      "num_tokens": 2350589.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.0533128878101707,
+      "epoch": 0.144,
+      "grad_norm": 0.90234375,
+      "learning_rate": 1.1933333333333335e-05,
+      "loss": 1.0494,
+      "mean_token_accuracy": 0.7364416709169745,
+      "num_tokens": 2483116.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.0085792429745197,
+      "epoch": 0.152,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.2600000000000001e-05,
+      "loss": 0.9875,
+      "mean_token_accuracy": 0.749194710329175,
+      "num_tokens": 2617958.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.000020370259881,
+      "epoch": 0.16,
+      "grad_norm": 0.90234375,
+      "learning_rate": 1.3266666666666668e-05,
+      "loss": 0.9952,
+      "mean_token_accuracy": 0.7486346989870072,
+      "num_tokens": 2752258.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 3.2312401638031005,
+      "eval_biology_loss": 3.4984476566314697,
+      "eval_biology_mean_token_accuracy": 0.4106140378713608,
+      "eval_biology_num_tokens": 2752258.0,
+      "eval_biology_runtime": 26.9494,
+      "eval_biology_samples_per_second": 18.553,
+      "eval_biology_steps_per_second": 9.277,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_chemistry_entropy": 1.9263038992881776,
+      "eval_chemistry_loss": 1.9608160257339478,
+      "eval_chemistry_mean_token_accuracy": 0.5982974277734756,
+      "eval_chemistry_num_tokens": 2752258.0,
+      "eval_chemistry_runtime": 34.4173,
+      "eval_chemistry_samples_per_second": 14.528,
+      "eval_chemistry_steps_per_second": 7.264,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_math_entropy": 1.0257348996400832,
+      "eval_math_loss": 1.029754638671875,
+      "eval_math_mean_token_accuracy": 0.7436505243778229,
+      "eval_math_num_tokens": 2752258.0,
+      "eval_math_runtime": 34.9579,
+      "eval_math_samples_per_second": 14.303,
+      "eval_math_steps_per_second": 7.151,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_physics_entropy": 1.5005105144977569,
+      "eval_physics_loss": 1.5203113555908203,
+      "eval_physics_mean_token_accuracy": 0.6669992462396622,
+      "eval_physics_num_tokens": 2752258.0,
+      "eval_physics_runtime": 40.5453,
+      "eval_physics_samples_per_second": 12.332,
+      "eval_physics_steps_per_second": 6.166,
+      "step": 200
+    },
+    {
+      "entropy": 1.0170325668528677,
+      "epoch": 0.168,
+      "grad_norm": 0.80859375,
+      "learning_rate": 1.3933333333333334e-05,
+      "loss": 1.0032,
+      "mean_token_accuracy": 0.7469347029924392,
+      "num_tokens": 2889249.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.9709451261907815,
+      "epoch": 0.176,
+      "grad_norm": 0.7734375,
+      "learning_rate": 1.46e-05,
+      "loss": 0.9299,
+      "mean_token_accuracy": 0.7556545663625002,
+      "num_tokens": 3030305.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.0022083457559348,
+      "epoch": 0.184,
+      "grad_norm": 0.90234375,
+      "learning_rate": 1.5266666666666667e-05,
+      "loss": 0.9815,
+      "mean_token_accuracy": 0.7520246226340532,
+      "num_tokens": 3167096.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.9930278725922108,
+      "epoch": 0.192,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.5933333333333336e-05,
+      "loss": 0.9637,
+      "mean_token_accuracy": 0.7547307416796685,
+      "num_tokens": 3306293.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.9768950846046209,
+      "epoch": 0.2,
+      "grad_norm": 0.7734375,
+      "learning_rate": 1.66e-05,
+      "loss": 0.9373,
+      "mean_token_accuracy": 0.7594478782266378,
+      "num_tokens": 3441895.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.9833385257050395,
+      "epoch": 0.208,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.726666666666667e-05,
+      "loss": 0.9384,
+      "mean_token_accuracy": 0.7573206882923842,
+      "num_tokens": 3578349.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.9361576547846198,
+      "epoch": 0.216,
+      "grad_norm": 0.74609375,
+      "learning_rate": 1.7933333333333333e-05,
+      "loss": 0.9004,
+      "mean_token_accuracy": 0.7651511203497648,
+      "num_tokens": 3718484.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.9637048680335283,
+      "epoch": 0.224,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.86e-05,
+      "loss": 0.9349,
+      "mean_token_accuracy": 0.7605184197425843,
+      "num_tokens": 3851369.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.9367688067257405,
+      "epoch": 0.232,
+      "grad_norm": 0.74609375,
+      "learning_rate": 1.926666666666667e-05,
+      "loss": 0.8962,
+      "mean_token_accuracy": 0.7659803122282028,
+      "num_tokens": 3992977.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.9229113388806581,
+      "epoch": 0.24,
+      "grad_norm": 0.7890625,
+      "learning_rate": 1.9933333333333334e-05,
+      "loss": 0.9024,
+      "mean_token_accuracy": 0.77176120467484,
+      "num_tokens": 4133723.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_biology_entropy": 3.018038348674774,
+      "eval_biology_loss": 3.2969858646392822,
+      "eval_biology_mean_token_accuracy": 0.43762873542308806,
+      "eval_biology_num_tokens": 4133723.0,
+      "eval_biology_runtime": 27.1593,
+      "eval_biology_samples_per_second": 18.41,
+      "eval_biology_steps_per_second": 9.205,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_chemistry_entropy": 1.778470279932022,
+      "eval_chemistry_loss": 1.8064701557159424,
+      "eval_chemistry_mean_token_accuracy": 0.6220316555500031,
+      "eval_chemistry_num_tokens": 4133723.0,
+      "eval_chemistry_runtime": 34.6422,
+      "eval_chemistry_samples_per_second": 14.433,
+      "eval_chemistry_steps_per_second": 7.217,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_math_entropy": 0.9541566734313964,
+      "eval_math_loss": 0.9326547980308533,
+      "eval_math_mean_token_accuracy": 0.7629979259967804,
+      "eval_math_num_tokens": 4133723.0,
+      "eval_math_runtime": 35.1904,
+      "eval_math_samples_per_second": 14.208,
+      "eval_math_steps_per_second": 7.104,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_physics_entropy": 1.387200428724289,
+      "eval_physics_loss": 1.3932181596755981,
+      "eval_physics_mean_token_accuracy": 0.6886798795461655,
+      "eval_physics_num_tokens": 4133723.0,
+      "eval_physics_runtime": 40.7999,
+      "eval_physics_samples_per_second": 12.255,
+      "eval_physics_steps_per_second": 6.127,
+      "step": 300
+    },
+    {
+      "entropy": 0.9078424336388707,
+      "epoch": 0.248,
+      "grad_norm": 0.75,
+      "learning_rate": 1.9933333333333334e-05,
+      "loss": 0.8542,
+      "mean_token_accuracy": 0.7745326109230518,
+      "num_tokens": 4273949.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.9262684918940067,
+      "epoch": 0.256,
+      "grad_norm": 0.84765625,
+      "learning_rate": 1.985925925925926e-05,
+      "loss": 0.8761,
+      "mean_token_accuracy": 0.7701829686760903,
+      "num_tokens": 4414102.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.9303955681622028,
+      "epoch": 0.264,
+      "grad_norm": 0.78515625,
+      "learning_rate": 1.9785185185185187e-05,
+      "loss": 0.8765,
+      "mean_token_accuracy": 0.7714239094406367,
+      "num_tokens": 4548402.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.9207985673099757,
+      "epoch": 0.272,
+      "grad_norm": 0.7109375,
+      "learning_rate": 1.971111111111111e-05,
+      "loss": 0.8728,
+      "mean_token_accuracy": 0.7721738632768392,
+      "num_tokens": 4685739.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.9172879874706268,
+      "epoch": 0.28,
+      "grad_norm": 0.80078125,
+      "learning_rate": 1.963703703703704e-05,
+      "loss": 0.869,
+      "mean_token_accuracy": 0.7719811659306288,
+      "num_tokens": 4827264.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.8906141923740506,
+      "epoch": 0.288,
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.9562962962962964e-05,
+      "loss": 0.8676,
+      "mean_token_accuracy": 0.779513431340456,
+      "num_tokens": 4963900.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.9126916229724884,
+      "epoch": 0.296,
+      "grad_norm": 0.671875,
+      "learning_rate": 1.948888888888889e-05,
+      "loss": 0.8727,
+      "mean_token_accuracy": 0.7722058985382318,
+      "num_tokens": 5106308.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.9109785668551922,
+      "epoch": 0.304,
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.9414814814814817e-05,
+      "loss": 0.8562,
+      "mean_token_accuracy": 0.7752636637538671,
+      "num_tokens": 5246290.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.9183575309813022,
+      "epoch": 0.312,
+      "grad_norm": 0.71875,
+      "learning_rate": 1.9340740740740743e-05,
+      "loss": 0.8621,
+      "mean_token_accuracy": 0.7713671930134296,
+      "num_tokens": 5389824.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.8591190708801151,
+      "epoch": 0.32,
+      "grad_norm": 0.82421875,
+      "learning_rate": 1.926666666666667e-05,
+      "loss": 0.8065,
+      "mean_token_accuracy": 0.7856244947761297,
+      "num_tokens": 5531902.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 2.974372909069061,
+      "eval_biology_loss": 3.2137467861175537,
+      "eval_biology_mean_token_accuracy": 0.44984077060222627,
+      "eval_biology_num_tokens": 5531902.0,
+      "eval_biology_runtime": 26.9787,
+      "eval_biology_samples_per_second": 18.533,
+      "eval_biology_steps_per_second": 9.267,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_chemistry_entropy": 1.7318408770561218,
+      "eval_chemistry_loss": 1.746923804283142,
+      "eval_chemistry_mean_token_accuracy": 0.632696654677391,
+      "eval_chemistry_num_tokens": 5531902.0,
+      "eval_chemistry_runtime": 34.5046,
+      "eval_chemistry_samples_per_second": 14.491,
+      "eval_chemistry_steps_per_second": 7.245,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_math_entropy": 0.9170646283626557,
+      "eval_math_loss": 0.8942311406135559,
+      "eval_math_mean_token_accuracy": 0.7709064717292786,
+      "eval_math_num_tokens": 5531902.0,
+      "eval_math_runtime": 35.0734,
+      "eval_math_samples_per_second": 14.256,
+      "eval_math_steps_per_second": 7.128,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_physics_entropy": 1.349606383562088,
+      "eval_physics_loss": 1.343300223350525,
+      "eval_physics_mean_token_accuracy": 0.6978710038661957,
+      "eval_physics_num_tokens": 5531902.0,
+      "eval_physics_runtime": 40.7692,
+      "eval_physics_samples_per_second": 12.264,
+      "eval_physics_steps_per_second": 6.132,
+      "step": 400
+    },
+    {
+      "entropy": 0.9086290217936039,
+      "epoch": 0.328,
+      "grad_norm": 0.796875,
+      "learning_rate": 1.9192592592592593e-05,
+      "loss": 0.847,
+      "mean_token_accuracy": 0.7745547294616699,
+      "num_tokens": 5669047.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.9156009530648589,
+      "epoch": 0.336,
+      "grad_norm": 0.7578125,
+      "learning_rate": 1.911851851851852e-05,
+      "loss": 0.8769,
+      "mean_token_accuracy": 0.7692474499344826,
+      "num_tokens": 5806889.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.8934111742302775,
+      "epoch": 0.344,
+      "grad_norm": 0.67578125,
+      "learning_rate": 1.9044444444444446e-05,
+      "loss": 0.8462,
+      "mean_token_accuracy": 0.7796071924269199,
+      "num_tokens": 5947019.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.8856402298435568,
+      "epoch": 0.352,
+      "grad_norm": 0.7578125,
+      "learning_rate": 1.8970370370370372e-05,
+      "loss": 0.8362,
+      "mean_token_accuracy": 0.7777645040303469,
+      "num_tokens": 6083443.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.8964365001767873,
+      "epoch": 0.36,
+      "grad_norm": 0.7109375,
+      "learning_rate": 1.8896296296296295e-05,
+      "loss": 0.8406,
+      "mean_token_accuracy": 0.776063571870327,
+      "num_tokens": 6215195.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.8810398764908314,
+      "epoch": 0.368,
+      "grad_norm": 0.69921875,
+      "learning_rate": 1.8822222222222225e-05,
+      "loss": 0.8247,
+      "mean_token_accuracy": 0.7789717759937048,
+      "num_tokens": 6357305.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.8900227572768926,
+      "epoch": 0.376,
+      "grad_norm": 0.6875,
+      "learning_rate": 1.874814814814815e-05,
+      "loss": 0.8424,
+      "mean_token_accuracy": 0.7792754102498293,
+      "num_tokens": 6498815.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.8727197473868727,
+      "epoch": 0.384,
+      "grad_norm": 0.68359375,
+      "learning_rate": 1.8674074074074075e-05,
+      "loss": 0.8209,
+      "mean_token_accuracy": 0.7820516306906938,
+      "num_tokens": 6635429.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.8815433232113719,
+      "epoch": 0.392,
+      "grad_norm": 0.73828125,
+      "learning_rate": 1.86e-05,
+      "loss": 0.8274,
+      "mean_token_accuracy": 0.7792554065585137,
+      "num_tokens": 6774408.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.9402453400194645,
+      "epoch": 0.4,
+      "grad_norm": 0.78515625,
+      "learning_rate": 1.8525925925925928e-05,
+      "loss": 0.9268,
+      "mean_token_accuracy": 0.7666260946542025,
+      "num_tokens": 6914489.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_biology_entropy": 2.984537806510925,
+      "eval_biology_loss": 3.180291175842285,
+      "eval_biology_mean_token_accuracy": 0.4551255764961243,
+      "eval_biology_num_tokens": 6914489.0,
+      "eval_biology_runtime": 26.9382,
+      "eval_biology_samples_per_second": 18.561,
+      "eval_biology_steps_per_second": 9.281,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_chemistry_entropy": 1.735370470046997,
+      "eval_chemistry_loss": 1.7186591625213623,
+      "eval_chemistry_mean_token_accuracy": 0.6382190730571747,
+      "eval_chemistry_num_tokens": 6914489.0,
+      "eval_chemistry_runtime": 34.4276,
+      "eval_chemistry_samples_per_second": 14.523,
+      "eval_chemistry_steps_per_second": 7.262,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_math_entropy": 0.9168894953727722,
+      "eval_math_loss": 0.8749539852142334,
+      "eval_math_mean_token_accuracy": 0.7746825203895569,
+      "eval_math_num_tokens": 6914489.0,
+      "eval_math_runtime": 34.9026,
+      "eval_math_samples_per_second": 14.326,
+      "eval_math_steps_per_second": 7.163,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_physics_entropy": 1.34889164853096,
+      "eval_physics_loss": 1.3155016899108887,
+      "eval_physics_mean_token_accuracy": 0.7029694278240204,
+      "eval_physics_num_tokens": 6914489.0,
+      "eval_physics_runtime": 40.5298,
+      "eval_physics_samples_per_second": 12.337,
+      "eval_physics_steps_per_second": 6.168,
+      "step": 500
+    },
+    {
+      "entropy": 0.8826847270131111,
+      "epoch": 0.408,
+      "grad_norm": 0.6328125,
+      "learning_rate": 1.8451851851851855e-05,
+      "loss": 0.8191,
+      "mean_token_accuracy": 0.7815356869250536,
+      "num_tokens": 7053065.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.888327713869512,
+      "epoch": 0.416,
+      "grad_norm": 0.67578125,
+      "learning_rate": 1.8377777777777778e-05,
+      "loss": 0.8458,
+      "mean_token_accuracy": 0.7794966764748097,
+      "num_tokens": 7187783.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.8952456833794713,
+      "epoch": 0.424,
+      "grad_norm": 0.66015625,
+      "learning_rate": 1.8303703703703704e-05,
+      "loss": 0.8535,
+      "mean_token_accuracy": 0.7754339158535004,
+      "num_tokens": 7320375.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.8508788656443358,
+      "epoch": 0.432,
+      "grad_norm": 0.65625,
+      "learning_rate": 1.822962962962963e-05,
+      "loss": 0.7973,
+      "mean_token_accuracy": 0.7881547212600708,
+      "num_tokens": 7454401.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.8842971453443169,
+      "epoch": 0.44,
+      "grad_norm": 0.71484375,
+      "learning_rate": 1.8155555555555557e-05,
+      "loss": 0.8373,
+      "mean_token_accuracy": 0.7774394288659096,
+      "num_tokens": 7593764.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.8545080715790391,
+      "epoch": 0.448,
+      "grad_norm": 0.75,
+      "learning_rate": 1.8081481481481484e-05,
+      "loss": 0.8067,
+      "mean_token_accuracy": 0.7855051398277283,
+      "num_tokens": 7732710.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.8548783052712678,
+      "epoch": 0.456,
+      "grad_norm": 0.70703125,
+      "learning_rate": 1.800740740740741e-05,
+      "loss": 0.8214,
+      "mean_token_accuracy": 0.7835484184324741,
+      "num_tokens": 7873721.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.8870714059099555,
+      "epoch": 0.464,
+      "grad_norm": 0.5859375,
+      "learning_rate": 1.7933333333333333e-05,
+      "loss": 0.8303,
+      "mean_token_accuracy": 0.7804129023104907,
+      "num_tokens": 8012478.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.8966930942609906,
+      "epoch": 0.472,
+      "grad_norm": 0.64453125,
+      "learning_rate": 1.785925925925926e-05,
+      "loss": 0.8442,
+      "mean_token_accuracy": 0.7768670186400414,
+      "num_tokens": 8153429.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.8990984376519918,
+      "epoch": 0.48,
+      "grad_norm": 0.7265625,
+      "learning_rate": 1.7785185185185186e-05,
+      "loss": 0.8363,
+      "mean_token_accuracy": 0.7791902955621481,
+      "num_tokens": 8288086.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 3.026648554325104,
+      "eval_biology_loss": 3.1774065494537354,
+      "eval_biology_mean_token_accuracy": 0.45753026235103605,
+      "eval_biology_num_tokens": 8288086.0,
+      "eval_biology_runtime": 26.9914,
+      "eval_biology_samples_per_second": 18.524,
+      "eval_biology_steps_per_second": 9.262,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_chemistry_entropy": 1.7211642112731933,
+      "eval_chemistry_loss": 1.7037526369094849,
+      "eval_chemistry_mean_token_accuracy": 0.6409431176185608,
+      "eval_chemistry_num_tokens": 8288086.0,
+      "eval_chemistry_runtime": 34.4519,
+      "eval_chemistry_samples_per_second": 14.513,
+      "eval_chemistry_steps_per_second": 7.256,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_math_entropy": 0.8927879617214203,
+      "eval_math_loss": 0.863281786441803,
+      "eval_math_mean_token_accuracy": 0.7770743372440339,
+      "eval_math_num_tokens": 8288086.0,
+      "eval_math_runtime": 34.9704,
+      "eval_math_samples_per_second": 14.298,
+      "eval_math_steps_per_second": 7.149,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_physics_entropy": 1.3300827517509461,
+      "eval_physics_loss": 1.3015602827072144,
+      "eval_physics_mean_token_accuracy": 0.7059740360975265,
+      "eval_physics_num_tokens": 8288086.0,
+      "eval_physics_runtime": 40.5426,
+      "eval_physics_samples_per_second": 12.333,
+      "eval_physics_steps_per_second": 6.166,
+      "step": 600
+    },
+    {
+      "entropy": 0.8605974804610014,
+      "epoch": 0.488,
+      "grad_norm": 0.65234375,
+      "learning_rate": 1.7711111111111113e-05,
+      "loss": 0.8123,
+      "mean_token_accuracy": 0.7824984133243561,
+      "num_tokens": 8424958.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.8556231519207358,
+      "epoch": 0.496,
+      "grad_norm": 0.73828125,
+      "learning_rate": 1.763703703703704e-05,
+      "loss": 0.8073,
+      "mean_token_accuracy": 0.7834980070590973,
+      "num_tokens": 8562656.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.8707238981500268,
+      "epoch": 0.504,
+      "grad_norm": 0.63671875,
+      "learning_rate": 1.7562962962962962e-05,
+      "loss": 0.8323,
+      "mean_token_accuracy": 0.7818343795835971,
+      "num_tokens": 8706097.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.8535389142110944,
+      "epoch": 0.512,
+      "grad_norm": 0.6953125,
+      "learning_rate": 1.7488888888888892e-05,
+      "loss": 0.7829,
+      "mean_token_accuracy": 0.7894029818475247,
+      "num_tokens": 8844762.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.8502879558131099,
+      "epoch": 0.52,
+      "grad_norm": 0.73828125,
+      "learning_rate": 1.7414814814814815e-05,
+      "loss": 0.8016,
+      "mean_token_accuracy": 0.789192034304142,
+      "num_tokens": 8983268.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.841147867590189,
+      "epoch": 0.528,
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.7340740740740742e-05,
+      "loss": 0.7936,
+      "mean_token_accuracy": 0.7880374405533075,
+      "num_tokens": 9132725.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.8636940447613597,
+      "epoch": 0.536,
+      "grad_norm": 0.796875,
+      "learning_rate": 1.726666666666667e-05,
+      "loss": 0.841,
+      "mean_token_accuracy": 0.7832605082541704,
+      "num_tokens": 9271490.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.8671527728438377,
+      "epoch": 0.544,
+      "grad_norm": 0.78125,
+      "learning_rate": 1.7192592592592595e-05,
+      "loss": 0.8081,
+      "mean_token_accuracy": 0.7864414434880018,
+      "num_tokens": 9412327.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.9130071861669421,
+      "epoch": 0.552,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.711851851851852e-05,
+      "loss": 0.8653,
+      "mean_token_accuracy": 0.7723087813705206,
+      "num_tokens": 9541196.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.8585997836664319,
+      "epoch": 0.56,
+      "grad_norm": 0.7265625,
+      "learning_rate": 1.7044444444444445e-05,
+      "loss": 0.794,
+      "mean_token_accuracy": 0.7848813243210315,
+      "num_tokens": 9676595.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_biology_entropy": 2.9588126630783083,
+      "eval_biology_loss": 3.134213447570801,
+      "eval_biology_mean_token_accuracy": 0.46100987315177916,
+      "eval_biology_num_tokens": 9676595.0,
+      "eval_biology_runtime": 27.0044,
+      "eval_biology_samples_per_second": 18.516,
+      "eval_biology_steps_per_second": 9.258,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_chemistry_entropy": 1.6786610894203187,
+      "eval_chemistry_loss": 1.6839625835418701,
+      "eval_chemistry_mean_token_accuracy": 0.6436767561435699,
+      "eval_chemistry_num_tokens": 9676595.0,
+      "eval_chemistry_runtime": 34.4136,
+      "eval_chemistry_samples_per_second": 14.529,
+      "eval_chemistry_steps_per_second": 7.265,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_math_entropy": 0.8769606277942658,
+      "eval_math_loss": 0.8541995882987976,
+      "eval_math_mean_token_accuracy": 0.7790154383182526,
+      "eval_math_num_tokens": 9676595.0,
+      "eval_math_runtime": 34.9482,
+      "eval_math_samples_per_second": 14.307,
+      "eval_math_steps_per_second": 7.153,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_physics_entropy": 1.302037154197693,
+      "eval_physics_loss": 1.289175271987915,
+      "eval_physics_mean_token_accuracy": 0.7078491554260254,
+      "eval_physics_num_tokens": 9676595.0,
+      "eval_physics_runtime": 40.5408,
+      "eval_physics_samples_per_second": 12.333,
+      "eval_physics_steps_per_second": 6.167,
+      "step": 700
+    },
+    {
+      "entropy": 0.8488044740632177,
+      "epoch": 0.568,
+      "grad_norm": 0.65625,
+      "learning_rate": 1.697037037037037e-05,
+      "loss": 0.8027,
+      "mean_token_accuracy": 0.7859589237719774,
+      "num_tokens": 9812856.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.8703772563487291,
+      "epoch": 0.576,
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.6896296296296298e-05,
+      "loss": 0.8196,
+      "mean_token_accuracy": 0.7833330363035202,
+      "num_tokens": 9953227.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.8656591676175595,
+      "epoch": 0.584,
+      "grad_norm": 0.7109375,
+      "learning_rate": 1.6822222222222224e-05,
+      "loss": 0.8193,
+      "mean_token_accuracy": 0.7825856961309909,
+      "num_tokens": 10089276.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.8539155850186944,
+      "epoch": 0.592,
+      "grad_norm": 0.62109375,
+      "learning_rate": 1.6748148148148147e-05,
+      "loss": 0.8051,
+      "mean_token_accuracy": 0.7846140246838331,
+      "num_tokens": 10234220.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.8651883415877819,
+      "epoch": 0.6,
+      "grad_norm": 0.62109375,
+      "learning_rate": 1.6674074074074077e-05,
+      "loss": 0.8268,
+      "mean_token_accuracy": 0.7814744919538498,
+      "num_tokens": 10370667.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.863803020492196,
+      "epoch": 0.608,
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.66e-05,
+      "loss": 0.8229,
+      "mean_token_accuracy": 0.783318106085062,
+      "num_tokens": 10515760.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.8657625541090965,
+      "epoch": 0.616,
+      "grad_norm": 0.609375,
+      "learning_rate": 1.6525925925925927e-05,
+      "loss": 0.8201,
+      "mean_token_accuracy": 0.781252521276474,
+      "num_tokens": 10660578.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.8430647728964686,
+      "epoch": 0.624,
+      "grad_norm": 0.73828125,
+      "learning_rate": 1.6451851851851853e-05,
+      "loss": 0.786,
+      "mean_token_accuracy": 0.7895114913582801,
+      "num_tokens": 10797566.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.8240785686299205,
+      "epoch": 0.632,
+      "grad_norm": 0.63671875,
+      "learning_rate": 1.637777777777778e-05,
+      "loss": 0.7805,
+      "mean_token_accuracy": 0.7922219831496478,
+      "num_tokens": 10938456.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.8485349236056209,
+      "epoch": 0.64,
+      "grad_norm": 0.65625,
+      "learning_rate": 1.6303703703703706e-05,
+      "loss": 0.7989,
+      "mean_token_accuracy": 0.7866497144103051,
+      "num_tokens": 11079348.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 2.932964876174927,
+      "eval_biology_loss": 3.1179301738739014,
+      "eval_biology_mean_token_accuracy": 0.46290264546871185,
+      "eval_biology_num_tokens": 11079348.0,
+      "eval_biology_runtime": 27.0006,
+      "eval_biology_samples_per_second": 18.518,
+      "eval_biology_steps_per_second": 9.259,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_chemistry_entropy": 1.6625523691177369,
+      "eval_chemistry_loss": 1.6735429763793945,
+      "eval_chemistry_mean_token_accuracy": 0.645549393773079,
+      "eval_chemistry_num_tokens": 11079348.0,
+      "eval_chemistry_runtime": 34.504,
+      "eval_chemistry_samples_per_second": 14.491,
+      "eval_chemistry_steps_per_second": 7.246,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_math_entropy": 0.8715307763814926,
+      "eval_math_loss": 0.8475318551063538,
+      "eval_math_mean_token_accuracy": 0.780082686662674,
+      "eval_math_num_tokens": 11079348.0,
+      "eval_math_runtime": 34.9793,
+      "eval_math_samples_per_second": 14.294,
+      "eval_math_steps_per_second": 7.147,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_physics_entropy": 1.292870044708252,
+      "eval_physics_loss": 1.2783890962600708,
+      "eval_physics_mean_token_accuracy": 0.7100210604667664,
+      "eval_physics_num_tokens": 11079348.0,
+      "eval_physics_runtime": 40.5422,
+      "eval_physics_samples_per_second": 12.333,
+      "eval_physics_steps_per_second": 6.166,
+      "step": 800
+    },
+    {
+      "entropy": 0.8196414019912481,
+      "epoch": 0.648,
+      "grad_norm": 0.6484375,
+      "learning_rate": 1.622962962962963e-05,
+      "loss": 0.7785,
+      "mean_token_accuracy": 0.7903594326227903,
+      "num_tokens": 11221757.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.8491896335035563,
+      "epoch": 0.656,
+      "grad_norm": 0.66796875,
+      "learning_rate": 1.6155555555555556e-05,
+      "loss": 0.8188,
+      "mean_token_accuracy": 0.7861174423247576,
+      "num_tokens": 11360239.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.8492498191073536,
+      "epoch": 0.664,
+      "grad_norm": 0.83984375,
+      "learning_rate": 1.6081481481481482e-05,
+      "loss": 0.7993,
+      "mean_token_accuracy": 0.7853016339242458,
+      "num_tokens": 11498677.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.8525434667244554,
+      "epoch": 0.672,
+      "grad_norm": 0.69140625,
+      "learning_rate": 1.600740740740741e-05,
+      "loss": 0.8167,
+      "mean_token_accuracy": 0.7853171724826098,
+      "num_tokens": 11644674.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.8380661699920893,
+      "epoch": 0.68,
+      "grad_norm": 0.73046875,
+      "learning_rate": 1.5933333333333336e-05,
+      "loss": 0.7928,
+      "mean_token_accuracy": 0.788429806381464,
+      "num_tokens": 11783080.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.8381451951339841,
+      "epoch": 0.688,
+      "grad_norm": 0.62109375,
+      "learning_rate": 1.5859259259259262e-05,
+      "loss": 0.7921,
+      "mean_token_accuracy": 0.7877590708434582,
+      "num_tokens": 11917713.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.8567008957266807,
+      "epoch": 0.696,
+      "grad_norm": 0.6640625,
+      "learning_rate": 1.5785185185185185e-05,
+      "loss": 0.7966,
+      "mean_token_accuracy": 0.7848114792257548,
+      "num_tokens": 12058193.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.8353428209200502,
+      "epoch": 0.704,
+      "grad_norm": 0.65234375,
+      "learning_rate": 1.571111111111111e-05,
+      "loss": 0.7868,
+      "mean_token_accuracy": 0.7881882593035698,
+      "num_tokens": 12199067.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.8710033852607012,
+      "epoch": 0.712,
+      "grad_norm": 0.671875,
+      "learning_rate": 1.5637037037037038e-05,
+      "loss": 0.8088,
+      "mean_token_accuracy": 0.7816738404333592,
+      "num_tokens": 12335740.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.8482802568003536,
+      "epoch": 0.72,
+      "grad_norm": 0.8671875,
+      "learning_rate": 1.5562962962962965e-05,
+      "loss": 0.8002,
+      "mean_token_accuracy": 0.7863489333540201,
+      "num_tokens": 12469017.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_biology_entropy": 2.9198746318817137,
+      "eval_biology_loss": 3.095550775527954,
+      "eval_biology_mean_token_accuracy": 0.4650931693315506,
+      "eval_biology_num_tokens": 12469017.0,
+      "eval_biology_runtime": 26.9459,
+      "eval_biology_samples_per_second": 18.556,
+      "eval_biology_steps_per_second": 9.278,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_chemistry_entropy": 1.654184314250946,
+      "eval_chemistry_loss": 1.6625065803527832,
+      "eval_chemistry_mean_token_accuracy": 0.6471669510602951,
+      "eval_chemistry_num_tokens": 12469017.0,
+      "eval_chemistry_runtime": 34.4235,
+      "eval_chemistry_samples_per_second": 14.525,
+      "eval_chemistry_steps_per_second": 7.262,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_math_entropy": 0.8690718128681183,
+      "eval_math_loss": 0.8421436548233032,
+      "eval_math_mean_token_accuracy": 0.7811065847873688,
+      "eval_math_num_tokens": 12469017.0,
+      "eval_math_runtime": 34.9224,
+      "eval_math_samples_per_second": 14.317,
+      "eval_math_steps_per_second": 7.159,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_physics_entropy": 1.2865873792171478,
+      "eval_physics_loss": 1.2697285413742065,
+      "eval_physics_mean_token_accuracy": 0.7113078062534333,
+      "eval_physics_num_tokens": 12469017.0,
+      "eval_physics_runtime": 40.5051,
+      "eval_physics_samples_per_second": 12.344,
+      "eval_physics_steps_per_second": 6.172,
+      "step": 900
+    },
+    {
+      "entropy": 0.8559863271191717,
+      "epoch": 0.728,
+      "grad_norm": 0.859375,
+      "learning_rate": 1.548888888888889e-05,
+      "loss": 0.8087,
+      "mean_token_accuracy": 0.7872065585106611,
+      "num_tokens": 12604524.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.8351603880524635,
+      "epoch": 0.736,
+      "grad_norm": 0.69921875,
+      "learning_rate": 1.5414814814814814e-05,
+      "loss": 0.7815,
+      "mean_token_accuracy": 0.7907860331237316,
+      "num_tokens": 12746145.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.8385633688420058,
+      "epoch": 0.744,
+      "grad_norm": 0.68359375,
+      "learning_rate": 1.5340740740740744e-05,
+      "loss": 0.7849,
+      "mean_token_accuracy": 0.7884344782680273,
+      "num_tokens": 12882140.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.8265942361205816,
+      "epoch": 0.752,
+      "grad_norm": 0.671875,
+      "learning_rate": 1.5266666666666667e-05,
+      "loss": 0.7683,
+      "mean_token_accuracy": 0.7920810837298632,
+      "num_tokens": 13019785.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.801921004243195,
+      "epoch": 0.76,
+      "grad_norm": 0.64453125,
+      "learning_rate": 1.5192592592592594e-05,
+      "loss": 0.7567,
+      "mean_token_accuracy": 0.7981402266770601,
+      "num_tokens": 13167975.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.8200628351420164,
+      "epoch": 0.768,
+      "grad_norm": 0.6015625,
+      "learning_rate": 1.5118518518518519e-05,
+      "loss": 0.7712,
+      "mean_token_accuracy": 0.7937319375574589,
+      "num_tokens": 13307071.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.844639228284359,
+      "epoch": 0.776,
+      "grad_norm": 0.68359375,
+      "learning_rate": 1.5044444444444445e-05,
+      "loss": 0.8066,
+      "mean_token_accuracy": 0.7887587692588568,
+      "num_tokens": 13444206.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.8560901025310159,
+      "epoch": 0.784,
+      "grad_norm": 0.7109375,
+      "learning_rate": 1.497037037037037e-05,
+      "loss": 0.8159,
+      "mean_token_accuracy": 0.7839560698717832,
+      "num_tokens": 13586074.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.8473082648590207,
+      "epoch": 0.792,
+      "grad_norm": 0.60546875,
+      "learning_rate": 1.4896296296296298e-05,
+      "loss": 0.8055,
+      "mean_token_accuracy": 0.785613476485014,
+      "num_tokens": 13727642.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.8641716981306672,
+      "epoch": 0.8,
+      "grad_norm": 0.73828125,
+      "learning_rate": 1.4822222222222225e-05,
+      "loss": 0.8161,
+      "mean_token_accuracy": 0.7843089539557695,
+      "num_tokens": 13869252.0,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 2.927406606197357,
+      "eval_biology_loss": 3.0816211700439453,
+      "eval_biology_mean_token_accuracy": 0.46697849237918854,
+      "eval_biology_num_tokens": 13869252.0,
+      "eval_biology_runtime": 26.9151,
+      "eval_biology_samples_per_second": 18.577,
+      "eval_biology_steps_per_second": 9.288,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_chemistry_entropy": 1.6538746666908264,
+      "eval_chemistry_loss": 1.6552778482437134,
+      "eval_chemistry_mean_token_accuracy": 0.6485219527482986,
+      "eval_chemistry_num_tokens": 13869252.0,
+      "eval_chemistry_runtime": 34.4246,
+      "eval_chemistry_samples_per_second": 14.525,
+      "eval_chemistry_steps_per_second": 7.262,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_math_entropy": 0.8677517844438553,
+      "eval_math_loss": 0.8380001187324524,
+      "eval_math_mean_token_accuracy": 0.7821021761894226,
+      "eval_math_num_tokens": 13869252.0,
+      "eval_math_runtime": 34.92,
+      "eval_math_samples_per_second": 14.318,
+      "eval_math_steps_per_second": 7.159,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_physics_entropy": 1.2842487235069275,
+      "eval_physics_loss": 1.2633119821548462,
+      "eval_physics_mean_token_accuracy": 0.7128965456485749,
+      "eval_physics_num_tokens": 13869252.0,
+      "eval_physics_runtime": 40.4838,
+      "eval_physics_samples_per_second": 12.351,
+      "eval_physics_steps_per_second": 6.175,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.281015907772006e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:468bf7cb033f721ea40664a43923a7a349b3c291ee5b69f9acfdcec3e7250e85
+size 6481