Upload folder using huggingface_hub

checkpoint-500/chat_template.jinja

@@ -0,0 +1,9 @@
+{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'user' %}{{ '<|user|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
+'  + message['content'] + eos_token + '
+' }}{% else %}{{ '<|assistant|>
+'  + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
+' }}{% endif %}{% endfor %}

checkpoint-500/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.56.1",
+  "use_cache": false,
+  "vocab_size": 100352
+}

checkpoint-500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.56.1"
+}

checkpoint-500/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05ae91e1171f1f1753b23df3a5a6740d89da377932b33a1661bb7485d6ec530e
+size 4970591184

checkpoint-500/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d41b81fe0d7e4c1d673a77beded5ee77080e39d6a6940b523e82f61eaa1dcbd
+size 4981161496

checkpoint-500/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:733a0ec9087cb54f6fe8fd3f8232b18a5f511855db1135fc1169c4b61fdde60d
+size 4645523448

checkpoint-500/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

checkpoint-500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8ea6c789b5f5b1b15c21573d30b70742faf419388f8b5f38ea1795ea7bed2e1
+size 3263319914

checkpoint-500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fde986f97ecc270a34049538e5bcb495edbc4e7f5914a76b3cad083de287dc73
+size 14645

checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddc00081d4f892d131a0d2b4d4827a214275362bfa62dfb8413f489c1bfad457
+size 1465

checkpoint-500/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-500/tokenizer_config.json

@@ -0,0 +1,189 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-500/trainer_state.json

@@ -0,0 +1,754 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.8,
+  "eval_steps": 100,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.6598687492311002,
+      "epoch": 0.016,
+      "grad_norm": 5.4375,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 3.4184,
+      "mean_token_accuracy": 0.4009484700858593,
+      "num_tokens": 198536.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.6535742968320846,
+      "epoch": 0.032,
+      "grad_norm": 5.28125,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 3.4159,
+      "mean_token_accuracy": 0.3995189921930432,
+      "num_tokens": 397151.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.6525759153068065,
+      "epoch": 0.048,
+      "grad_norm": 5.75,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 3.4205,
+      "mean_token_accuracy": 0.39969325363636016,
+      "num_tokens": 599281.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.6467667452991008,
+      "epoch": 0.064,
+      "grad_norm": 5.25,
+      "learning_rate": 2.6e-06,
+      "loss": 3.387,
+      "mean_token_accuracy": 0.40206527076661586,
+      "num_tokens": 804066.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.6710013456642627,
+      "epoch": 0.08,
+      "grad_norm": 5.375,
+      "learning_rate": 3.266666666666667e-06,
+      "loss": 3.3419,
+      "mean_token_accuracy": 0.4037432588636875,
+      "num_tokens": 1006489.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.6651621140539645,
+      "epoch": 0.096,
+      "grad_norm": 5.1875,
+      "learning_rate": 3.9333333333333335e-06,
+      "loss": 3.3559,
+      "mean_token_accuracy": 0.40558033119887116,
+      "num_tokens": 1201392.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.6933312088251113,
+      "epoch": 0.112,
+      "grad_norm": 4.875,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 3.3441,
+      "mean_token_accuracy": 0.4004572981968522,
+      "num_tokens": 1408433.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.66386479139328,
+      "epoch": 0.128,
+      "grad_norm": 4.46875,
+      "learning_rate": 5.2666666666666665e-06,
+      "loss": 3.1947,
+      "mean_token_accuracy": 0.41622882448136805,
+      "num_tokens": 1608968.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.6450926899909972,
+      "epoch": 0.144,
+      "grad_norm": 3.921875,
+      "learning_rate": 5.933333333333335e-06,
+      "loss": 3.0907,
+      "mean_token_accuracy": 0.42884849160909655,
+      "num_tokens": 1804244.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.6662526927888393,
+      "epoch": 0.16,
+      "grad_norm": 3.875,
+      "learning_rate": 6.600000000000001e-06,
+      "loss": 3.0495,
+      "mean_token_accuracy": 0.4341392319649458,
+      "num_tokens": 2004403.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 1.6570097875595093,
+      "eval_biology_loss": 2.966970443725586,
+      "eval_biology_mean_token_accuracy": 0.4370625810623169,
+      "eval_biology_num_tokens": 2004403.0,
+      "eval_biology_runtime": 22.5729,
+      "eval_biology_samples_per_second": 22.15,
+      "eval_biology_steps_per_second": 5.538,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_chemistry_entropy": 1.3844526329040527,
+      "eval_chemistry_loss": 3.124437093734741,
+      "eval_chemistry_mean_token_accuracy": 0.43499850821495056,
+      "eval_chemistry_num_tokens": 2004403.0,
+      "eval_chemistry_runtime": 28.1082,
+      "eval_chemistry_samples_per_second": 17.788,
+      "eval_chemistry_steps_per_second": 4.447,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_math_entropy": 1.2807003631591798,
+      "eval_math_loss": 3.788844585418701,
+      "eval_math_mean_token_accuracy": 0.38194972825050355,
+      "eval_math_num_tokens": 2004403.0,
+      "eval_math_runtime": 28.8838,
+      "eval_math_samples_per_second": 17.311,
+      "eval_math_steps_per_second": 4.328,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_physics_entropy": 1.369970760345459,
+      "eval_physics_loss": 3.51516056060791,
+      "eval_physics_mean_token_accuracy": 0.40637755012512206,
+      "eval_physics_num_tokens": 2004403.0,
+      "eval_physics_runtime": 32.7838,
+      "eval_physics_samples_per_second": 15.251,
+      "eval_physics_steps_per_second": 3.813,
+      "step": 100
+    },
+    {
+      "entropy": 1.6597946420311929,
+      "epoch": 0.176,
+      "grad_norm": 3.421875,
+      "learning_rate": 7.266666666666668e-06,
+      "loss": 2.8927,
+      "mean_token_accuracy": 0.4443680657073855,
+      "num_tokens": 2205969.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.7204043805599212,
+      "epoch": 0.192,
+      "grad_norm": 3.15625,
+      "learning_rate": 7.933333333333334e-06,
+      "loss": 2.7652,
+      "mean_token_accuracy": 0.45243199188262223,
+      "num_tokens": 2406270.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.7757249370217323,
+      "epoch": 0.208,
+      "grad_norm": 2.71875,
+      "learning_rate": 8.6e-06,
+      "loss": 2.5728,
+      "mean_token_accuracy": 0.47095262724906206,
+      "num_tokens": 2602359.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.872027115523815,
+      "epoch": 0.224,
+      "grad_norm": 2.234375,
+      "learning_rate": 9.266666666666667e-06,
+      "loss": 2.4424,
+      "mean_token_accuracy": 0.4867780333384871,
+      "num_tokens": 2802210.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.972200370579958,
+      "epoch": 0.24,
+      "grad_norm": 1.6328125,
+      "learning_rate": 9.933333333333334e-06,
+      "loss": 2.2693,
+      "mean_token_accuracy": 0.5071418788284063,
+      "num_tokens": 2997201.0,
+      "step": 150
+    },
+    {
+      "entropy": 2.105553387105465,
+      "epoch": 0.256,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.0600000000000002e-05,
+      "loss": 2.1465,
+      "mean_token_accuracy": 0.5260775003582239,
+      "num_tokens": 3193718.0,
+      "step": 160
+    },
+    {
+      "entropy": 2.0858327887952326,
+      "epoch": 0.272,
+      "grad_norm": 1.15625,
+      "learning_rate": 1.1266666666666668e-05,
+      "loss": 2.0273,
+      "mean_token_accuracy": 0.5381948189809919,
+      "num_tokens": 3392801.0,
+      "step": 170
+    },
+    {
+      "entropy": 2.036754435300827,
+      "epoch": 0.288,
+      "grad_norm": 1.15625,
+      "learning_rate": 1.1933333333333335e-05,
+      "loss": 1.9681,
+      "mean_token_accuracy": 0.5518869832158089,
+      "num_tokens": 3599909.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.9979013696312904,
+      "epoch": 0.304,
+      "grad_norm": 0.97265625,
+      "learning_rate": 1.2600000000000001e-05,
+      "loss": 1.8981,
+      "mean_token_accuracy": 0.5629408935084939,
+      "num_tokens": 3796564.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.9476586610078812,
+      "epoch": 0.32,
+      "grad_norm": 0.984375,
+      "learning_rate": 1.3266666666666668e-05,
+      "loss": 1.8576,
+      "mean_token_accuracy": 0.569432912953198,
+      "num_tokens": 3987270.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 1.9238368101119996,
+      "eval_biology_loss": 1.833174705505371,
+      "eval_biology_mean_token_accuracy": 0.5778562531471253,
+      "eval_biology_num_tokens": 3987270.0,
+      "eval_biology_runtime": 22.5333,
+      "eval_biology_samples_per_second": 22.189,
+      "eval_biology_steps_per_second": 5.547,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_chemistry_entropy": 1.7481628065109254,
+      "eval_chemistry_loss": 1.781923770904541,
+      "eval_chemistry_mean_token_accuracy": 0.5858608253002167,
+      "eval_chemistry_num_tokens": 3987270.0,
+      "eval_chemistry_runtime": 28.0618,
+      "eval_chemistry_samples_per_second": 17.818,
+      "eval_chemistry_steps_per_second": 4.454,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_math_entropy": 1.6787789669036866,
+      "eval_math_loss": 1.9706600904464722,
+      "eval_math_mean_token_accuracy": 0.5599996938705444,
+      "eval_math_num_tokens": 3987270.0,
+      "eval_math_runtime": 28.4332,
+      "eval_math_samples_per_second": 17.585,
+      "eval_math_steps_per_second": 4.396,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_physics_entropy": 1.775694995880127,
+      "eval_physics_loss": 1.9076323509216309,
+      "eval_physics_mean_token_accuracy": 0.5728228716850281,
+      "eval_physics_num_tokens": 3987270.0,
+      "eval_physics_runtime": 32.7353,
+      "eval_physics_samples_per_second": 15.274,
+      "eval_physics_steps_per_second": 3.819,
+      "step": 200
+    },
+    {
+      "entropy": 1.8970093309879303,
+      "epoch": 0.336,
+      "grad_norm": 0.953125,
+      "learning_rate": 1.3933333333333334e-05,
+      "loss": 1.8023,
+      "mean_token_accuracy": 0.5797937730327248,
+      "num_tokens": 4184648.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.8955006778240204,
+      "epoch": 0.352,
+      "grad_norm": 0.87890625,
+      "learning_rate": 1.46e-05,
+      "loss": 1.7875,
+      "mean_token_accuracy": 0.5795410269871354,
+      "num_tokens": 4381090.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.8653724752366543,
+      "epoch": 0.368,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.5266666666666667e-05,
+      "loss": 1.7645,
+      "mean_token_accuracy": 0.5859273863956332,
+      "num_tokens": 4581377.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.8357174098491669,
+      "epoch": 0.384,
+      "grad_norm": 0.94140625,
+      "learning_rate": 1.5933333333333336e-05,
+      "loss": 1.7277,
+      "mean_token_accuracy": 0.5924989800900221,
+      "num_tokens": 4774720.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.8090209499001504,
+      "epoch": 0.4,
+      "grad_norm": 0.87109375,
+      "learning_rate": 1.66e-05,
+      "loss": 1.7107,
+      "mean_token_accuracy": 0.5951010771095753,
+      "num_tokens": 4971194.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.8345165871083737,
+      "epoch": 0.416,
+      "grad_norm": 0.875,
+      "learning_rate": 1.726666666666667e-05,
+      "loss": 1.7275,
+      "mean_token_accuracy": 0.5940118793398141,
+      "num_tokens": 5172436.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.7847119137644767,
+      "epoch": 0.432,
+      "grad_norm": 0.9453125,
+      "learning_rate": 1.7933333333333333e-05,
+      "loss": 1.6937,
+      "mean_token_accuracy": 0.6004892244935036,
+      "num_tokens": 5362026.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.7922390177845955,
+      "epoch": 0.448,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.86e-05,
+      "loss": 1.6952,
+      "mean_token_accuracy": 0.5999779261648654,
+      "num_tokens": 5555103.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.7521770052611827,
+      "epoch": 0.464,
+      "grad_norm": 0.89453125,
+      "learning_rate": 1.926666666666667e-05,
+      "loss": 1.6483,
+      "mean_token_accuracy": 0.6085187628865242,
+      "num_tokens": 5751863.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.7609217807650566,
+      "epoch": 0.48,
+      "grad_norm": 0.8125,
+      "learning_rate": 1.9933333333333334e-05,
+      "loss": 1.6596,
+      "mean_token_accuracy": 0.606584096327424,
+      "num_tokens": 5951516.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 1.729413685798645,
+      "eval_biology_loss": 1.6440614461898804,
+      "eval_biology_mean_token_accuracy": 0.6129752106666565,
+      "eval_biology_num_tokens": 5951516.0,
+      "eval_biology_runtime": 22.5388,
+      "eval_biology_samples_per_second": 22.184,
+      "eval_biology_steps_per_second": 5.546,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_chemistry_entropy": 1.5809336042404174,
+      "eval_chemistry_loss": 1.5447371006011963,
+      "eval_chemistry_mean_token_accuracy": 0.6340514001846314,
+      "eval_chemistry_num_tokens": 5951516.0,
+      "eval_chemistry_runtime": 28.0564,
+      "eval_chemistry_samples_per_second": 17.821,
+      "eval_chemistry_steps_per_second": 4.455,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_math_entropy": 1.5071503019332886,
+      "eval_math_loss": 1.7231581211090088,
+      "eval_math_mean_token_accuracy": 0.6083300161361694,
+      "eval_math_num_tokens": 5951516.0,
+      "eval_math_runtime": 28.4264,
+      "eval_math_samples_per_second": 17.589,
+      "eval_math_steps_per_second": 4.397,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_physics_entropy": 1.6148314781188964,
+      "eval_physics_loss": 1.6281670331954956,
+      "eval_physics_mean_token_accuracy": 0.6277218542098999,
+      "eval_physics_num_tokens": 5951516.0,
+      "eval_physics_runtime": 32.7584,
+      "eval_physics_samples_per_second": 15.263,
+      "eval_physics_steps_per_second": 3.816,
+      "step": 300
+    },
+    {
+      "entropy": 1.6838763318955898,
+      "epoch": 0.496,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.9933333333333334e-05,
+      "loss": 1.5791,
+      "mean_token_accuracy": 0.6194305628538131,
+      "num_tokens": 6151316.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.7321165360510349,
+      "epoch": 0.512,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.985925925925926e-05,
+      "loss": 1.6328,
+      "mean_token_accuracy": 0.6095499075949192,
+      "num_tokens": 6348783.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.7121967576444148,
+      "epoch": 0.528,
+      "grad_norm": 0.80859375,
+      "learning_rate": 1.9785185185185187e-05,
+      "loss": 1.6219,
+      "mean_token_accuracy": 0.6149081885814667,
+      "num_tokens": 6542101.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.7171216145157815,
+      "epoch": 0.544,
+      "grad_norm": 0.78125,
+      "learning_rate": 1.971111111111111e-05,
+      "loss": 1.6231,
+      "mean_token_accuracy": 0.6155458513647318,
+      "num_tokens": 6741134.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.706493303924799,
+      "epoch": 0.56,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.963703703703704e-05,
+      "loss": 1.6033,
+      "mean_token_accuracy": 0.6156127706170083,
+      "num_tokens": 6939309.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.6751246757805347,
+      "epoch": 0.576,
+      "grad_norm": 0.83984375,
+      "learning_rate": 1.9562962962962964e-05,
+      "loss": 1.5857,
+      "mean_token_accuracy": 0.6199627369642258,
+      "num_tokens": 7143677.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.6992512352764606,
+      "epoch": 0.592,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.948888888888889e-05,
+      "loss": 1.5839,
+      "mean_token_accuracy": 0.6194002132862806,
+      "num_tokens": 7339299.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.674944642186165,
+      "epoch": 0.608,
+      "grad_norm": 0.91015625,
+      "learning_rate": 1.9414814814814817e-05,
+      "loss": 1.5937,
+      "mean_token_accuracy": 0.6201793253421783,
+      "num_tokens": 7545927.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.687514978647232,
+      "epoch": 0.624,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.9340740740740743e-05,
+      "loss": 1.5843,
+      "mean_token_accuracy": 0.6189679309725762,
+      "num_tokens": 7745251.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.694458904117346,
+      "epoch": 0.64,
+      "grad_norm": 0.83203125,
+      "learning_rate": 1.926666666666667e-05,
+      "loss": 1.592,
+      "mean_token_accuracy": 0.6188682828098535,
+      "num_tokens": 7937611.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 1.6453915300369262,
+      "eval_biology_loss": 1.570678472518921,
+      "eval_biology_mean_token_accuracy": 0.626722119808197,
+      "eval_biology_num_tokens": 7937611.0,
+      "eval_biology_runtime": 22.5208,
+      "eval_biology_samples_per_second": 22.202,
+      "eval_biology_steps_per_second": 5.55,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_chemistry_entropy": 1.4844962959289552,
+      "eval_chemistry_loss": 1.456817626953125,
+      "eval_chemistry_mean_token_accuracy": 0.6511806635856628,
+      "eval_chemistry_num_tokens": 7937611.0,
+      "eval_chemistry_runtime": 28.0495,
+      "eval_chemistry_samples_per_second": 17.826,
+      "eval_chemistry_steps_per_second": 4.456,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_math_entropy": 1.3942026481628418,
+      "eval_math_loss": 1.6414613723754883,
+      "eval_math_mean_token_accuracy": 0.6246838383674621,
+      "eval_math_num_tokens": 7937611.0,
+      "eval_math_runtime": 28.4483,
+      "eval_math_samples_per_second": 17.576,
+      "eval_math_steps_per_second": 4.394,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_physics_entropy": 1.5108685884475708,
+      "eval_physics_loss": 1.5289188623428345,
+      "eval_physics_mean_token_accuracy": 0.6458710317611694,
+      "eval_physics_num_tokens": 7937611.0,
+      "eval_physics_runtime": 32.7797,
+      "eval_physics_samples_per_second": 15.253,
+      "eval_physics_steps_per_second": 3.813,
+      "step": 400
+    },
+    {
+      "entropy": 1.6825137794017793,
+      "epoch": 0.656,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.9192592592592593e-05,
+      "loss": 1.585,
+      "mean_token_accuracy": 0.6200674671679736,
+      "num_tokens": 8135736.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.676036175340414,
+      "epoch": 0.672,
+      "grad_norm": 0.7890625,
+      "learning_rate": 1.911851851851852e-05,
+      "loss": 1.5788,
+      "mean_token_accuracy": 0.6214150108397007,
+      "num_tokens": 8326210.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.6576065368950368,
+      "epoch": 0.688,
+      "grad_norm": 0.7734375,
+      "learning_rate": 1.9044444444444446e-05,
+      "loss": 1.5687,
+      "mean_token_accuracy": 0.6240637619048357,
+      "num_tokens": 8532555.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.6678082942962646,
+      "epoch": 0.704,
+      "grad_norm": 0.75,
+      "learning_rate": 1.8970370370370372e-05,
+      "loss": 1.5606,
+      "mean_token_accuracy": 0.6247469738125802,
+      "num_tokens": 8728117.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.616765297204256,
+      "epoch": 0.72,
+      "grad_norm": 0.7890625,
+      "learning_rate": 1.8896296296296295e-05,
+      "loss": 1.5094,
+      "mean_token_accuracy": 0.632154631242156,
+      "num_tokens": 8934366.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.628147255629301,
+      "epoch": 0.736,
+      "grad_norm": 0.765625,
+      "learning_rate": 1.8822222222222225e-05,
+      "loss": 1.5205,
+      "mean_token_accuracy": 0.6305314172059298,
+      "num_tokens": 9134944.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.6318965889513493,
+      "epoch": 0.752,
+      "grad_norm": 0.8671875,
+      "learning_rate": 1.874814814814815e-05,
+      "loss": 1.5359,
+      "mean_token_accuracy": 0.6287370190024376,
+      "num_tokens": 9330503.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.6036403521895408,
+      "epoch": 0.768,
+      "grad_norm": 0.84765625,
+      "learning_rate": 1.8674074074074075e-05,
+      "loss": 1.5092,
+      "mean_token_accuracy": 0.633948576822877,
+      "num_tokens": 9532527.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.6391747564077377,
+      "epoch": 0.784,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.86e-05,
+      "loss": 1.5291,
+      "mean_token_accuracy": 0.6272940371185541,
+      "num_tokens": 9733097.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.647210680693388,
+      "epoch": 0.8,
+      "grad_norm": 0.88671875,
+      "learning_rate": 1.8525925925925928e-05,
+      "loss": 1.5582,
+      "mean_token_accuracy": 0.6245432507246733,
+      "num_tokens": 9931992.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 1.6242998161315918,
+      "eval_biology_loss": 1.533657193183899,
+      "eval_biology_mean_token_accuracy": 0.6338145670890808,
+      "eval_biology_num_tokens": 9931992.0,
+      "eval_biology_runtime": 22.531,
+      "eval_biology_samples_per_second": 22.192,
+      "eval_biology_steps_per_second": 5.548,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_chemistry_entropy": 1.467040044784546,
+      "eval_chemistry_loss": 1.4095803499221802,
+      "eval_chemistry_mean_token_accuracy": 0.6610978055000305,
+      "eval_chemistry_num_tokens": 9931992.0,
+      "eval_chemistry_runtime": 27.9615,
+      "eval_chemistry_samples_per_second": 17.882,
+      "eval_chemistry_steps_per_second": 4.47,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_math_entropy": 1.370791340827942,
+      "eval_math_loss": 1.5665466785430908,
+      "eval_math_mean_token_accuracy": 0.640302906036377,
+      "eval_math_num_tokens": 9931992.0,
+      "eval_math_runtime": 28.4102,
+      "eval_math_samples_per_second": 17.599,
+      "eval_math_steps_per_second": 4.4,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_physics_entropy": 1.49333540058136,
+      "eval_physics_loss": 1.4734785556793213,
+      "eval_physics_mean_token_accuracy": 0.6578547234535217,
+      "eval_physics_num_tokens": 9931992.0,
+      "eval_physics_runtime": 32.8174,
+      "eval_physics_samples_per_second": 15.236,
+      "eval_physics_steps_per_second": 3.809,
+      "step": 500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.834798540072223e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a387cb57d59ea7ca732b146979c08bd91e2b41d93ff2b0bc7a7cf89c84374f1
+size 6481