Upload folder using huggingface_hub

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-2000/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

checkpoint-2000/chat_template.jinja

@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

checkpoint-2000/config.json

@@ -0,0 +1,86 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3840,
+  "initializer_range": 0.02,
+  "intermediate_size": 15360,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": {
+    "factor": 8.0,
+    "rope_type": "linear"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 1024,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 262208
+}

checkpoint-2000/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 2,
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.56.1"
+}

checkpoint-2000/model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a4c14f38ffc9e23d873fa999eea1fc965dcedcec267e7a99f07c6fdbe8760d8
+size 4915892992

checkpoint-2000/model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0335ce336b31e38c581326acb77ff28330c2073764b1924ad7108d788b56ba68
+size 4931294472

checkpoint-2000/model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0d6a60cf6a1708be0613ab7c4d10457ad99b98c02e40ebd5f320fe02aa0d50c
+size 4931294528

checkpoint-2000/model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bd9340bc368427bcc805e329f32ec7f6a73ab12274d00a8b9c03d49fe0853d0
+size 4931294528

checkpoint-2000/model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d80ec30fa09df9cb78149b3af4da5e7f22bbe89087198871d4f36c70903e30a6
+size 3822364808

checkpoint-2000/model.safetensors.index.json

@@ -0,0 +1,634 @@
+{
+  "metadata": {
+    "total_parameters": 11766034176,
+    "total_size": 23532068352
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.norm.weight": "model-00005-of-00005.safetensors"
+  }
+}

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20306f40758b096547d3a313ec64b52ecabe0ac60e4a133f416c6b08b28f0f2d
+size 18355

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7993dc63b304673dcc82de3f9b7ca8dddf6548aee44d77968103b480a102016
+size 14645

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e95ad2f3950cee1b83b106cf88750c8604566acab211d78973b3b5c4917b1a30
+size 1465

checkpoint-2000/special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

checkpoint-2000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

checkpoint-2000/trainer_state.json

@@ -0,0 +1,2474 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.7798364212986146,
+  "eval_steps": 100,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.4347459591925142,
+      "epoch": 0.00890224225226729,
+      "grad_norm": 0.00732421875,
+      "learning_rate": 1.8e-07,
+      "loss": 3.3392,
+      "mean_token_accuracy": 0.4414398778229952,
+      "num_tokens": 128968.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.5189025782048704,
+      "epoch": 0.01780448450453458,
+      "grad_norm": 0.0098876953125,
+      "learning_rate": 3.8e-07,
+      "loss": 3.4461,
+      "mean_token_accuracy": 0.42950069066137075,
+      "num_tokens": 257189.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.4521907046437263,
+      "epoch": 0.026706726756801868,
+      "grad_norm": 0.007659912109375,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 3.3929,
+      "mean_token_accuracy": 0.43435896690934894,
+      "num_tokens": 394654.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.4908889502286913,
+      "epoch": 0.03560896900906916,
+      "grad_norm": 0.00909423828125,
+      "learning_rate": 7.8e-07,
+      "loss": 3.3731,
+      "mean_token_accuracy": 0.4337937039323151,
+      "num_tokens": 528285.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.4656582184135916,
+      "epoch": 0.04451121126133645,
+      "grad_norm": 0.0078125,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 3.3705,
+      "mean_token_accuracy": 0.4305401614867151,
+      "num_tokens": 663126.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.5354390598833563,
+      "epoch": 0.053413453513603736,
+      "grad_norm": 0.007659912109375,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 3.4166,
+      "mean_token_accuracy": 0.42836275026202203,
+      "num_tokens": 801635.0,
+      "step": 60
+    },
+    {
+      "entropy": 2.496990965306759,
+      "epoch": 0.06231569576587103,
+      "grad_norm": 0.00885009765625,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 3.4009,
+      "mean_token_accuracy": 0.43142524575814606,
+      "num_tokens": 931344.0,
+      "step": 70
+    },
+    {
+      "entropy": 2.5406345002353192,
+      "epoch": 0.07121793801813832,
+      "grad_norm": 0.007171630859375,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 3.4506,
+      "mean_token_accuracy": 0.42618109108880164,
+      "num_tokens": 1060812.0,
+      "step": 80
+    },
+    {
+      "entropy": 2.4690978921949864,
+      "epoch": 0.08012018027040561,
+      "grad_norm": 0.0111083984375,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 3.375,
+      "mean_token_accuracy": 0.4317979410290718,
+      "num_tokens": 1200650.0,
+      "step": 90
+    },
+    {
+      "entropy": 2.537689981609583,
+      "epoch": 0.0890224225226729,
+      "grad_norm": 0.009033203125,
+      "learning_rate": 1.98e-06,
+      "loss": 3.4588,
+      "mean_token_accuracy": 0.4282337296754122,
+      "num_tokens": 1327380.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.0890224225226729,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 1327380.0,
+      "eval_coding_runtime": 96.2988,
+      "eval_coding_samples_per_second": 5.192,
+      "eval_coding_steps_per_second": 2.596,
+      "step": 100
+    },
+    {
+      "epoch": 0.0890224225226729,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 1327380.0,
+      "eval_chemistry_runtime": 53.689,
+      "eval_chemistry_samples_per_second": 9.313,
+      "eval_chemistry_steps_per_second": 4.656,
+      "step": 100
+    },
+    {
+      "entropy": 2.541736252605915,
+      "epoch": 0.09792466477494019,
+      "grad_norm": 0.0079345703125,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 3.4778,
+      "mean_token_accuracy": 0.42182709854096173,
+      "num_tokens": 1460130.0,
+      "step": 110
+    },
+    {
+      "entropy": 2.4891932401806116,
+      "epoch": 0.10682690702720747,
+      "grad_norm": 0.0091552734375,
+      "learning_rate": 2.38e-06,
+      "loss": 3.4123,
+      "mean_token_accuracy": 0.4318336697295308,
+      "num_tokens": 1597405.0,
+      "step": 120
+    },
+    {
+      "entropy": 2.549714620411396,
+      "epoch": 0.11572914927947477,
+      "grad_norm": 0.013427734375,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 3.5233,
+      "mean_token_accuracy": 0.4176134932786226,
+      "num_tokens": 1728207.0,
+      "step": 130
+    },
+    {
+      "entropy": 2.5017867423594,
+      "epoch": 0.12463139153174206,
+      "grad_norm": 0.00921630859375,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 3.4252,
+      "mean_token_accuracy": 0.4325200604274869,
+      "num_tokens": 1859684.0,
+      "step": 140
+    },
+    {
+      "entropy": 2.607860314100981,
+      "epoch": 0.13353363378400934,
+      "grad_norm": 0.0086669921875,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 3.5854,
+      "mean_token_accuracy": 0.41672497261315583,
+      "num_tokens": 1985766.0,
+      "step": 150
+    },
+    {
+      "entropy": 2.546873392164707,
+      "epoch": 0.14243587603627664,
+      "grad_norm": 0.0096435546875,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 3.4451,
+      "mean_token_accuracy": 0.42558788452297447,
+      "num_tokens": 2118068.0,
+      "step": 160
+    },
+    {
+      "entropy": 2.5062545992434027,
+      "epoch": 0.15133811828854393,
+      "grad_norm": 0.0125732421875,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 3.4525,
+      "mean_token_accuracy": 0.42451597433537247,
+      "num_tokens": 2252650.0,
+      "step": 170
+    },
+    {
+      "entropy": 2.490347370505333,
+      "epoch": 0.16024036054081123,
+      "grad_norm": 0.0106201171875,
+      "learning_rate": 3.58e-06,
+      "loss": 3.398,
+      "mean_token_accuracy": 0.4340389359742403,
+      "num_tokens": 2388824.0,
+      "step": 180
+    },
+    {
+      "entropy": 2.5069186124950646,
+      "epoch": 0.1691426027930785,
+      "grad_norm": 0.01055908203125,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 3.4995,
+      "mean_token_accuracy": 0.42545853555202484,
+      "num_tokens": 2515325.0,
+      "step": 190
+    },
+    {
+      "entropy": 2.569493807852268,
+      "epoch": 0.1780448450453458,
+      "grad_norm": 0.0174560546875,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 3.5293,
+      "mean_token_accuracy": 0.4203047439455986,
+      "num_tokens": 2644330.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.1780448450453458,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 2644330.0,
+      "eval_coding_runtime": 95.9249,
+      "eval_coding_samples_per_second": 5.212,
+      "eval_coding_steps_per_second": 2.606,
+      "step": 200
+    },
+    {
+      "epoch": 0.1780448450453458,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 2644330.0,
+      "eval_chemistry_runtime": 53.6456,
+      "eval_chemistry_samples_per_second": 9.32,
+      "eval_chemistry_steps_per_second": 4.66,
+      "step": 200
+    },
+    {
+      "entropy": 2.463035849481821,
+      "epoch": 0.18694708729761308,
+      "grad_norm": 0.01226806640625,
+      "learning_rate": 4.18e-06,
+      "loss": 3.3809,
+      "mean_token_accuracy": 0.43558520982041954,
+      "num_tokens": 2783261.0,
+      "step": 210
+    },
+    {
+      "entropy": 2.5025305368006228,
+      "epoch": 0.19584932954988038,
+      "grad_norm": 0.007476806640625,
+      "learning_rate": 4.38e-06,
+      "loss": 3.4253,
+      "mean_token_accuracy": 0.4310411293059587,
+      "num_tokens": 2913700.0,
+      "step": 220
+    },
+    {
+      "entropy": 2.455498095601797,
+      "epoch": 0.20475157180214767,
+      "grad_norm": 0.0068359375,
+      "learning_rate": 4.58e-06,
+      "loss": 3.3586,
+      "mean_token_accuracy": 0.43719491101801394,
+      "num_tokens": 3048973.0,
+      "step": 230
+    },
+    {
+      "entropy": 2.5426308810710907,
+      "epoch": 0.21365381405441494,
+      "grad_norm": 0.007598876953125,
+      "learning_rate": 4.78e-06,
+      "loss": 3.406,
+      "mean_token_accuracy": 0.43309843018651006,
+      "num_tokens": 3185255.0,
+      "step": 240
+    },
+    {
+      "entropy": 2.444308698922396,
+      "epoch": 0.22255605630668224,
+      "grad_norm": 0.01324462890625,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 3.337,
+      "mean_token_accuracy": 0.44262526389211415,
+      "num_tokens": 3322823.0,
+      "step": 250
+    },
+    {
+      "entropy": 2.526992666721344,
+      "epoch": 0.23145829855894953,
+      "grad_norm": 0.0103759765625,
+      "learning_rate": 5.18e-06,
+      "loss": 3.478,
+      "mean_token_accuracy": 0.42772163953632114,
+      "num_tokens": 3454750.0,
+      "step": 260
+    },
+    {
+      "entropy": 2.468804422393441,
+      "epoch": 0.24036054081121683,
+      "grad_norm": 0.0084228515625,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 3.3899,
+      "mean_token_accuracy": 0.4310757084749639,
+      "num_tokens": 3589449.0,
+      "step": 270
+    },
+    {
+      "entropy": 2.5963171511888503,
+      "epoch": 0.24926278306348412,
+      "grad_norm": 0.00823974609375,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 3.5597,
+      "mean_token_accuracy": 0.4164831433445215,
+      "num_tokens": 3719113.0,
+      "step": 280
+    },
+    {
+      "entropy": 2.6820507936179636,
+      "epoch": 0.2581650253157514,
+      "grad_norm": 0.008056640625,
+      "learning_rate": 5.78e-06,
+      "loss": 3.6014,
+      "mean_token_accuracy": 0.4143718365579844,
+      "num_tokens": 3852276.0,
+      "step": 290
+    },
+    {
+      "entropy": 2.5068675063550474,
+      "epoch": 0.2670672675680187,
+      "grad_norm": 0.01226806640625,
+      "learning_rate": 5.98e-06,
+      "loss": 3.4183,
+      "mean_token_accuracy": 0.4291886316612363,
+      "num_tokens": 3990505.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.2670672675680187,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 3990505.0,
+      "eval_coding_runtime": 95.8518,
+      "eval_coding_samples_per_second": 5.216,
+      "eval_coding_steps_per_second": 2.608,
+      "step": 300
+    },
+    {
+      "epoch": 0.2670672675680187,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 3990505.0,
+      "eval_chemistry_runtime": 53.7283,
+      "eval_chemistry_samples_per_second": 9.306,
+      "eval_chemistry_steps_per_second": 4.653,
+      "step": 300
+    },
+    {
+      "entropy": 2.4614722445607184,
+      "epoch": 0.275969509820286,
+      "grad_norm": 0.0091552734375,
+      "learning_rate": 6.18e-06,
+      "loss": 3.3329,
+      "mean_token_accuracy": 0.4380343802273273,
+      "num_tokens": 4133119.0,
+      "step": 310
+    },
+    {
+      "entropy": 2.4565507762134073,
+      "epoch": 0.2848717520725533,
+      "grad_norm": 0.0074462890625,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 3.376,
+      "mean_token_accuracy": 0.4339581592008471,
+      "num_tokens": 4267403.0,
+      "step": 320
+    },
+    {
+      "entropy": 2.4726619601249693,
+      "epoch": 0.29377399432482054,
+      "grad_norm": 0.007781982421875,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 3.3584,
+      "mean_token_accuracy": 0.43662677630782126,
+      "num_tokens": 4400242.0,
+      "step": 330
+    },
+    {
+      "entropy": 2.4780636690557003,
+      "epoch": 0.30267623657708786,
+      "grad_norm": 0.009765625,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 3.3561,
+      "mean_token_accuracy": 0.4329976743087173,
+      "num_tokens": 4535458.0,
+      "step": 340
+    },
+    {
+      "entropy": 2.4669046089053155,
+      "epoch": 0.31157847882935513,
+      "grad_norm": 0.01031494140625,
+      "learning_rate": 6.98e-06,
+      "loss": 3.4031,
+      "mean_token_accuracy": 0.43433742690831423,
+      "num_tokens": 4662903.0,
+      "step": 350
+    },
+    {
+      "entropy": 2.5689725764095783,
+      "epoch": 0.32048072108162245,
+      "grad_norm": 0.00732421875,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 3.4889,
+      "mean_token_accuracy": 0.4287034338340163,
+      "num_tokens": 4796815.0,
+      "step": 360
+    },
+    {
+      "entropy": 2.4410855643451215,
+      "epoch": 0.3293829633338897,
+      "grad_norm": 0.00836181640625,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 3.3643,
+      "mean_token_accuracy": 0.43710872549563645,
+      "num_tokens": 4932671.0,
+      "step": 370
+    },
+    {
+      "entropy": 2.5445352010428905,
+      "epoch": 0.338285205586157,
+      "grad_norm": 0.01129150390625,
+      "learning_rate": 7.58e-06,
+      "loss": 3.4525,
+      "mean_token_accuracy": 0.42816965784877536,
+      "num_tokens": 5066948.0,
+      "step": 380
+    },
+    {
+      "entropy": 2.565714708715677,
+      "epoch": 0.3471874478384243,
+      "grad_norm": 0.01470947265625,
+      "learning_rate": 7.78e-06,
+      "loss": 3.4661,
+      "mean_token_accuracy": 0.4264904214069247,
+      "num_tokens": 5193519.0,
+      "step": 390
+    },
+    {
+      "entropy": 2.516895131766796,
+      "epoch": 0.3560896900906916,
+      "grad_norm": 0.01031494140625,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 3.4401,
+      "mean_token_accuracy": 0.4265883769840002,
+      "num_tokens": 5324751.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.3560896900906916,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 5324751.0,
+      "eval_coding_runtime": 95.8822,
+      "eval_coding_samples_per_second": 5.215,
+      "eval_coding_steps_per_second": 2.607,
+      "step": 400
+    },
+    {
+      "epoch": 0.3560896900906916,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 5324751.0,
+      "eval_chemistry_runtime": 53.6645,
+      "eval_chemistry_samples_per_second": 9.317,
+      "eval_chemistry_steps_per_second": 4.659,
+      "step": 400
+    },
+    {
+      "entropy": 2.4508687026798723,
+      "epoch": 0.3649919323429589,
+      "grad_norm": 0.010986328125,
+      "learning_rate": 8.18e-06,
+      "loss": 3.3874,
+      "mean_token_accuracy": 0.4385754197835922,
+      "num_tokens": 5458519.0,
+      "step": 410
+    },
+    {
+      "entropy": 2.4801586009562016,
+      "epoch": 0.37389417459522617,
+      "grad_norm": 0.00714111328125,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 3.4012,
+      "mean_token_accuracy": 0.43559422083199023,
+      "num_tokens": 5585508.0,
+      "step": 420
+    },
+    {
+      "entropy": 2.4704861216247083,
+      "epoch": 0.38279641684749344,
+      "grad_norm": 0.007659912109375,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 3.4353,
+      "mean_token_accuracy": 0.43275914546102284,
+      "num_tokens": 5716321.0,
+      "step": 430
+    },
+    {
+      "entropy": 2.537934695184231,
+      "epoch": 0.39169865909976076,
+      "grad_norm": 0.0081787109375,
+      "learning_rate": 8.78e-06,
+      "loss": 3.4176,
+      "mean_token_accuracy": 0.4291856364347041,
+      "num_tokens": 5848889.0,
+      "step": 440
+    },
+    {
+      "entropy": 2.513765400648117,
+      "epoch": 0.400600901352028,
+      "grad_norm": 0.00927734375,
+      "learning_rate": 8.98e-06,
+      "loss": 3.4106,
+      "mean_token_accuracy": 0.42827712278813124,
+      "num_tokens": 5980229.0,
+      "step": 450
+    },
+    {
+      "entropy": 2.4634178556501864,
+      "epoch": 0.40950314360429535,
+      "grad_norm": 0.01068115234375,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 3.4007,
+      "mean_token_accuracy": 0.4337839787825942,
+      "num_tokens": 6114855.0,
+      "step": 460
+    },
+    {
+      "entropy": 2.5320842534303667,
+      "epoch": 0.4184053858565626,
+      "grad_norm": 0.007049560546875,
+      "learning_rate": 9.38e-06,
+      "loss": 3.4216,
+      "mean_token_accuracy": 0.4268041457980871,
+      "num_tokens": 6244987.0,
+      "step": 470
+    },
+    {
+      "entropy": 2.4959075897932053,
+      "epoch": 0.4273076281088299,
+      "grad_norm": 0.0079345703125,
+      "learning_rate": 9.58e-06,
+      "loss": 3.3715,
+      "mean_token_accuracy": 0.432720182929188,
+      "num_tokens": 6378152.0,
+      "step": 480
+    },
+    {
+      "entropy": 2.503755620121956,
+      "epoch": 0.4362098703610972,
+      "grad_norm": 0.01708984375,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 3.4343,
+      "mean_token_accuracy": 0.43189559010788797,
+      "num_tokens": 6510231.0,
+      "step": 490
+    },
+    {
+      "entropy": 2.539106211811304,
+      "epoch": 0.4451121126133645,
+      "grad_norm": 0.0115966796875,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 3.4747,
+      "mean_token_accuracy": 0.42332094311714175,
+      "num_tokens": 6637273.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.4451121126133645,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 6637273.0,
+      "eval_coding_runtime": 95.8134,
+      "eval_coding_samples_per_second": 5.218,
+      "eval_coding_steps_per_second": 2.609,
+      "step": 500
+    },
+    {
+      "epoch": 0.4451121126133645,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 6637273.0,
+      "eval_chemistry_runtime": 53.3031,
+      "eval_chemistry_samples_per_second": 9.38,
+      "eval_chemistry_steps_per_second": 4.69,
+      "step": 500
+    },
+    {
+      "entropy": 2.536660289019346,
+      "epoch": 0.4540143548656318,
+      "grad_norm": 0.0146484375,
+      "learning_rate": 1.018e-05,
+      "loss": 3.4613,
+      "mean_token_accuracy": 0.4232974941842258,
+      "num_tokens": 6765798.0,
+      "step": 510
+    },
+    {
+      "entropy": 2.459078107774258,
+      "epoch": 0.46291659711789906,
+      "grad_norm": 0.0079345703125,
+      "learning_rate": 1.038e-05,
+      "loss": 3.3655,
+      "mean_token_accuracy": 0.43400888703763485,
+      "num_tokens": 6896684.0,
+      "step": 520
+    },
+    {
+      "entropy": 2.454593952000141,
+      "epoch": 0.4718188393701664,
+      "grad_norm": 0.014404296875,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 3.3614,
+      "mean_token_accuracy": 0.43178336657583716,
+      "num_tokens": 7033417.0,
+      "step": 530
+    },
+    {
+      "entropy": 2.558536158502102,
+      "epoch": 0.48072108162243365,
+      "grad_norm": 0.018310546875,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 3.4999,
+      "mean_token_accuracy": 0.4232194619253278,
+      "num_tokens": 7166608.0,
+      "step": 540
+    },
+    {
+      "entropy": 2.4503342963755133,
+      "epoch": 0.4896233238747009,
+      "grad_norm": 0.0150146484375,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 3.3626,
+      "mean_token_accuracy": 0.4358119173906744,
+      "num_tokens": 7311968.0,
+      "step": 550
+    },
+    {
+      "entropy": 2.4068671748042108,
+      "epoch": 0.49852556612696824,
+      "grad_norm": 0.00836181640625,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 3.2928,
+      "mean_token_accuracy": 0.44231050591915844,
+      "num_tokens": 7444923.0,
+      "step": 560
+    },
+    {
+      "entropy": 2.5063270702958107,
+      "epoch": 0.5074278083792355,
+      "grad_norm": 0.007171630859375,
+      "learning_rate": 1.138e-05,
+      "loss": 3.4094,
+      "mean_token_accuracy": 0.42552004270255567,
+      "num_tokens": 7576456.0,
+      "step": 570
+    },
+    {
+      "entropy": 2.5186563402414324,
+      "epoch": 0.5163300506315028,
+      "grad_norm": 0.00994873046875,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 3.4523,
+      "mean_token_accuracy": 0.42800150997936726,
+      "num_tokens": 7706502.0,
+      "step": 580
+    },
+    {
+      "entropy": 2.500379876047373,
+      "epoch": 0.52523229288377,
+      "grad_norm": 0.016357421875,
+      "learning_rate": 1.178e-05,
+      "loss": 3.4366,
+      "mean_token_accuracy": 0.42906679548323157,
+      "num_tokens": 7838516.0,
+      "step": 590
+    },
+    {
+      "entropy": 2.535025441646576,
+      "epoch": 0.5341345351360374,
+      "grad_norm": 0.0087890625,
+      "learning_rate": 1.198e-05,
+      "loss": 3.5082,
+      "mean_token_accuracy": 0.42478432394564153,
+      "num_tokens": 7969704.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.5341345351360374,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 7969704.0,
+      "eval_coding_runtime": 95.8333,
+      "eval_coding_samples_per_second": 5.217,
+      "eval_coding_steps_per_second": 2.609,
+      "step": 600
+    },
+    {
+      "epoch": 0.5341345351360374,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 7969704.0,
+      "eval_chemistry_runtime": 53.6736,
+      "eval_chemistry_samples_per_second": 9.316,
+      "eval_chemistry_steps_per_second": 4.658,
+      "step": 600
+    },
+    {
+      "entropy": 2.542660507559776,
+      "epoch": 0.5430367773883047,
+      "grad_norm": 0.009765625,
+      "learning_rate": 1.218e-05,
+      "loss": 3.4959,
+      "mean_token_accuracy": 0.42042196970433,
+      "num_tokens": 8103869.0,
+      "step": 610
+    },
+    {
+      "entropy": 2.5036643177270888,
+      "epoch": 0.551939019640572,
+      "grad_norm": 0.007781982421875,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 3.4222,
+      "mean_token_accuracy": 0.43322109896689653,
+      "num_tokens": 8242162.0,
+      "step": 620
+    },
+    {
+      "entropy": 2.5407710492610933,
+      "epoch": 0.5608412618928392,
+      "grad_norm": 0.01031494140625,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 3.5112,
+      "mean_token_accuracy": 0.424757894128561,
+      "num_tokens": 8365617.0,
+      "step": 630
+    },
+    {
+      "entropy": 2.4822314344346523,
+      "epoch": 0.5697435041451065,
+      "grad_norm": 0.009033203125,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 3.4362,
+      "mean_token_accuracy": 0.43353409245610236,
+      "num_tokens": 8497852.0,
+      "step": 640
+    },
+    {
+      "entropy": 2.5407922022044658,
+      "epoch": 0.5786457463973739,
+      "grad_norm": 0.0172119140625,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 3.5164,
+      "mean_token_accuracy": 0.42170737963169813,
+      "num_tokens": 8622868.0,
+      "step": 650
+    },
+    {
+      "entropy": 2.577139265835285,
+      "epoch": 0.5875479886496411,
+      "grad_norm": 0.01007080078125,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 3.5058,
+      "mean_token_accuracy": 0.41903221048414707,
+      "num_tokens": 8757753.0,
+      "step": 660
+    },
+    {
+      "entropy": 2.475165160000324,
+      "epoch": 0.5964502309019084,
+      "grad_norm": 0.01422119140625,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 3.4273,
+      "mean_token_accuracy": 0.4251199818216264,
+      "num_tokens": 8886401.0,
+      "step": 670
+    },
+    {
+      "entropy": 2.4363515108823774,
+      "epoch": 0.6053524731541757,
+      "grad_norm": 0.01300048828125,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 3.3275,
+      "mean_token_accuracy": 0.43763177264481784,
+      "num_tokens": 9024677.0,
+      "step": 680
+    },
+    {
+      "entropy": 2.56791525632143,
+      "epoch": 0.614254715406443,
+      "grad_norm": 0.0074462890625,
+      "learning_rate": 1.378e-05,
+      "loss": 3.4703,
+      "mean_token_accuracy": 0.42369468677788974,
+      "num_tokens": 9153771.0,
+      "step": 690
+    },
+    {
+      "entropy": 2.484988895058632,
+      "epoch": 0.6231569576587103,
+      "grad_norm": 0.0126953125,
+      "learning_rate": 1.398e-05,
+      "loss": 3.3769,
+      "mean_token_accuracy": 0.4326890670694411,
+      "num_tokens": 9291760.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.6231569576587103,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 9291760.0,
+      "eval_coding_runtime": 95.8189,
+      "eval_coding_samples_per_second": 5.218,
+      "eval_coding_steps_per_second": 2.609,
+      "step": 700
+    },
+    {
+      "epoch": 0.6231569576587103,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 9291760.0,
+      "eval_chemistry_runtime": 53.6652,
+      "eval_chemistry_samples_per_second": 9.317,
+      "eval_chemistry_steps_per_second": 4.659,
+      "step": 700
+    },
+    {
+      "entropy": 2.4618755050003527,
+      "epoch": 0.6320591999109776,
+      "grad_norm": 0.00921630859375,
+      "learning_rate": 1.418e-05,
+      "loss": 3.4035,
+      "mean_token_accuracy": 0.4318504808470607,
+      "num_tokens": 9425022.0,
+      "step": 710
+    },
+    {
+      "entropy": 2.4683789536356926,
+      "epoch": 0.6409614421632449,
+      "grad_norm": 0.0076904296875,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 3.3571,
+      "mean_token_accuracy": 0.43439894206821916,
+      "num_tokens": 9561091.0,
+      "step": 720
+    },
+    {
+      "entropy": 2.4340669311583043,
+      "epoch": 0.6498636844155121,
+      "grad_norm": 0.00970458984375,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 3.3348,
+      "mean_token_accuracy": 0.4359587837010622,
+      "num_tokens": 9695723.0,
+      "step": 730
+    },
+    {
+      "entropy": 2.504717104136944,
+      "epoch": 0.6587659266677794,
+      "grad_norm": 0.0174560546875,
+      "learning_rate": 1.478e-05,
+      "loss": 3.4563,
+      "mean_token_accuracy": 0.4270874824374914,
+      "num_tokens": 9827272.0,
+      "step": 740
+    },
+    {
+      "entropy": 2.4963896624743938,
+      "epoch": 0.6676681689200468,
+      "grad_norm": 0.010498046875,
+      "learning_rate": 1.498e-05,
+      "loss": 3.3969,
+      "mean_token_accuracy": 0.4328357223421335,
+      "num_tokens": 9955369.0,
+      "step": 750
+    },
+    {
+      "entropy": 2.47789601534605,
+      "epoch": 0.676570411172314,
+      "grad_norm": 0.015869140625,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 3.3672,
+      "mean_token_accuracy": 0.43644896559417246,
+      "num_tokens": 10096065.0,
+      "step": 760
+    },
+    {
+      "entropy": 2.4533426530659197,
+      "epoch": 0.6854726534245813,
+      "grad_norm": 0.00982666015625,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 3.3724,
+      "mean_token_accuracy": 0.43634699024260043,
+      "num_tokens": 10233806.0,
+      "step": 770
+    },
+    {
+      "entropy": 2.5112246222794057,
+      "epoch": 0.6943748956768486,
+      "grad_norm": 0.00885009765625,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 3.4341,
+      "mean_token_accuracy": 0.4313540508970618,
+      "num_tokens": 10364601.0,
+      "step": 780
+    },
+    {
+      "entropy": 2.467985014617443,
+      "epoch": 0.703277137929116,
+      "grad_norm": 0.01171875,
+      "learning_rate": 1.578e-05,
+      "loss": 3.379,
+      "mean_token_accuracy": 0.4354724214412272,
+      "num_tokens": 10500891.0,
+      "step": 790
+    },
+    {
+      "entropy": 2.4910646095871924,
+      "epoch": 0.7121793801813832,
+      "grad_norm": 0.0089111328125,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 3.4174,
+      "mean_token_accuracy": 0.4264952681958675,
+      "num_tokens": 10633325.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.7121793801813832,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 10633325.0,
+      "eval_coding_runtime": 95.7207,
+      "eval_coding_samples_per_second": 5.224,
+      "eval_coding_steps_per_second": 2.612,
+      "step": 800
+    },
+    {
+      "epoch": 0.7121793801813832,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 10633325.0,
+      "eval_chemistry_runtime": 53.6314,
+      "eval_chemistry_samples_per_second": 9.323,
+      "eval_chemistry_steps_per_second": 4.661,
+      "step": 800
+    },
+    {
+      "entropy": 2.421719251573086,
+      "epoch": 0.7210816224336505,
+      "grad_norm": 0.009033203125,
+      "learning_rate": 1.618e-05,
+      "loss": 3.373,
+      "mean_token_accuracy": 0.4373985629528761,
+      "num_tokens": 10764319.0,
+      "step": 810
+    },
+    {
+      "entropy": 2.43545116186142,
+      "epoch": 0.7299838646859178,
+      "grad_norm": 0.01287841796875,
+      "learning_rate": 1.638e-05,
+      "loss": 3.3444,
+      "mean_token_accuracy": 0.43578007342293856,
+      "num_tokens": 10897916.0,
+      "step": 820
+    },
+    {
+      "entropy": 2.5303331069648265,
+      "epoch": 0.738886106938185,
+      "grad_norm": 0.0125732421875,
+      "learning_rate": 1.658e-05,
+      "loss": 3.4252,
+      "mean_token_accuracy": 0.4248075334355235,
+      "num_tokens": 11030485.0,
+      "step": 830
+    },
+    {
+      "entropy": 2.4731515564024447,
+      "epoch": 0.7477883491904523,
+      "grad_norm": 0.007659912109375,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 3.4258,
+      "mean_token_accuracy": 0.43433472914621235,
+      "num_tokens": 11165356.0,
+      "step": 840
+    },
+    {
+      "entropy": 2.4785347141325476,
+      "epoch": 0.7566905914427197,
+      "grad_norm": 0.0087890625,
+      "learning_rate": 1.698e-05,
+      "loss": 3.4161,
+      "mean_token_accuracy": 0.4306844290345907,
+      "num_tokens": 11299223.0,
+      "step": 850
+    },
+    {
+      "entropy": 2.4768461272120477,
+      "epoch": 0.7655928336949869,
+      "grad_norm": 0.01287841796875,
+      "learning_rate": 1.718e-05,
+      "loss": 3.3776,
+      "mean_token_accuracy": 0.4344117846339941,
+      "num_tokens": 11436799.0,
+      "step": 860
+    },
+    {
+      "entropy": 2.5141974300146104,
+      "epoch": 0.7744950759472542,
+      "grad_norm": 0.00830078125,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 3.3958,
+      "mean_token_accuracy": 0.4330358326435089,
+      "num_tokens": 11567422.0,
+      "step": 870
+    },
+    {
+      "entropy": 2.515311509370804,
+      "epoch": 0.7833973181995215,
+      "grad_norm": 0.009765625,
+      "learning_rate": 1.758e-05,
+      "loss": 3.4031,
+      "mean_token_accuracy": 0.4312752116471529,
+      "num_tokens": 11703496.0,
+      "step": 880
+    },
+    {
+      "entropy": 2.453214881569147,
+      "epoch": 0.7922995604517888,
+      "grad_norm": 0.0093994140625,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 3.3834,
+      "mean_token_accuracy": 0.4389944301918149,
+      "num_tokens": 11840317.0,
+      "step": 890
+    },
+    {
+      "entropy": 2.6043091461062433,
+      "epoch": 0.801201802704056,
+      "grad_norm": 0.01116943359375,
+      "learning_rate": 1.798e-05,
+      "loss": 3.5713,
+      "mean_token_accuracy": 0.4135642783716321,
+      "num_tokens": 11965530.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.801201802704056,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 11965530.0,
+      "eval_coding_runtime": 95.8462,
+      "eval_coding_samples_per_second": 5.217,
+      "eval_coding_steps_per_second": 2.608,
+      "step": 900
+    },
+    {
+      "epoch": 0.801201802704056,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 11965530.0,
+      "eval_chemistry_runtime": 53.6177,
+      "eval_chemistry_samples_per_second": 9.325,
+      "eval_chemistry_steps_per_second": 4.663,
+      "step": 900
+    },
+    {
+      "entropy": 2.4632391795516013,
+      "epoch": 0.8101040449563234,
+      "grad_norm": 0.00836181640625,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 3.4352,
+      "mean_token_accuracy": 0.42954989019781353,
+      "num_tokens": 12097917.0,
+      "step": 910
+    },
+    {
+      "entropy": 2.468993365764618,
+      "epoch": 0.8190062872085907,
+      "grad_norm": 0.0089111328125,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 3.4071,
+      "mean_token_accuracy": 0.4282375952228904,
+      "num_tokens": 12224427.0,
+      "step": 920
+    },
+    {
+      "entropy": 2.468296863883734,
+      "epoch": 0.8279085294608579,
+      "grad_norm": 0.008056640625,
+      "learning_rate": 1.858e-05,
+      "loss": 3.3355,
+      "mean_token_accuracy": 0.43817838532850145,
+      "num_tokens": 12365332.0,
+      "step": 930
+    },
+    {
+      "entropy": 2.427542605996132,
+      "epoch": 0.8368107717131252,
+      "grad_norm": 0.01031494140625,
+      "learning_rate": 1.878e-05,
+      "loss": 3.317,
+      "mean_token_accuracy": 0.44397469637915493,
+      "num_tokens": 12509124.0,
+      "step": 940
+    },
+    {
+      "entropy": 2.4841547794640064,
+      "epoch": 0.8457130139653926,
+      "grad_norm": 0.018798828125,
+      "learning_rate": 1.898e-05,
+      "loss": 3.3654,
+      "mean_token_accuracy": 0.434748101234436,
+      "num_tokens": 12642651.0,
+      "step": 950
+    },
+    {
+      "entropy": 2.449143522232771,
+      "epoch": 0.8546152562176598,
+      "grad_norm": 0.00738525390625,
+      "learning_rate": 1.918e-05,
+      "loss": 3.4035,
+      "mean_token_accuracy": 0.43319537229835986,
+      "num_tokens": 12778408.0,
+      "step": 960
+    },
+    {
+      "entropy": 2.387574986368418,
+      "epoch": 0.8635174984699271,
+      "grad_norm": 0.0072021484375,
+      "learning_rate": 1.938e-05,
+      "loss": 3.2822,
+      "mean_token_accuracy": 0.4413015801459551,
+      "num_tokens": 12910630.0,
+      "step": 970
+    },
+    {
+      "entropy": 2.5341439098119736,
+      "epoch": 0.8724197407221944,
+      "grad_norm": 0.01055908203125,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 3.4749,
+      "mean_token_accuracy": 0.4271688721142709,
+      "num_tokens": 13046473.0,
+      "step": 980
+    },
+    {
+      "entropy": 2.5607902660965918,
+      "epoch": 0.8813219829744617,
+      "grad_norm": 0.00848388671875,
+      "learning_rate": 1.978e-05,
+      "loss": 3.4981,
+      "mean_token_accuracy": 0.4231617606244981,
+      "num_tokens": 13167768.0,
+      "step": 990
+    },
+    {
+      "entropy": 2.4919896230101584,
+      "epoch": 0.890224225226729,
+      "grad_norm": 0.009765625,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 3.3681,
+      "mean_token_accuracy": 0.43273249492049215,
+      "num_tokens": 13301659.0,
+      "step": 1000
+    },
+    {
+      "epoch": 0.890224225226729,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 13301659.0,
+      "eval_coding_runtime": 95.788,
+      "eval_coding_samples_per_second": 5.22,
+      "eval_coding_steps_per_second": 2.61,
+      "step": 1000
+    },
+    {
+      "epoch": 0.890224225226729,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 13301659.0,
+      "eval_chemistry_runtime": 53.3313,
+      "eval_chemistry_samples_per_second": 9.375,
+      "eval_chemistry_steps_per_second": 4.688,
+      "step": 1000
+    },
+    {
+      "entropy": 2.482378314435482,
+      "epoch": 0.8991264674789963,
+      "grad_norm": 0.0098876953125,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 3.4139,
+      "mean_token_accuracy": 0.4351740827783942,
+      "num_tokens": 13434364.0,
+      "step": 1010
+    },
+    {
+      "entropy": 2.397843288630247,
+      "epoch": 0.9080287097312636,
+      "grad_norm": 0.00909423828125,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 3.2598,
+      "mean_token_accuracy": 0.4428169864229858,
+      "num_tokens": 13570986.0,
+      "step": 1020
+    },
+    {
+      "entropy": 2.426850029826164,
+      "epoch": 0.9169309519835308,
+      "grad_norm": 0.0191650390625,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 3.3704,
+      "mean_token_accuracy": 0.43931040428578855,
+      "num_tokens": 13706343.0,
+      "step": 1030
+    },
+    {
+      "entropy": 2.5369828954339027,
+      "epoch": 0.9258331942357981,
+      "grad_norm": 0.013671875,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 3.4473,
+      "mean_token_accuracy": 0.42489275773987173,
+      "num_tokens": 13833901.0,
+      "step": 1040
+    },
+    {
+      "entropy": 2.495298910140991,
+      "epoch": 0.9347354364880655,
+      "grad_norm": 0.0186767578125,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 3.4641,
+      "mean_token_accuracy": 0.4274964226409793,
+      "num_tokens": 13954258.0,
+      "step": 1050
+    },
+    {
+      "entropy": 2.4657325744628906,
+      "epoch": 0.9436376787403328,
+      "grad_norm": 0.0081787109375,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 3.3954,
+      "mean_token_accuracy": 0.4306732133962214,
+      "num_tokens": 14093888.0,
+      "step": 1060
+    },
+    {
+      "entropy": 2.533431290090084,
+      "epoch": 0.9525399209926,
+      "grad_norm": 0.0133056640625,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 3.4682,
+      "mean_token_accuracy": 0.4237683041021228,
+      "num_tokens": 14222626.0,
+      "step": 1070
+    },
+    {
+      "entropy": 2.535231586545706,
+      "epoch": 0.9614421632448673,
+      "grad_norm": 0.00823974609375,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 3.4095,
+      "mean_token_accuracy": 0.4307479955255985,
+      "num_tokens": 14354918.0,
+      "step": 1080
+    },
+    {
+      "entropy": 2.507918354868889,
+      "epoch": 0.9703444054971346,
+      "grad_norm": 0.0076904296875,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 3.4265,
+      "mean_token_accuracy": 0.4296759738586843,
+      "num_tokens": 14493349.0,
+      "step": 1090
+    },
+    {
+      "entropy": 2.4675292283296586,
+      "epoch": 0.9792466477494018,
+      "grad_norm": 0.0084228515625,
+      "learning_rate": 1.978e-05,
+      "loss": 3.4234,
+      "mean_token_accuracy": 0.42840409390628337,
+      "num_tokens": 14620244.0,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9792466477494018,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 14620244.0,
+      "eval_coding_runtime": 95.7872,
+      "eval_coding_samples_per_second": 5.22,
+      "eval_coding_steps_per_second": 2.61,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9792466477494018,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 14620244.0,
+      "eval_chemistry_runtime": 53.6439,
+      "eval_chemistry_samples_per_second": 9.321,
+      "eval_chemistry_steps_per_second": 4.66,
+      "step": 1100
+    },
+    {
+      "entropy": 2.4005719617009165,
+      "epoch": 0.9881488900016692,
+      "grad_norm": 0.00958251953125,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 3.3593,
+      "mean_token_accuracy": 0.4352086253464222,
+      "num_tokens": 14760951.0,
+      "step": 1110
+    },
+    {
+      "entropy": 2.558166432380676,
+      "epoch": 0.9970511322539365,
+      "grad_norm": 0.00933837890625,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 3.5454,
+      "mean_token_accuracy": 0.42158968318253753,
+      "num_tokens": 14886593.0,
+      "step": 1120
+    },
+    {
+      "entropy": 2.456293193285897,
+      "epoch": 1.0053413453513604,
+      "grad_norm": 0.008544921875,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 3.329,
+      "mean_token_accuracy": 0.431678411744585,
+      "num_tokens": 15009964.0,
+      "step": 1130
+    },
+    {
+      "entropy": 2.3623578011989594,
+      "epoch": 1.0142435876036278,
+      "grad_norm": 0.01019287109375,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 3.2394,
+      "mean_token_accuracy": 0.4485295919701457,
+      "num_tokens": 15146469.0,
+      "step": 1140
+    },
+    {
+      "entropy": 2.537761741876602,
+      "epoch": 1.0231458298558949,
+      "grad_norm": 0.017578125,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 3.4544,
+      "mean_token_accuracy": 0.42700372273102405,
+      "num_tokens": 15274532.0,
+      "step": 1150
+    },
+    {
+      "entropy": 2.560138799250126,
+      "epoch": 1.0320480721081622,
+      "grad_norm": 0.01312255859375,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 3.5139,
+      "mean_token_accuracy": 0.42576750814914704,
+      "num_tokens": 15404569.0,
+      "step": 1160
+    },
+    {
+      "entropy": 2.4245593287050724,
+      "epoch": 1.0409503143604295,
+      "grad_norm": 0.00665283203125,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 3.3292,
+      "mean_token_accuracy": 0.4398623496294022,
+      "num_tokens": 15550449.0,
+      "step": 1170
+    },
+    {
+      "entropy": 2.447620178759098,
+      "epoch": 1.0498525566126968,
+      "grad_norm": 0.00872802734375,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 3.4233,
+      "mean_token_accuracy": 0.4298626586794853,
+      "num_tokens": 15681709.0,
+      "step": 1180
+    },
+    {
+      "entropy": 2.481611392647028,
+      "epoch": 1.0587547988649642,
+      "grad_norm": 0.01165771484375,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 3.3803,
+      "mean_token_accuracy": 0.43227153811603786,
+      "num_tokens": 15816502.0,
+      "step": 1190
+    },
+    {
+      "entropy": 2.441844770312309,
+      "epoch": 1.0676570411172315,
+      "grad_norm": 0.00958251953125,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 3.3808,
+      "mean_token_accuracy": 0.4364687578752637,
+      "num_tokens": 15949421.0,
+      "step": 1200
+    },
+    {
+      "epoch": 1.0676570411172315,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 15949421.0,
+      "eval_coding_runtime": 95.6237,
+      "eval_coding_samples_per_second": 5.229,
+      "eval_coding_steps_per_second": 2.614,
+      "step": 1200
+    },
+    {
+      "epoch": 1.0676570411172315,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 15949421.0,
+      "eval_chemistry_runtime": 53.6265,
+      "eval_chemistry_samples_per_second": 9.324,
+      "eval_chemistry_steps_per_second": 4.662,
+      "step": 1200
+    },
+    {
+      "entropy": 2.4238644503057003,
+      "epoch": 1.0765592833694986,
+      "grad_norm": 0.00921630859375,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 3.3419,
+      "mean_token_accuracy": 0.43726830892264845,
+      "num_tokens": 16088471.0,
+      "step": 1210
+    },
+    {
+      "entropy": 2.4736556738615034,
+      "epoch": 1.085461525621766,
+      "grad_norm": 0.0079345703125,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 3.403,
+      "mean_token_accuracy": 0.43131555467844007,
+      "num_tokens": 16221916.0,
+      "step": 1220
+    },
+    {
+      "entropy": 2.5666570000350477,
+      "epoch": 1.0943637678740332,
+      "grad_norm": 0.0107421875,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 3.4806,
+      "mean_token_accuracy": 0.4240891897119582,
+      "num_tokens": 16353558.0,
+      "step": 1230
+    },
+    {
+      "entropy": 2.5181381188333036,
+      "epoch": 1.1032660101263005,
+      "grad_norm": 0.00799560546875,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 3.4298,
+      "mean_token_accuracy": 0.4259687649086118,
+      "num_tokens": 16488154.0,
+      "step": 1240
+    },
+    {
+      "entropy": 2.514085105806589,
+      "epoch": 1.1121682523785679,
+      "grad_norm": 0.010498046875,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 3.4249,
+      "mean_token_accuracy": 0.42915988666936755,
+      "num_tokens": 16618151.0,
+      "step": 1250
+    },
+    {
+      "entropy": 2.4717010237276553,
+      "epoch": 1.1210704946308352,
+      "grad_norm": 0.00811767578125,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 3.3908,
+      "mean_token_accuracy": 0.43228524839505555,
+      "num_tokens": 16756828.0,
+      "step": 1260
+    },
+    {
+      "entropy": 2.517887058854103,
+      "epoch": 1.1299727368831025,
+      "grad_norm": 0.01141357421875,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 3.4328,
+      "mean_token_accuracy": 0.42816000785678626,
+      "num_tokens": 16890026.0,
+      "step": 1270
+    },
+    {
+      "entropy": 2.4103298880159856,
+      "epoch": 1.1388749791353696,
+      "grad_norm": 0.01025390625,
+      "learning_rate": 1.938e-05,
+      "loss": 3.3448,
+      "mean_token_accuracy": 0.4348219852894545,
+      "num_tokens": 17023681.0,
+      "step": 1280
+    },
+    {
+      "entropy": 2.509786891192198,
+      "epoch": 1.147777221387637,
+      "grad_norm": 0.0179443359375,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 3.4512,
+      "mean_token_accuracy": 0.4285705531015992,
+      "num_tokens": 17155619.0,
+      "step": 1290
+    },
+    {
+      "entropy": 2.503946328163147,
+      "epoch": 1.1566794636399043,
+      "grad_norm": 0.01123046875,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 3.4103,
+      "mean_token_accuracy": 0.4285626333206892,
+      "num_tokens": 17286672.0,
+      "step": 1300
+    },
+    {
+      "epoch": 1.1566794636399043,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 17286672.0,
+      "eval_coding_runtime": 95.8416,
+      "eval_coding_samples_per_second": 5.217,
+      "eval_coding_steps_per_second": 2.608,
+      "step": 1300
+    },
+    {
+      "epoch": 1.1566794636399043,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 17286672.0,
+      "eval_chemistry_runtime": 53.6206,
+      "eval_chemistry_samples_per_second": 9.325,
+      "eval_chemistry_steps_per_second": 4.662,
+      "step": 1300
+    },
+    {
+      "entropy": 2.511377315223217,
+      "epoch": 1.1655817058921716,
+      "grad_norm": 0.00958251953125,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 3.493,
+      "mean_token_accuracy": 0.4217289647087455,
+      "num_tokens": 17418216.0,
+      "step": 1310
+    },
+    {
+      "entropy": 2.5961489491164684,
+      "epoch": 1.174483948144439,
+      "grad_norm": 0.00958251953125,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 3.4532,
+      "mean_token_accuracy": 0.4226861286908388,
+      "num_tokens": 17551581.0,
+      "step": 1320
+    },
+    {
+      "entropy": 2.4733773186802863,
+      "epoch": 1.1833861903967062,
+      "grad_norm": 0.0078125,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 3.406,
+      "mean_token_accuracy": 0.43328642603009937,
+      "num_tokens": 17685328.0,
+      "step": 1330
+    },
+    {
+      "entropy": 2.504808610677719,
+      "epoch": 1.1922884326489736,
+      "grad_norm": 0.010986328125,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 3.3902,
+      "mean_token_accuracy": 0.43533999361097814,
+      "num_tokens": 17821205.0,
+      "step": 1340
+    },
+    {
+      "entropy": 2.554856265336275,
+      "epoch": 1.2011906749012407,
+      "grad_norm": 0.008544921875,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 3.5145,
+      "mean_token_accuracy": 0.4222357952967286,
+      "num_tokens": 17950081.0,
+      "step": 1350
+    },
+    {
+      "entropy": 2.4698183365166186,
+      "epoch": 1.210092917153508,
+      "grad_norm": 0.007568359375,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 3.3678,
+      "mean_token_accuracy": 0.4337294826284051,
+      "num_tokens": 18087759.0,
+      "step": 1360
+    },
+    {
+      "entropy": 2.491861944645643,
+      "epoch": 1.2189951594057753,
+      "grad_norm": 0.00933837890625,
+      "learning_rate": 1.918e-05,
+      "loss": 3.3956,
+      "mean_token_accuracy": 0.43155251136049627,
+      "num_tokens": 18229440.0,
+      "step": 1370
+    },
+    {
+      "entropy": 2.544880051165819,
+      "epoch": 1.2278974016580426,
+      "grad_norm": 0.01019287109375,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 3.4837,
+      "mean_token_accuracy": 0.4227764903567731,
+      "num_tokens": 18355732.0,
+      "step": 1380
+    },
+    {
+      "entropy": 2.608750829845667,
+      "epoch": 1.23679964391031,
+      "grad_norm": 0.0096435546875,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 3.5353,
+      "mean_token_accuracy": 0.42066937144845723,
+      "num_tokens": 18489654.0,
+      "step": 1390
+    },
+    {
+      "entropy": 2.4528128817677497,
+      "epoch": 1.2457018861625773,
+      "grad_norm": 0.0067138671875,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 3.3402,
+      "mean_token_accuracy": 0.43648717207834126,
+      "num_tokens": 18626245.0,
+      "step": 1400
+    },
+    {
+      "epoch": 1.2457018861625773,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 18626245.0,
+      "eval_coding_runtime": 95.8265,
+      "eval_coding_samples_per_second": 5.218,
+      "eval_coding_steps_per_second": 2.609,
+      "step": 1400
+    },
+    {
+      "epoch": 1.2457018861625773,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 18626245.0,
+      "eval_chemistry_runtime": 53.6283,
+      "eval_chemistry_samples_per_second": 9.323,
+      "eval_chemistry_steps_per_second": 4.662,
+      "step": 1400
+    },
+    {
+      "entropy": 2.4507749184966086,
+      "epoch": 1.2546041284148446,
+      "grad_norm": 0.00872802734375,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 3.3585,
+      "mean_token_accuracy": 0.43596437461674215,
+      "num_tokens": 18767785.0,
+      "step": 1410
+    },
+    {
+      "entropy": 2.432799159735441,
+      "epoch": 1.2635063706671117,
+      "grad_norm": 0.012451171875,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 3.309,
+      "mean_token_accuracy": 0.4405426589772105,
+      "num_tokens": 18899632.0,
+      "step": 1420
+    },
+    {
+      "entropy": 2.523366042226553,
+      "epoch": 1.272408612919379,
+      "grad_norm": 0.01068115234375,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 3.4447,
+      "mean_token_accuracy": 0.42682514479383826,
+      "num_tokens": 19039727.0,
+      "step": 1430
+    },
+    {
+      "entropy": 2.522237445414066,
+      "epoch": 1.2813108551716463,
+      "grad_norm": 0.01190185546875,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 3.4611,
+      "mean_token_accuracy": 0.4245713768526912,
+      "num_tokens": 19173635.0,
+      "step": 1440
+    },
+    {
+      "entropy": 2.4513756319880486,
+      "epoch": 1.2902130974239137,
+      "grad_norm": 0.0079345703125,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 3.3933,
+      "mean_token_accuracy": 0.4320486688986421,
+      "num_tokens": 19317295.0,
+      "step": 1450
+    },
+    {
+      "entropy": 2.417076099663973,
+      "epoch": 1.299115339676181,
+      "grad_norm": 0.01165771484375,
+      "learning_rate": 1.898e-05,
+      "loss": 3.3298,
+      "mean_token_accuracy": 0.4392012171447277,
+      "num_tokens": 19453495.0,
+      "step": 1460
+    },
+    {
+      "entropy": 2.4171031497418882,
+      "epoch": 1.3080175819284483,
+      "grad_norm": 0.007720947265625,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 3.3321,
+      "mean_token_accuracy": 0.4389203391969204,
+      "num_tokens": 19588860.0,
+      "step": 1470
+    },
+    {
+      "entropy": 2.475263304263353,
+      "epoch": 1.3169198241807156,
+      "grad_norm": 0.00787353515625,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 3.423,
+      "mean_token_accuracy": 0.42983661517500876,
+      "num_tokens": 19719086.0,
+      "step": 1480
+    },
+    {
+      "entropy": 2.4915241256356238,
+      "epoch": 1.3258220664329827,
+      "grad_norm": 0.01251220703125,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 3.3932,
+      "mean_token_accuracy": 0.4291253449395299,
+      "num_tokens": 19851321.0,
+      "step": 1490
+    },
+    {
+      "entropy": 2.515060818940401,
+      "epoch": 1.33472430868525,
+      "grad_norm": 0.01190185546875,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 3.4452,
+      "mean_token_accuracy": 0.4248286823742092,
+      "num_tokens": 19981272.0,
+      "step": 1500
+    },
+    {
+      "epoch": 1.33472430868525,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 19981272.0,
+      "eval_coding_runtime": 95.9024,
+      "eval_coding_samples_per_second": 5.214,
+      "eval_coding_steps_per_second": 2.607,
+      "step": 1500
+    },
+    {
+      "epoch": 1.33472430868525,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 19981272.0,
+      "eval_chemistry_runtime": 53.3916,
+      "eval_chemistry_samples_per_second": 9.365,
+      "eval_chemistry_steps_per_second": 4.682,
+      "step": 1500
+    },
+    {
+      "entropy": 2.537444035708904,
+      "epoch": 1.3436265509375174,
+      "grad_norm": 0.01397705078125,
+      "learning_rate": 1.886888888888889e-05,
+      "loss": 3.4621,
+      "mean_token_accuracy": 0.42645752560347316,
+      "num_tokens": 20112097.0,
+      "step": 1510
+    },
+    {
+      "entropy": 2.4578563667833806,
+      "epoch": 1.3525287931897847,
+      "grad_norm": 0.00701904296875,
+      "learning_rate": 1.884666666666667e-05,
+      "loss": 3.4011,
+      "mean_token_accuracy": 0.43212775196880104,
+      "num_tokens": 20243034.0,
+      "step": 1520
+    },
+    {
+      "entropy": 2.5289953693747522,
+      "epoch": 1.361431035442052,
+      "grad_norm": 0.015380859375,
+      "learning_rate": 1.8824444444444445e-05,
+      "loss": 3.4629,
+      "mean_token_accuracy": 0.4234733306802809,
+      "num_tokens": 20372999.0,
+      "step": 1530
+    },
+    {
+      "entropy": 2.4936239890754224,
+      "epoch": 1.3703332776943191,
+      "grad_norm": 0.007659912109375,
+      "learning_rate": 1.8802222222222226e-05,
+      "loss": 3.4047,
+      "mean_token_accuracy": 0.43090203367173674,
+      "num_tokens": 20509028.0,
+      "step": 1540
+    },
+    {
+      "entropy": 2.6229711353778837,
+      "epoch": 1.3792355199465867,
+      "grad_norm": 0.0081787109375,
+      "learning_rate": 1.878e-05,
+      "loss": 3.6425,
+      "mean_token_accuracy": 0.41157348044216635,
+      "num_tokens": 20633479.0,
+      "step": 1550
+    },
+    {
+      "entropy": 2.507130479812622,
+      "epoch": 1.3881377621988538,
+      "grad_norm": 0.01025390625,
+      "learning_rate": 1.875777777777778e-05,
+      "loss": 3.4246,
+      "mean_token_accuracy": 0.42894090730696915,
+      "num_tokens": 20762668.0,
+      "step": 1560
+    },
+    {
+      "entropy": 2.56754671856761,
+      "epoch": 1.397040004451121,
+      "grad_norm": 0.00732421875,
+      "learning_rate": 1.873555555555556e-05,
+      "loss": 3.4484,
+      "mean_token_accuracy": 0.4255117969587445,
+      "num_tokens": 20894609.0,
+      "step": 1570
+    },
+    {
+      "entropy": 2.6108976893126963,
+      "epoch": 1.4059422467033884,
+      "grad_norm": 0.01123046875,
+      "learning_rate": 1.8713333333333336e-05,
+      "loss": 3.5745,
+      "mean_token_accuracy": 0.4152322731912136,
+      "num_tokens": 21025277.0,
+      "step": 1580
+    },
+    {
+      "entropy": 2.575570172071457,
+      "epoch": 1.4148444889556557,
+      "grad_norm": 0.016357421875,
+      "learning_rate": 1.8691111111111114e-05,
+      "loss": 3.5483,
+      "mean_token_accuracy": 0.4183428635820746,
+      "num_tokens": 21156848.0,
+      "step": 1590
+    },
+    {
+      "entropy": 2.4909443251788614,
+      "epoch": 1.423746731207923,
+      "grad_norm": 0.01080322265625,
+      "learning_rate": 1.866888888888889e-05,
+      "loss": 3.4169,
+      "mean_token_accuracy": 0.42875550473108887,
+      "num_tokens": 21292320.0,
+      "step": 1600
+    },
+    {
+      "epoch": 1.423746731207923,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 21292320.0,
+      "eval_coding_runtime": 95.8321,
+      "eval_coding_samples_per_second": 5.217,
+      "eval_coding_steps_per_second": 2.609,
+      "step": 1600
+    },
+    {
+      "epoch": 1.423746731207923,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 21292320.0,
+      "eval_chemistry_runtime": 53.6996,
+      "eval_chemistry_samples_per_second": 9.311,
+      "eval_chemistry_steps_per_second": 4.656,
+      "step": 1600
+    },
+    {
+      "entropy": 2.5857157841324807,
+      "epoch": 1.4326489734601902,
+      "grad_norm": 0.0120849609375,
+      "learning_rate": 1.864666666666667e-05,
+      "loss": 3.5036,
+      "mean_token_accuracy": 0.4268370010890067,
+      "num_tokens": 21426944.0,
+      "step": 1610
+    },
+    {
+      "entropy": 2.471120712906122,
+      "epoch": 1.4415512157124577,
+      "grad_norm": 0.01239013671875,
+      "learning_rate": 1.8624444444444446e-05,
+      "loss": 3.4141,
+      "mean_token_accuracy": 0.43633277975022794,
+      "num_tokens": 21563668.0,
+      "step": 1620
+    },
+    {
+      "entropy": 2.572203493118286,
+      "epoch": 1.4504534579647248,
+      "grad_norm": 0.0074462890625,
+      "learning_rate": 1.8602222222222224e-05,
+      "loss": 3.5546,
+      "mean_token_accuracy": 0.416438902169466,
+      "num_tokens": 21687993.0,
+      "step": 1630
+    },
+    {
+      "entropy": 2.4553592331707477,
+      "epoch": 1.4593557002169921,
+      "grad_norm": 0.00927734375,
+      "learning_rate": 1.858e-05,
+      "loss": 3.42,
+      "mean_token_accuracy": 0.43670414835214616,
+      "num_tokens": 21817298.0,
+      "step": 1640
+    },
+    {
+      "entropy": 2.5596740052103995,
+      "epoch": 1.4682579424692594,
+      "grad_norm": 0.00897216796875,
+      "learning_rate": 1.855777777777778e-05,
+      "loss": 3.479,
+      "mean_token_accuracy": 0.42424469059333203,
+      "num_tokens": 21947593.0,
+      "step": 1650
+    },
+    {
+      "entropy": 2.558923949301243,
+      "epoch": 1.4771601847215268,
+      "grad_norm": 0.01104736328125,
+      "learning_rate": 1.8535555555555557e-05,
+      "loss": 3.4637,
+      "mean_token_accuracy": 0.42529286816716194,
+      "num_tokens": 22083010.0,
+      "step": 1660
+    },
+    {
+      "entropy": 2.5305933080613614,
+      "epoch": 1.486062426973794,
+      "grad_norm": 0.01385498046875,
+      "learning_rate": 1.8513333333333335e-05,
+      "loss": 3.4958,
+      "mean_token_accuracy": 0.4228696158155799,
+      "num_tokens": 22216759.0,
+      "step": 1670
+    },
+    {
+      "entropy": 2.5268216267228127,
+      "epoch": 1.4949646692260612,
+      "grad_norm": 0.010498046875,
+      "learning_rate": 1.8491111111111112e-05,
+      "loss": 3.4217,
+      "mean_token_accuracy": 0.4293845698237419,
+      "num_tokens": 22350046.0,
+      "step": 1680
+    },
+    {
+      "entropy": 2.5090344458818437,
+      "epoch": 1.5038669114783287,
+      "grad_norm": 0.01373291015625,
+      "learning_rate": 1.846888888888889e-05,
+      "loss": 3.4432,
+      "mean_token_accuracy": 0.4325392559170723,
+      "num_tokens": 22485677.0,
+      "step": 1690
+    },
+    {
+      "entropy": 2.4932730071246625,
+      "epoch": 1.5127691537305958,
+      "grad_norm": 0.00860595703125,
+      "learning_rate": 1.8446666666666667e-05,
+      "loss": 3.3879,
+      "mean_token_accuracy": 0.4324304299429059,
+      "num_tokens": 22615933.0,
+      "step": 1700
+    },
+    {
+      "epoch": 1.5127691537305958,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 22615933.0,
+      "eval_coding_runtime": 96.0736,
+      "eval_coding_samples_per_second": 5.204,
+      "eval_coding_steps_per_second": 2.602,
+      "step": 1700
+    },
+    {
+      "epoch": 1.5127691537305958,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 22615933.0,
+      "eval_chemistry_runtime": 53.6801,
+      "eval_chemistry_samples_per_second": 9.314,
+      "eval_chemistry_steps_per_second": 4.657,
+      "step": 1700
+    },
+    {
+      "entropy": 2.499319192022085,
+      "epoch": 1.5216713959828632,
+      "grad_norm": 0.01531982421875,
+      "learning_rate": 1.842444444444445e-05,
+      "loss": 3.4393,
+      "mean_token_accuracy": 0.42634991379454734,
+      "num_tokens": 22749756.0,
+      "step": 1710
+    },
+    {
+      "entropy": 2.3928098790347576,
+      "epoch": 1.5305736382351305,
+      "grad_norm": 0.00811767578125,
+      "learning_rate": 1.8402222222222223e-05,
+      "loss": 3.2777,
+      "mean_token_accuracy": 0.44273359794169664,
+      "num_tokens": 22886843.0,
+      "step": 1720
+    },
+    {
+      "entropy": 2.515210550278425,
+      "epoch": 1.5394758804873978,
+      "grad_norm": 0.0145263671875,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 3.4278,
+      "mean_token_accuracy": 0.430076424498111,
+      "num_tokens": 23013890.0,
+      "step": 1730
+    },
+    {
+      "entropy": 2.511288223415613,
+      "epoch": 1.5483781227396651,
+      "grad_norm": 0.0174560546875,
+      "learning_rate": 1.8357777777777778e-05,
+      "loss": 3.4467,
+      "mean_token_accuracy": 0.42953401962295173,
+      "num_tokens": 23140244.0,
+      "step": 1740
+    },
+    {
+      "entropy": 2.5117117315530777,
+      "epoch": 1.5572803649919322,
+      "grad_norm": 0.01177978515625,
+      "learning_rate": 1.833555555555556e-05,
+      "loss": 3.437,
+      "mean_token_accuracy": 0.43029365949332715,
+      "num_tokens": 23268188.0,
+      "step": 1750
+    },
+    {
+      "entropy": 2.472087085992098,
+      "epoch": 1.5661826072441998,
+      "grad_norm": 0.00958251953125,
+      "learning_rate": 1.8313333333333333e-05,
+      "loss": 3.3994,
+      "mean_token_accuracy": 0.4351077421568334,
+      "num_tokens": 23405121.0,
+      "step": 1760
+    },
+    {
+      "entropy": 2.5334790728986265,
+      "epoch": 1.5750848494964669,
+      "grad_norm": 0.00897216796875,
+      "learning_rate": 1.8291111111111114e-05,
+      "loss": 3.456,
+      "mean_token_accuracy": 0.42388113122433424,
+      "num_tokens": 23535787.0,
+      "step": 1770
+    },
+    {
+      "entropy": 2.5175728268921373,
+      "epoch": 1.5839870917487342,
+      "grad_norm": 0.007049560546875,
+      "learning_rate": 1.8268888888888888e-05,
+      "loss": 3.4153,
+      "mean_token_accuracy": 0.426735052280128,
+      "num_tokens": 23671826.0,
+      "step": 1780
+    },
+    {
+      "entropy": 2.522493792325258,
+      "epoch": 1.5928893340010015,
+      "grad_norm": 0.007781982421875,
+      "learning_rate": 1.824666666666667e-05,
+      "loss": 3.4456,
+      "mean_token_accuracy": 0.42770035564899445,
+      "num_tokens": 23805612.0,
+      "step": 1790
+    },
+    {
+      "entropy": 2.5342708349227907,
+      "epoch": 1.6017915762532688,
+      "grad_norm": 0.0174560546875,
+      "learning_rate": 1.8224444444444447e-05,
+      "loss": 3.4542,
+      "mean_token_accuracy": 0.4265250092372298,
+      "num_tokens": 23933374.0,
+      "step": 1800
+    },
+    {
+      "epoch": 1.6017915762532688,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 23933374.0,
+      "eval_coding_runtime": 95.8632,
+      "eval_coding_samples_per_second": 5.216,
+      "eval_coding_steps_per_second": 2.608,
+      "step": 1800
+    },
+    {
+      "epoch": 1.6017915762532688,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 23933374.0,
+      "eval_chemistry_runtime": 53.6651,
+      "eval_chemistry_samples_per_second": 9.317,
+      "eval_chemistry_steps_per_second": 4.659,
+      "step": 1800
+    },
+    {
+      "entropy": 2.4730284474790096,
+      "epoch": 1.6106938185055362,
+      "grad_norm": 0.01123046875,
+      "learning_rate": 1.8202222222222225e-05,
+      "loss": 3.4249,
+      "mean_token_accuracy": 0.4301054811105132,
+      "num_tokens": 24070188.0,
+      "step": 1810
+    },
+    {
+      "entropy": 2.4333154663443564,
+      "epoch": 1.6195960607578033,
+      "grad_norm": 0.0162353515625,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 3.3849,
+      "mean_token_accuracy": 0.4358032012358308,
+      "num_tokens": 24210156.0,
+      "step": 1820
+    },
+    {
+      "entropy": 2.548785130679607,
+      "epoch": 1.6284983030100708,
+      "grad_norm": 0.00994873046875,
+      "learning_rate": 1.815777777777778e-05,
+      "loss": 3.4663,
+      "mean_token_accuracy": 0.4266028679907322,
+      "num_tokens": 24335746.0,
+      "step": 1830
+    },
+    {
+      "entropy": 2.493963433802128,
+      "epoch": 1.637400545262338,
+      "grad_norm": 0.0115966796875,
+      "learning_rate": 1.8135555555555557e-05,
+      "loss": 3.4112,
+      "mean_token_accuracy": 0.43058306723833084,
+      "num_tokens": 24462726.0,
+      "step": 1840
+    },
+    {
+      "entropy": 2.4432963475584986,
+      "epoch": 1.6463027875146052,
+      "grad_norm": 0.00848388671875,
+      "learning_rate": 1.8113333333333335e-05,
+      "loss": 3.3659,
+      "mean_token_accuracy": 0.4354290099814534,
+      "num_tokens": 24597391.0,
+      "step": 1850
+    },
+    {
+      "entropy": 2.4352858766913412,
+      "epoch": 1.6552050297668726,
+      "grad_norm": 0.0150146484375,
+      "learning_rate": 1.8091111111111113e-05,
+      "loss": 3.3265,
+      "mean_token_accuracy": 0.44209435898810623,
+      "num_tokens": 24732815.0,
+      "step": 1860
+    },
+    {
+      "entropy": 2.494772292673588,
+      "epoch": 1.6641072720191397,
+      "grad_norm": 0.00811767578125,
+      "learning_rate": 1.806888888888889e-05,
+      "loss": 3.4225,
+      "mean_token_accuracy": 0.42676153369247916,
+      "num_tokens": 24869428.0,
+      "step": 1870
+    },
+    {
+      "entropy": 2.4837465301156043,
+      "epoch": 1.6730095142714072,
+      "grad_norm": 0.0181884765625,
+      "learning_rate": 1.8046666666666668e-05,
+      "loss": 3.3856,
+      "mean_token_accuracy": 0.43438148498535156,
+      "num_tokens": 25002091.0,
+      "step": 1880
+    },
+    {
+      "entropy": 2.5113090477883815,
+      "epoch": 1.6819117565236743,
+      "grad_norm": 0.0087890625,
+      "learning_rate": 1.8024444444444445e-05,
+      "loss": 3.4708,
+      "mean_token_accuracy": 0.426338127348572,
+      "num_tokens": 25133714.0,
+      "step": 1890
+    },
+    {
+      "entropy": 2.4073795929551123,
+      "epoch": 1.6908139987759419,
+      "grad_norm": 0.0106201171875,
+      "learning_rate": 1.8002222222222223e-05,
+      "loss": 3.3643,
+      "mean_token_accuracy": 0.4404640804976225,
+      "num_tokens": 25267949.0,
+      "step": 1900
+    },
+    {
+      "epoch": 1.6908139987759419,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 25267949.0,
+      "eval_coding_runtime": 95.8242,
+      "eval_coding_samples_per_second": 5.218,
+      "eval_coding_steps_per_second": 2.609,
+      "step": 1900
+    },
+    {
+      "epoch": 1.6908139987759419,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 25267949.0,
+      "eval_chemistry_runtime": 53.6517,
+      "eval_chemistry_samples_per_second": 9.319,
+      "eval_chemistry_steps_per_second": 4.66,
+      "step": 1900
+    },
+    {
+      "entropy": 2.4964852534234523,
+      "epoch": 1.699716241028209,
+      "grad_norm": 0.01220703125,
+      "learning_rate": 1.798e-05,
+      "loss": 3.3746,
+      "mean_token_accuracy": 0.43742703888565304,
+      "num_tokens": 25401385.0,
+      "step": 1910
+    },
+    {
+      "entropy": 2.5503848932683466,
+      "epoch": 1.7086184832804763,
+      "grad_norm": 0.008056640625,
+      "learning_rate": 1.7957777777777778e-05,
+      "loss": 3.4842,
+      "mean_token_accuracy": 0.42500705253332854,
+      "num_tokens": 25525772.0,
+      "step": 1920
+    },
+    {
+      "entropy": 2.5723555073142053,
+      "epoch": 1.7175207255327436,
+      "grad_norm": 0.01055908203125,
+      "learning_rate": 1.7935555555555556e-05,
+      "loss": 3.4885,
+      "mean_token_accuracy": 0.4221608478575945,
+      "num_tokens": 25657946.0,
+      "step": 1930
+    },
+    {
+      "entropy": 2.5482451558113097,
+      "epoch": 1.7264229677850107,
+      "grad_norm": 0.01416015625,
+      "learning_rate": 1.7913333333333337e-05,
+      "loss": 3.4748,
+      "mean_token_accuracy": 0.42012830562889575,
+      "num_tokens": 25787727.0,
+      "step": 1940
+    },
+    {
+      "entropy": 2.4831707894802095,
+      "epoch": 1.7353252100372782,
+      "grad_norm": 0.008544921875,
+      "learning_rate": 1.789111111111111e-05,
+      "loss": 3.402,
+      "mean_token_accuracy": 0.4355609222315252,
+      "num_tokens": 25925569.0,
+      "step": 1950
+    },
+    {
+      "entropy": 2.4309555172920225,
+      "epoch": 1.7442274522895453,
+      "grad_norm": 0.0072021484375,
+      "learning_rate": 1.7868888888888892e-05,
+      "loss": 3.3031,
+      "mean_token_accuracy": 0.4397842838428915,
+      "num_tokens": 26058026.0,
+      "step": 1960
+    },
+    {
+      "entropy": 2.4790328785777094,
+      "epoch": 1.7531296945418127,
+      "grad_norm": 0.00823974609375,
+      "learning_rate": 1.7846666666666666e-05,
+      "loss": 3.3659,
+      "mean_token_accuracy": 0.4327216723002493,
+      "num_tokens": 26195268.0,
+      "step": 1970
+    },
+    {
+      "entropy": 2.3836306020617486,
+      "epoch": 1.76203193679408,
+      "grad_norm": 0.00848388671875,
+      "learning_rate": 1.7824444444444447e-05,
+      "loss": 3.3147,
+      "mean_token_accuracy": 0.4457951638847589,
+      "num_tokens": 26326631.0,
+      "step": 1980
+    },
+    {
+      "entropy": 2.5131599582731723,
+      "epoch": 1.7709341790463473,
+      "grad_norm": 0.0086669921875,
+      "learning_rate": 1.780222222222222e-05,
+      "loss": 3.4339,
+      "mean_token_accuracy": 0.42837768141180277,
+      "num_tokens": 26462379.0,
+      "step": 1990
+    },
+    {
+      "entropy": 2.4848237335681915,
+      "epoch": 1.7798364212986146,
+      "grad_norm": 0.00921630859375,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 3.3823,
+      "mean_token_accuracy": 0.4355517864227295,
+      "num_tokens": 26591735.0,
+      "step": 2000
+    },
+    {
+      "epoch": 1.7798364212986146,
+      "eval_coding_entropy": 1.266241003036499,
+      "eval_coding_loss": 1.2625732421875,
+      "eval_coding_mean_token_accuracy": 0.6838934738636017,
+      "eval_coding_num_tokens": 26591735.0,
+      "eval_coding_runtime": 95.8115,
+      "eval_coding_samples_per_second": 5.219,
+      "eval_coding_steps_per_second": 2.609,
+      "step": 2000
+    },
+    {
+      "epoch": 1.7798364212986146,
+      "eval_chemistry_entropy": 2.520227357387543,
+      "eval_chemistry_loss": 3.5777227878570557,
+      "eval_chemistry_mean_token_accuracy": 0.4262875291109085,
+      "eval_chemistry_num_tokens": 26591735.0,
+      "eval_chemistry_runtime": 53.3564,
+      "eval_chemistry_samples_per_second": 9.371,
+      "eval_chemistry_steps_per_second": 4.685,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.2087693595689667e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98f6a4eed050ba43d4f1f37cb84e4a0e4c7e84f48c9b66045451e2ca37bc478d
+size 6417