Upload folder using huggingface_hub

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-1500/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

checkpoint-1500/chat_template.jinja

@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

checkpoint-1500/config.json

@@ -0,0 +1,86 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3840,
+  "initializer_range": 0.02,
+  "intermediate_size": 15360,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": {
+    "factor": 8.0,
+    "rope_type": "linear"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 1024,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 262208
+}

checkpoint-1500/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 2,
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.56.1"
+}

checkpoint-1500/model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a4c14f38ffc9e23d873fa999eea1fc965dcedcec267e7a99f07c6fdbe8760d8
+size 4915892992

checkpoint-1500/model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0335ce336b31e38c581326acb77ff28330c2073764b1924ad7108d788b56ba68
+size 4931294472

checkpoint-1500/model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0d6a60cf6a1708be0613ab7c4d10457ad99b98c02e40ebd5f320fe02aa0d50c
+size 4931294528

checkpoint-1500/model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bd9340bc368427bcc805e329f32ec7f6a73ab12274d00a8b9c03d49fe0853d0
+size 4931294528

checkpoint-1500/model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d80ec30fa09df9cb78149b3af4da5e7f22bbe89087198871d4f36c70903e30a6
+size 3822364808

checkpoint-1500/model.safetensors.index.json

@@ -0,0 +1,634 @@
+{
+  "metadata": {
+    "total_parameters": 11766034176,
+    "total_size": 23532068352
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.norm.weight": "model-00005-of-00005.safetensors"
+  }
+}

checkpoint-1500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf05c833622814569f76ab59887875cc6f724c722b9f075d3d7e4b135d9edef9
+size 18355

checkpoint-1500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7610f10af744f2895441f2aca56d71350a94d1d448becf2f03f07437fe847e7
+size 14645

checkpoint-1500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb8afd6405fc2f72bcdc3e89147a10597d475bf4457b8f98d9c27aa5442f3ef9
+size 1465

checkpoint-1500/special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

checkpoint-1500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

checkpoint-1500/trainer_state.json

@@ -0,0 +1,1699 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.4,
+  "eval_steps": 100,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 3.6945120811462404,
+      "epoch": 0.016,
+      "grad_norm": 0.015869140625,
+      "learning_rate": 1.8e-07,
+      "loss": 5.0851,
+      "mean_token_accuracy": 0.2927759636193514,
+      "num_tokens": 195524.0,
+      "step": 10
+    },
+    {
+      "entropy": 3.7313055276870726,
+      "epoch": 0.032,
+      "grad_norm": 0.01495361328125,
+      "learning_rate": 3.8e-07,
+      "loss": 5.0802,
+      "mean_token_accuracy": 0.289933389890939,
+      "num_tokens": 390903.0,
+      "step": 20
+    },
+    {
+      "entropy": 3.705480118095875,
+      "epoch": 0.048,
+      "grad_norm": 0.0252685546875,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 5.0794,
+      "mean_token_accuracy": 0.29482015436515213,
+      "num_tokens": 589868.0,
+      "step": 30
+    },
+    {
+      "entropy": 3.727609448134899,
+      "epoch": 0.064,
+      "grad_norm": 0.023193359375,
+      "learning_rate": 7.8e-07,
+      "loss": 5.0815,
+      "mean_token_accuracy": 0.29059267388656734,
+      "num_tokens": 791190.0,
+      "step": 40
+    },
+    {
+      "entropy": 3.7325011074542997,
+      "epoch": 0.08,
+      "grad_norm": 0.0198974609375,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 5.0765,
+      "mean_token_accuracy": 0.29154324913397434,
+      "num_tokens": 989860.0,
+      "step": 50
+    },
+    {
+      "entropy": 3.7289695993065832,
+      "epoch": 0.096,
+      "grad_norm": 0.015869140625,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 5.084,
+      "mean_token_accuracy": 0.2919090397655964,
+      "num_tokens": 1181777.0,
+      "step": 60
+    },
+    {
+      "entropy": 3.696782369911671,
+      "epoch": 0.112,
+      "grad_norm": 0.0157470703125,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 5.051,
+      "mean_token_accuracy": 0.2921869015321136,
+      "num_tokens": 1385513.0,
+      "step": 70
+    },
+    {
+      "entropy": 3.7105689987540247,
+      "epoch": 0.128,
+      "grad_norm": 0.0186767578125,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 5.1046,
+      "mean_token_accuracy": 0.2894379127770662,
+      "num_tokens": 1582368.0,
+      "step": 80
+    },
+    {
+      "entropy": 3.6783719427883623,
+      "epoch": 0.144,
+      "grad_norm": 0.018798828125,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 5.049,
+      "mean_token_accuracy": 0.29489915193989874,
+      "num_tokens": 1773764.0,
+      "step": 90
+    },
+    {
+      "entropy": 3.7497646793723107,
+      "epoch": 0.16,
+      "grad_norm": 0.0184326171875,
+      "learning_rate": 1.98e-06,
+      "loss": 5.1421,
+      "mean_token_accuracy": 0.285641394648701,
+      "num_tokens": 1970077.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 1970077.0,
+      "eval_biology_runtime": 48.3951,
+      "eval_biology_samples_per_second": 10.332,
+      "eval_biology_steps_per_second": 2.583,
+      "step": 100
+    },
+    {
+      "entropy": 3.7205758571624754,
+      "epoch": 0.176,
+      "grad_norm": 0.022216796875,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 5.1124,
+      "mean_token_accuracy": 0.2885961330495775,
+      "num_tokens": 2168354.0,
+      "step": 110
+    },
+    {
+      "entropy": 3.711766928434372,
+      "epoch": 0.192,
+      "grad_norm": 0.0169677734375,
+      "learning_rate": 2.38e-06,
+      "loss": 5.091,
+      "mean_token_accuracy": 0.29016080107539893,
+      "num_tokens": 2365822.0,
+      "step": 120
+    },
+    {
+      "entropy": 3.755408400297165,
+      "epoch": 0.208,
+      "grad_norm": 0.01953125,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 5.1415,
+      "mean_token_accuracy": 0.28556298119947315,
+      "num_tokens": 2558762.0,
+      "step": 130
+    },
+    {
+      "entropy": 3.6979280173778535,
+      "epoch": 0.224,
+      "grad_norm": 0.013671875,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 5.0488,
+      "mean_token_accuracy": 0.2934382151812315,
+      "num_tokens": 2755347.0,
+      "step": 140
+    },
+    {
+      "entropy": 3.67200947701931,
+      "epoch": 0.24,
+      "grad_norm": 0.0220947265625,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 5.0267,
+      "mean_token_accuracy": 0.29687621584162116,
+      "num_tokens": 2947346.0,
+      "step": 150
+    },
+    {
+      "entropy": 3.7363524213433266,
+      "epoch": 0.256,
+      "grad_norm": 0.0224609375,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 5.1623,
+      "mean_token_accuracy": 0.28543900661170485,
+      "num_tokens": 3139957.0,
+      "step": 160
+    },
+    {
+      "entropy": 3.737903955578804,
+      "epoch": 0.272,
+      "grad_norm": 0.01708984375,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 5.0962,
+      "mean_token_accuracy": 0.2901335723698139,
+      "num_tokens": 3335951.0,
+      "step": 170
+    },
+    {
+      "entropy": 3.7267358005046844,
+      "epoch": 0.288,
+      "grad_norm": 0.014404296875,
+      "learning_rate": 3.58e-06,
+      "loss": 5.1225,
+      "mean_token_accuracy": 0.2877850123681128,
+      "num_tokens": 3539731.0,
+      "step": 180
+    },
+    {
+      "entropy": 3.6973107740283013,
+      "epoch": 0.304,
+      "grad_norm": 0.0218505859375,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 5.0415,
+      "mean_token_accuracy": 0.291966971475631,
+      "num_tokens": 3733488.0,
+      "step": 190
+    },
+    {
+      "entropy": 3.723296643793583,
+      "epoch": 0.32,
+      "grad_norm": 0.01458740234375,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 5.1057,
+      "mean_token_accuracy": 0.2924021276645362,
+      "num_tokens": 3920545.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 3920545.0,
+      "eval_biology_runtime": 47.9931,
+      "eval_biology_samples_per_second": 10.418,
+      "eval_biology_steps_per_second": 2.605,
+      "step": 200
+    },
+    {
+      "entropy": 3.7122732952237127,
+      "epoch": 0.336,
+      "grad_norm": 0.0234375,
+      "learning_rate": 4.18e-06,
+      "loss": 5.0914,
+      "mean_token_accuracy": 0.29149032663553953,
+      "num_tokens": 4114077.0,
+      "step": 210
+    },
+    {
+      "entropy": 3.7262283325195313,
+      "epoch": 0.352,
+      "grad_norm": 0.0185546875,
+      "learning_rate": 4.38e-06,
+      "loss": 5.0925,
+      "mean_token_accuracy": 0.2936550347134471,
+      "num_tokens": 4306949.0,
+      "step": 220
+    },
+    {
+      "entropy": 3.723622639477253,
+      "epoch": 0.368,
+      "grad_norm": 0.017822265625,
+      "learning_rate": 4.58e-06,
+      "loss": 5.0811,
+      "mean_token_accuracy": 0.2895682736299932,
+      "num_tokens": 4504001.0,
+      "step": 230
+    },
+    {
+      "entropy": 3.7035663962364196,
+      "epoch": 0.384,
+      "grad_norm": 0.0247802734375,
+      "learning_rate": 4.78e-06,
+      "loss": 5.0597,
+      "mean_token_accuracy": 0.2904216184280813,
+      "num_tokens": 4693812.0,
+      "step": 240
+    },
+    {
+      "entropy": 3.7003923773765566,
+      "epoch": 0.4,
+      "grad_norm": 0.01434326171875,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 5.0458,
+      "mean_token_accuracy": 0.2927747514098883,
+      "num_tokens": 4887094.0,
+      "step": 250
+    },
+    {
+      "entropy": 3.716284817457199,
+      "epoch": 0.416,
+      "grad_norm": 0.016357421875,
+      "learning_rate": 5.18e-06,
+      "loss": 5.0684,
+      "mean_token_accuracy": 0.29001498911529777,
+      "num_tokens": 5085369.0,
+      "step": 260
+    },
+    {
+      "entropy": 3.74227339476347,
+      "epoch": 0.432,
+      "grad_norm": 0.0203857421875,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 5.1343,
+      "mean_token_accuracy": 0.2884894346818328,
+      "num_tokens": 5271275.0,
+      "step": 270
+    },
+    {
+      "entropy": 3.726644665002823,
+      "epoch": 0.448,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 5.1273,
+      "mean_token_accuracy": 0.2874998924322426,
+      "num_tokens": 5460559.0,
+      "step": 280
+    },
+    {
+      "entropy": 3.6816380873322485,
+      "epoch": 0.464,
+      "grad_norm": 0.0224609375,
+      "learning_rate": 5.78e-06,
+      "loss": 5.0719,
+      "mean_token_accuracy": 0.29172550728544594,
+      "num_tokens": 5653809.0,
+      "step": 290
+    },
+    {
+      "entropy": 3.7142979234457014,
+      "epoch": 0.48,
+      "grad_norm": 0.015625,
+      "learning_rate": 5.98e-06,
+      "loss": 5.0955,
+      "mean_token_accuracy": 0.28865716215223075,
+      "num_tokens": 5850176.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 5850176.0,
+      "eval_biology_runtime": 47.7142,
+      "eval_biology_samples_per_second": 10.479,
+      "eval_biology_steps_per_second": 2.62,
+      "step": 300
+    },
+    {
+      "entropy": 3.6794839769601824,
+      "epoch": 0.496,
+      "grad_norm": 0.0181884765625,
+      "learning_rate": 6.18e-06,
+      "loss": 5.0187,
+      "mean_token_accuracy": 0.29856595005840064,
+      "num_tokens": 6046503.0,
+      "step": 310
+    },
+    {
+      "entropy": 3.7517301350831986,
+      "epoch": 0.512,
+      "grad_norm": 0.0245361328125,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 5.1625,
+      "mean_token_accuracy": 0.2865315512754023,
+      "num_tokens": 6240456.0,
+      "step": 320
+    },
+    {
+      "entropy": 3.6911791786551476,
+      "epoch": 0.528,
+      "grad_norm": 0.016845703125,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 5.0801,
+      "mean_token_accuracy": 0.2921980186365545,
+      "num_tokens": 6430555.0,
+      "step": 330
+    },
+    {
+      "entropy": 3.7338990345597267,
+      "epoch": 0.544,
+      "grad_norm": 0.0262451171875,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 5.1089,
+      "mean_token_accuracy": 0.29122597351670265,
+      "num_tokens": 6626006.0,
+      "step": 340
+    },
+    {
+      "entropy": 3.714175473153591,
+      "epoch": 0.56,
+      "grad_norm": 0.0194091796875,
+      "learning_rate": 6.98e-06,
+      "loss": 5.0795,
+      "mean_token_accuracy": 0.28960168017074467,
+      "num_tokens": 6820754.0,
+      "step": 350
+    },
+    {
+      "entropy": 3.692426808178425,
+      "epoch": 0.576,
+      "grad_norm": 0.0191650390625,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 5.0757,
+      "mean_token_accuracy": 0.29286856260150673,
+      "num_tokens": 7021844.0,
+      "step": 360
+    },
+    {
+      "entropy": 3.727046549320221,
+      "epoch": 0.592,
+      "grad_norm": 0.0213623046875,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 5.1248,
+      "mean_token_accuracy": 0.28685210049152376,
+      "num_tokens": 7213951.0,
+      "step": 370
+    },
+    {
+      "entropy": 3.736472050845623,
+      "epoch": 0.608,
+      "grad_norm": 0.0177001953125,
+      "learning_rate": 7.58e-06,
+      "loss": 5.1462,
+      "mean_token_accuracy": 0.28853774573653934,
+      "num_tokens": 7416773.0,
+      "step": 380
+    },
+    {
+      "entropy": 3.74992755651474,
+      "epoch": 0.624,
+      "grad_norm": 0.01708984375,
+      "learning_rate": 7.78e-06,
+      "loss": 5.1298,
+      "mean_token_accuracy": 0.2880241921171546,
+      "num_tokens": 7612843.0,
+      "step": 390
+    },
+    {
+      "entropy": 3.7259780153632165,
+      "epoch": 0.64,
+      "grad_norm": 0.0208740234375,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 5.0975,
+      "mean_token_accuracy": 0.29221961721777917,
+      "num_tokens": 7801633.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 7801633.0,
+      "eval_biology_runtime": 47.8899,
+      "eval_biology_samples_per_second": 10.441,
+      "eval_biology_steps_per_second": 2.61,
+      "step": 400
+    },
+    {
+      "entropy": 3.76672201231122,
+      "epoch": 0.656,
+      "grad_norm": 0.013916015625,
+      "learning_rate": 8.18e-06,
+      "loss": 5.1778,
+      "mean_token_accuracy": 0.28632718725129963,
+      "num_tokens": 7995843.0,
+      "step": 410
+    },
+    {
+      "entropy": 3.7688263028860094,
+      "epoch": 0.672,
+      "grad_norm": 0.026611328125,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 5.1731,
+      "mean_token_accuracy": 0.2852769985795021,
+      "num_tokens": 8183103.0,
+      "step": 420
+    },
+    {
+      "entropy": 3.7213131189346313,
+      "epoch": 0.688,
+      "grad_norm": 0.01708984375,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 5.0812,
+      "mean_token_accuracy": 0.29185254173353314,
+      "num_tokens": 8385976.0,
+      "step": 430
+    },
+    {
+      "entropy": 3.7499947547912598,
+      "epoch": 0.704,
+      "grad_norm": 0.015380859375,
+      "learning_rate": 8.78e-06,
+      "loss": 5.1107,
+      "mean_token_accuracy": 0.28764023520052434,
+      "num_tokens": 8578431.0,
+      "step": 440
+    },
+    {
+      "entropy": 3.6844282656908036,
+      "epoch": 0.72,
+      "grad_norm": 0.017822265625,
+      "learning_rate": 8.98e-06,
+      "loss": 5.0192,
+      "mean_token_accuracy": 0.2957433703355491,
+      "num_tokens": 8781342.0,
+      "step": 450
+    },
+    {
+      "entropy": 3.7240318074822425,
+      "epoch": 0.736,
+      "grad_norm": 0.01953125,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 5.0727,
+      "mean_token_accuracy": 0.2915121610276401,
+      "num_tokens": 8977918.0,
+      "step": 460
+    },
+    {
+      "entropy": 3.723066192865372,
+      "epoch": 0.752,
+      "grad_norm": 0.016357421875,
+      "learning_rate": 9.38e-06,
+      "loss": 5.0973,
+      "mean_token_accuracy": 0.29267845982685686,
+      "num_tokens": 9169322.0,
+      "step": 470
+    },
+    {
+      "entropy": 3.702293635904789,
+      "epoch": 0.768,
+      "grad_norm": 0.0167236328125,
+      "learning_rate": 9.58e-06,
+      "loss": 5.0474,
+      "mean_token_accuracy": 0.29300098568201066,
+      "num_tokens": 9368141.0,
+      "step": 480
+    },
+    {
+      "entropy": 3.722616221010685,
+      "epoch": 0.784,
+      "grad_norm": 0.01361083984375,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 5.1082,
+      "mean_token_accuracy": 0.29106166921556,
+      "num_tokens": 9565236.0,
+      "step": 490
+    },
+    {
+      "entropy": 3.6929100595414637,
+      "epoch": 0.8,
+      "grad_norm": 0.0198974609375,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 5.055,
+      "mean_token_accuracy": 0.2922526111826301,
+      "num_tokens": 9761227.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 9761227.0,
+      "eval_biology_runtime": 47.6484,
+      "eval_biology_samples_per_second": 10.494,
+      "eval_biology_steps_per_second": 2.623,
+      "step": 500
+    },
+    {
+      "entropy": 3.7020211443305016,
+      "epoch": 0.816,
+      "grad_norm": 0.01513671875,
+      "learning_rate": 1.018e-05,
+      "loss": 5.0565,
+      "mean_token_accuracy": 0.29223762024194,
+      "num_tokens": 9958727.0,
+      "step": 510
+    },
+    {
+      "entropy": 3.690947251021862,
+      "epoch": 0.832,
+      "grad_norm": 0.0225830078125,
+      "learning_rate": 1.038e-05,
+      "loss": 5.0388,
+      "mean_token_accuracy": 0.2948582774028182,
+      "num_tokens": 10155771.0,
+      "step": 520
+    },
+    {
+      "entropy": 3.688454346358776,
+      "epoch": 0.848,
+      "grad_norm": 0.0238037109375,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 5.0305,
+      "mean_token_accuracy": 0.29634183505550027,
+      "num_tokens": 10357721.0,
+      "step": 530
+    },
+    {
+      "entropy": 3.711380937695503,
+      "epoch": 0.864,
+      "grad_norm": 0.01171875,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 5.058,
+      "mean_token_accuracy": 0.293431665468961,
+      "num_tokens": 10552495.0,
+      "step": 540
+    },
+    {
+      "entropy": 3.7067453160882,
+      "epoch": 0.88,
+      "grad_norm": 0.01409912109375,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 5.0502,
+      "mean_token_accuracy": 0.2919998188503087,
+      "num_tokens": 10748749.0,
+      "step": 550
+    },
+    {
+      "entropy": 3.722704839706421,
+      "epoch": 0.896,
+      "grad_norm": 0.0196533203125,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 5.1088,
+      "mean_token_accuracy": 0.28890208089724184,
+      "num_tokens": 10943319.0,
+      "step": 560
+    },
+    {
+      "entropy": 3.7139005795121194,
+      "epoch": 0.912,
+      "grad_norm": 0.0216064453125,
+      "learning_rate": 1.138e-05,
+      "loss": 5.0686,
+      "mean_token_accuracy": 0.2921730065718293,
+      "num_tokens": 11136935.0,
+      "step": 570
+    },
+    {
+      "entropy": 3.752144718170166,
+      "epoch": 0.928,
+      "grad_norm": 0.0172119140625,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 5.0906,
+      "mean_token_accuracy": 0.29035252146422863,
+      "num_tokens": 11331098.0,
+      "step": 580
+    },
+    {
+      "entropy": 3.714654731750488,
+      "epoch": 0.944,
+      "grad_norm": 0.0162353515625,
+      "learning_rate": 1.178e-05,
+      "loss": 5.0467,
+      "mean_token_accuracy": 0.2949915431439877,
+      "num_tokens": 11530550.0,
+      "step": 590
+    },
+    {
+      "entropy": 3.7143523931503295,
+      "epoch": 0.96,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 1.198e-05,
+      "loss": 5.0766,
+      "mean_token_accuracy": 0.29139310084283354,
+      "num_tokens": 11729645.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 11729645.0,
+      "eval_biology_runtime": 48.0459,
+      "eval_biology_samples_per_second": 10.407,
+      "eval_biology_steps_per_second": 2.602,
+      "step": 600
+    },
+    {
+      "entropy": 3.6984829902648926,
+      "epoch": 0.976,
+      "grad_norm": 0.014892578125,
+      "learning_rate": 1.218e-05,
+      "loss": 5.0862,
+      "mean_token_accuracy": 0.2942219194956124,
+      "num_tokens": 11924644.0,
+      "step": 610
+    },
+    {
+      "entropy": 3.6905887737870215,
+      "epoch": 0.992,
+      "grad_norm": 0.01507568359375,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 5.0574,
+      "mean_token_accuracy": 0.29360062861815095,
+      "num_tokens": 12123059.0,
+      "step": 620
+    },
+    {
+      "entropy": 3.7239044919610023,
+      "epoch": 1.008,
+      "grad_norm": 0.0157470703125,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 5.1152,
+      "mean_token_accuracy": 0.29114634413272145,
+      "num_tokens": 12319366.0,
+      "step": 630
+    },
+    {
+      "entropy": 3.6866162970662115,
+      "epoch": 1.024,
+      "grad_norm": 0.01470947265625,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 5.0343,
+      "mean_token_accuracy": 0.2939268404617906,
+      "num_tokens": 12524183.0,
+      "step": 640
+    },
+    {
+      "entropy": 3.72215301245451,
+      "epoch": 1.04,
+      "grad_norm": 0.0155029296875,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 5.1367,
+      "mean_token_accuracy": 0.28700590375810864,
+      "num_tokens": 12718593.0,
+      "step": 650
+    },
+    {
+      "entropy": 3.681245695054531,
+      "epoch": 1.056,
+      "grad_norm": 0.0191650390625,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 5.014,
+      "mean_token_accuracy": 0.2961495415307581,
+      "num_tokens": 12917803.0,
+      "step": 660
+    },
+    {
+      "entropy": 3.7183008253574372,
+      "epoch": 1.072,
+      "grad_norm": 0.02197265625,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 5.0868,
+      "mean_token_accuracy": 0.2896940400823951,
+      "num_tokens": 13105826.0,
+      "step": 670
+    },
+    {
+      "entropy": 3.742721700668335,
+      "epoch": 1.088,
+      "grad_norm": 0.016845703125,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 5.1107,
+      "mean_token_accuracy": 0.2900996576994658,
+      "num_tokens": 13298619.0,
+      "step": 680
+    },
+    {
+      "entropy": 3.7236127987504006,
+      "epoch": 1.104,
+      "grad_norm": 0.0150146484375,
+      "learning_rate": 1.378e-05,
+      "loss": 5.1279,
+      "mean_token_accuracy": 0.28985656797885895,
+      "num_tokens": 13491486.0,
+      "step": 690
+    },
+    {
+      "entropy": 3.7264697343111037,
+      "epoch": 1.12,
+      "grad_norm": 0.0186767578125,
+      "learning_rate": 1.398e-05,
+      "loss": 5.1227,
+      "mean_token_accuracy": 0.2900970630347729,
+      "num_tokens": 13674663.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 13674663.0,
+      "eval_biology_runtime": 47.6988,
+      "eval_biology_samples_per_second": 10.482,
+      "eval_biology_steps_per_second": 2.621,
+      "step": 700
+    },
+    {
+      "entropy": 3.7335189118981362,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 0.0230712890625,
+      "learning_rate": 1.418e-05,
+      "loss": 5.0857,
+      "mean_token_accuracy": 0.29160243617370724,
+      "num_tokens": 13869134.0,
+      "step": 710
+    },
+    {
+      "entropy": 3.6999334722757338,
+      "epoch": 1.152,
+      "grad_norm": 0.0179443359375,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 5.0276,
+      "mean_token_accuracy": 0.29345897743478416,
+      "num_tokens": 14078365.0,
+      "step": 720
+    },
+    {
+      "entropy": 3.7237380519509315,
+      "epoch": 1.168,
+      "grad_norm": 0.02734375,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 5.1027,
+      "mean_token_accuracy": 0.2900115196593106,
+      "num_tokens": 14266831.0,
+      "step": 730
+    },
+    {
+      "entropy": 3.728756721317768,
+      "epoch": 1.184,
+      "grad_norm": 0.028076171875,
+      "learning_rate": 1.478e-05,
+      "loss": 5.1063,
+      "mean_token_accuracy": 0.29116632882505655,
+      "num_tokens": 14465660.0,
+      "step": 740
+    },
+    {
+      "entropy": 3.694381882250309,
+      "epoch": 1.2,
+      "grad_norm": 0.0185546875,
+      "learning_rate": 1.498e-05,
+      "loss": 5.069,
+      "mean_token_accuracy": 0.2926430401392281,
+      "num_tokens": 14653228.0,
+      "step": 750
+    },
+    {
+      "entropy": 3.743997333943844,
+      "epoch": 1.216,
+      "grad_norm": 0.0235595703125,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 5.1078,
+      "mean_token_accuracy": 0.2893166967667639,
+      "num_tokens": 14857782.0,
+      "step": 760
+    },
+    {
+      "entropy": 3.6960197538137436,
+      "epoch": 1.232,
+      "grad_norm": 0.01220703125,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 5.062,
+      "mean_token_accuracy": 0.2938588274642825,
+      "num_tokens": 15047356.0,
+      "step": 770
+    },
+    {
+      "entropy": 3.700982950627804,
+      "epoch": 1.248,
+      "grad_norm": 0.0201416015625,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 5.0837,
+      "mean_token_accuracy": 0.29082163264974953,
+      "num_tokens": 15241098.0,
+      "step": 780
+    },
+    {
+      "entropy": 3.7153700664639473,
+      "epoch": 1.264,
+      "grad_norm": 0.017822265625,
+      "learning_rate": 1.578e-05,
+      "loss": 5.0565,
+      "mean_token_accuracy": 0.2915728730149567,
+      "num_tokens": 15437657.0,
+      "step": 790
+    },
+    {
+      "entropy": 3.7014693170785904,
+      "epoch": 1.28,
+      "grad_norm": 0.0245361328125,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 5.0562,
+      "mean_token_accuracy": 0.2920363646000624,
+      "num_tokens": 15630795.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 15630795.0,
+      "eval_biology_runtime": 47.6391,
+      "eval_biology_samples_per_second": 10.496,
+      "eval_biology_steps_per_second": 2.624,
+      "step": 800
+    },
+    {
+      "entropy": 3.7237909018993376,
+      "epoch": 1.296,
+      "grad_norm": 0.0181884765625,
+      "learning_rate": 1.618e-05,
+      "loss": 5.0864,
+      "mean_token_accuracy": 0.29013621537014844,
+      "num_tokens": 15827105.0,
+      "step": 810
+    },
+    {
+      "entropy": 3.6957172751426697,
+      "epoch": 1.312,
+      "grad_norm": 0.026123046875,
+      "learning_rate": 1.638e-05,
+      "loss": 5.0904,
+      "mean_token_accuracy": 0.2900472991168499,
+      "num_tokens": 16019645.0,
+      "step": 820
+    },
+    {
+      "entropy": 3.728809031844139,
+      "epoch": 1.328,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 1.658e-05,
+      "loss": 5.1059,
+      "mean_token_accuracy": 0.2894112995825708,
+      "num_tokens": 16221726.0,
+      "step": 830
+    },
+    {
+      "entropy": 3.7269085705280305,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 0.0174560546875,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 5.0921,
+      "mean_token_accuracy": 0.2916096639819443,
+      "num_tokens": 16427594.0,
+      "step": 840
+    },
+    {
+      "entropy": 3.727548821270466,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 0.022216796875,
+      "learning_rate": 1.698e-05,
+      "loss": 5.0558,
+      "mean_token_accuracy": 0.2904492556117475,
+      "num_tokens": 16621605.0,
+      "step": 850
+    },
+    {
+      "entropy": 3.7071838200092317,
+      "epoch": 1.376,
+      "grad_norm": 0.0272216796875,
+      "learning_rate": 1.718e-05,
+      "loss": 5.0775,
+      "mean_token_accuracy": 0.29065996296703817,
+      "num_tokens": 16813444.0,
+      "step": 860
+    },
+    {
+      "entropy": 3.7128281995654104,
+      "epoch": 1.392,
+      "grad_norm": 0.021484375,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 5.1021,
+      "mean_token_accuracy": 0.2892353082075715,
+      "num_tokens": 17006509.0,
+      "step": 870
+    },
+    {
+      "entropy": 3.700386196374893,
+      "epoch": 1.408,
+      "grad_norm": 0.0233154296875,
+      "learning_rate": 1.758e-05,
+      "loss": 5.0966,
+      "mean_token_accuracy": 0.29179619075730445,
+      "num_tokens": 17197870.0,
+      "step": 880
+    },
+    {
+      "entropy": 3.6790005937218666,
+      "epoch": 1.424,
+      "grad_norm": 0.0185546875,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 5.0503,
+      "mean_token_accuracy": 0.294497752469033,
+      "num_tokens": 17394390.0,
+      "step": 890
+    },
+    {
+      "entropy": 3.714966467022896,
+      "epoch": 1.44,
+      "grad_norm": 0.0220947265625,
+      "learning_rate": 1.798e-05,
+      "loss": 5.0895,
+      "mean_token_accuracy": 0.2897561041638255,
+      "num_tokens": 17587777.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 17587777.0,
+      "eval_biology_runtime": 47.5786,
+      "eval_biology_samples_per_second": 10.509,
+      "eval_biology_steps_per_second": 2.627,
+      "step": 900
+    },
+    {
+      "entropy": 3.718673199415207,
+      "epoch": 1.456,
+      "grad_norm": 0.0169677734375,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 5.0906,
+      "mean_token_accuracy": 0.29184630075469614,
+      "num_tokens": 17788456.0,
+      "step": 910
+    },
+    {
+      "entropy": 3.714043965935707,
+      "epoch": 1.472,
+      "grad_norm": 0.0184326171875,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 5.1039,
+      "mean_token_accuracy": 0.2923115941695869,
+      "num_tokens": 17984063.0,
+      "step": 920
+    },
+    {
+      "entropy": 3.7208774775266646,
+      "epoch": 1.488,
+      "grad_norm": 0.01226806640625,
+      "learning_rate": 1.858e-05,
+      "loss": 5.1261,
+      "mean_token_accuracy": 0.2891128743067384,
+      "num_tokens": 18175640.0,
+      "step": 930
+    },
+    {
+      "entropy": 3.7392901137471197,
+      "epoch": 1.504,
+      "grad_norm": 0.027587890625,
+      "learning_rate": 1.878e-05,
+      "loss": 5.1405,
+      "mean_token_accuracy": 0.28787331804633143,
+      "num_tokens": 18367857.0,
+      "step": 940
+    },
+    {
+      "entropy": 3.7357000544667245,
+      "epoch": 1.52,
+      "grad_norm": 0.0235595703125,
+      "learning_rate": 1.898e-05,
+      "loss": 5.0929,
+      "mean_token_accuracy": 0.2901941016316414,
+      "num_tokens": 18569146.0,
+      "step": 950
+    },
+    {
+      "entropy": 3.71133918017149,
+      "epoch": 1.536,
+      "grad_norm": 0.01953125,
+      "learning_rate": 1.918e-05,
+      "loss": 5.0759,
+      "mean_token_accuracy": 0.2910002091899514,
+      "num_tokens": 18755079.0,
+      "step": 960
+    },
+    {
+      "entropy": 3.679325211048126,
+      "epoch": 1.552,
+      "grad_norm": 0.0179443359375,
+      "learning_rate": 1.938e-05,
+      "loss": 5.0271,
+      "mean_token_accuracy": 0.29568239795044066,
+      "num_tokens": 18956248.0,
+      "step": 970
+    },
+    {
+      "entropy": 3.7200872167944907,
+      "epoch": 1.568,
+      "grad_norm": 0.0194091796875,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 5.0739,
+      "mean_token_accuracy": 0.290937721170485,
+      "num_tokens": 19150315.0,
+      "step": 980
+    },
+    {
+      "entropy": 3.6731236964464187,
+      "epoch": 1.584,
+      "grad_norm": 0.0179443359375,
+      "learning_rate": 1.978e-05,
+      "loss": 4.9996,
+      "mean_token_accuracy": 0.2993267074227333,
+      "num_tokens": 19344260.0,
+      "step": 990
+    },
+    {
+      "entropy": 3.7825915426015855,
+      "epoch": 1.6,
+      "grad_norm": 0.0172119140625,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 5.2041,
+      "mean_token_accuracy": 0.2822197400033474,
+      "num_tokens": 19532552.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 19532552.0,
+      "eval_biology_runtime": 47.6646,
+      "eval_biology_samples_per_second": 10.49,
+      "eval_biology_steps_per_second": 2.622,
+      "step": 1000
+    },
+    {
+      "entropy": 3.7059972777962686,
+      "epoch": 1.616,
+      "grad_norm": 0.0159912109375,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 5.0574,
+      "mean_token_accuracy": 0.29472711179405453,
+      "num_tokens": 19732719.0,
+      "step": 1010
+    },
+    {
+      "entropy": 3.736002130806446,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 0.01556396484375,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 5.1263,
+      "mean_token_accuracy": 0.28895843513309954,
+      "num_tokens": 19926830.0,
+      "step": 1020
+    },
+    {
+      "entropy": 3.726553238928318,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 0.016845703125,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 5.1039,
+      "mean_token_accuracy": 0.2898056315258145,
+      "num_tokens": 20118800.0,
+      "step": 1030
+    },
+    {
+      "entropy": 3.7194370150566103,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 0.017578125,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 5.0616,
+      "mean_token_accuracy": 0.29428425850346684,
+      "num_tokens": 20320511.0,
+      "step": 1040
+    },
+    {
+      "entropy": 3.7065395385026934,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 0.01397705078125,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 5.0716,
+      "mean_token_accuracy": 0.29314093850553036,
+      "num_tokens": 20513393.0,
+      "step": 1050
+    },
+    {
+      "entropy": 3.7188143908977507,
+      "epoch": 1.696,
+      "grad_norm": 0.022705078125,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 5.0778,
+      "mean_token_accuracy": 0.29205265222117305,
+      "num_tokens": 20707237.0,
+      "step": 1060
+    },
+    {
+      "entropy": 3.7139120802283285,
+      "epoch": 1.712,
+      "grad_norm": 0.025146484375,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 5.078,
+      "mean_token_accuracy": 0.2915316676720977,
+      "num_tokens": 20910419.0,
+      "step": 1070
+    },
+    {
+      "entropy": 3.723431368172169,
+      "epoch": 1.728,
+      "grad_norm": 0.0177001953125,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 5.0998,
+      "mean_token_accuracy": 0.2917724488303065,
+      "num_tokens": 21107498.0,
+      "step": 1080
+    },
+    {
+      "entropy": 3.729191516339779,
+      "epoch": 1.744,
+      "grad_norm": 0.01373291015625,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 5.0911,
+      "mean_token_accuracy": 0.2921753362752497,
+      "num_tokens": 21303955.0,
+      "step": 1090
+    },
+    {
+      "entropy": 3.713176865875721,
+      "epoch": 1.76,
+      "grad_norm": 0.023681640625,
+      "learning_rate": 1.978e-05,
+      "loss": 5.1136,
+      "mean_token_accuracy": 0.2927762917242944,
+      "num_tokens": 21499572.0,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 21499572.0,
+      "eval_biology_runtime": 47.7303,
+      "eval_biology_samples_per_second": 10.476,
+      "eval_biology_steps_per_second": 2.619,
+      "step": 1100
+    },
+    {
+      "entropy": 3.683187483251095,
+      "epoch": 1.776,
+      "grad_norm": 0.01165771484375,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 5.0493,
+      "mean_token_accuracy": 0.2950046713463962,
+      "num_tokens": 21692804.0,
+      "step": 1110
+    },
+    {
+      "entropy": 3.723648378252983,
+      "epoch": 1.792,
+      "grad_norm": 0.0172119140625,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 5.0692,
+      "mean_token_accuracy": 0.29330655848607423,
+      "num_tokens": 21894218.0,
+      "step": 1120
+    },
+    {
+      "entropy": 3.680108018219471,
+      "epoch": 1.808,
+      "grad_norm": 0.02197265625,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 5.0566,
+      "mean_token_accuracy": 0.29332200549542903,
+      "num_tokens": 22082522.0,
+      "step": 1130
+    },
+    {
+      "entropy": 3.7113044396042825,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 0.0283203125,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 5.0721,
+      "mean_token_accuracy": 0.29031074047088623,
+      "num_tokens": 22278933.0,
+      "step": 1140
+    },
+    {
+      "entropy": 3.761154365539551,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 0.0223388671875,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 5.1725,
+      "mean_token_accuracy": 0.28511819140985606,
+      "num_tokens": 22473801.0,
+      "step": 1150
+    },
+    {
+      "entropy": 3.734527000784874,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 0.0224609375,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 5.126,
+      "mean_token_accuracy": 0.28791883746162056,
+      "num_tokens": 22677853.0,
+      "step": 1160
+    },
+    {
+      "entropy": 3.7064869537949563,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 0.031494140625,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 5.0564,
+      "mean_token_accuracy": 0.29391895309090615,
+      "num_tokens": 22874965.0,
+      "step": 1170
+    },
+    {
+      "entropy": 3.7308966740965843,
+      "epoch": 1.888,
+      "grad_norm": 0.013671875,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 5.1087,
+      "mean_token_accuracy": 0.28873477103188633,
+      "num_tokens": 23068892.0,
+      "step": 1180
+    },
+    {
+      "entropy": 3.7291680380702017,
+      "epoch": 1.904,
+      "grad_norm": 0.020263671875,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 5.1221,
+      "mean_token_accuracy": 0.28796409014612434,
+      "num_tokens": 23263827.0,
+      "step": 1190
+    },
+    {
+      "entropy": 3.6863365471363068,
+      "epoch": 1.92,
+      "grad_norm": 0.02490234375,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 5.0534,
+      "mean_token_accuracy": 0.29244121424853803,
+      "num_tokens": 23463627.0,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 23463627.0,
+      "eval_biology_runtime": 47.71,
+      "eval_biology_samples_per_second": 10.48,
+      "eval_biology_steps_per_second": 2.62,
+      "step": 1200
+    },
+    {
+      "entropy": 3.6975948810577393,
+      "epoch": 1.936,
+      "grad_norm": 0.0177001953125,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 5.0404,
+      "mean_token_accuracy": 0.29399057319387795,
+      "num_tokens": 23660418.0,
+      "step": 1210
+    },
+    {
+      "entropy": 3.724119584262371,
+      "epoch": 1.952,
+      "grad_norm": 0.01214599609375,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 5.0911,
+      "mean_token_accuracy": 0.2915867816656828,
+      "num_tokens": 23858145.0,
+      "step": 1220
+    },
+    {
+      "entropy": 3.7461824998259545,
+      "epoch": 1.968,
+      "grad_norm": 0.0155029296875,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 5.1581,
+      "mean_token_accuracy": 0.28603497641161085,
+      "num_tokens": 24053364.0,
+      "step": 1230
+    },
+    {
+      "entropy": 3.7114603489637377,
+      "epoch": 1.984,
+      "grad_norm": 0.0157470703125,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 5.0847,
+      "mean_token_accuracy": 0.2926376985386014,
+      "num_tokens": 24249465.0,
+      "step": 1240
+    },
+    {
+      "entropy": 3.713132253289223,
+      "epoch": 2.0,
+      "grad_norm": 0.021728515625,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 5.0979,
+      "mean_token_accuracy": 0.29202904291450976,
+      "num_tokens": 24442582.0,
+      "step": 1250
+    },
+    {
+      "entropy": 3.7105937510728837,
+      "epoch": 2.016,
+      "grad_norm": 0.020751953125,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 5.1146,
+      "mean_token_accuracy": 0.2896880513057113,
+      "num_tokens": 24632353.0,
+      "step": 1260
+    },
+    {
+      "entropy": 3.7421246364712717,
+      "epoch": 2.032,
+      "grad_norm": 0.0234375,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 5.156,
+      "mean_token_accuracy": 0.2862941346131265,
+      "num_tokens": 24822715.0,
+      "step": 1270
+    },
+    {
+      "entropy": 3.729320552945137,
+      "epoch": 2.048,
+      "grad_norm": 0.011962890625,
+      "learning_rate": 1.938e-05,
+      "loss": 5.0711,
+      "mean_token_accuracy": 0.2908482993021607,
+      "num_tokens": 25023118.0,
+      "step": 1280
+    },
+    {
+      "entropy": 3.716141538321972,
+      "epoch": 2.064,
+      "grad_norm": 0.0189208984375,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 5.1159,
+      "mean_token_accuracy": 0.28827573377639054,
+      "num_tokens": 25217414.0,
+      "step": 1290
+    },
+    {
+      "entropy": 3.7167277559638023,
+      "epoch": 2.08,
+      "grad_norm": 0.020263671875,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 5.091,
+      "mean_token_accuracy": 0.2915657116100192,
+      "num_tokens": 25408961.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 25408961.0,
+      "eval_biology_runtime": 47.587,
+      "eval_biology_samples_per_second": 10.507,
+      "eval_biology_steps_per_second": 2.627,
+      "step": 1300
+    },
+    {
+      "entropy": 3.714424678683281,
+      "epoch": 2.096,
+      "grad_norm": 0.0184326171875,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 5.0943,
+      "mean_token_accuracy": 0.29086006721481683,
+      "num_tokens": 25600511.0,
+      "step": 1310
+    },
+    {
+      "entropy": 3.743735647201538,
+      "epoch": 2.112,
+      "grad_norm": 0.01531982421875,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 5.107,
+      "mean_token_accuracy": 0.2900062766857445,
+      "num_tokens": 25796565.0,
+      "step": 1320
+    },
+    {
+      "entropy": 3.711001531779766,
+      "epoch": 2.128,
+      "grad_norm": 0.016845703125,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 5.0988,
+      "mean_token_accuracy": 0.2924064252525568,
+      "num_tokens": 25991156.0,
+      "step": 1330
+    },
+    {
+      "entropy": 3.698951078951359,
+      "epoch": 2.144,
+      "grad_norm": 0.01611328125,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 5.063,
+      "mean_token_accuracy": 0.29314082926139234,
+      "num_tokens": 26193237.0,
+      "step": 1340
+    },
+    {
+      "entropy": 3.7248923525214197,
+      "epoch": 2.16,
+      "grad_norm": 0.0264892578125,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 5.0923,
+      "mean_token_accuracy": 0.29320395886898043,
+      "num_tokens": 26387993.0,
+      "step": 1350
+    },
+    {
+      "entropy": 3.7421302527189253,
+      "epoch": 2.176,
+      "grad_norm": 0.014404296875,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 5.1362,
+      "mean_token_accuracy": 0.287891149520874,
+      "num_tokens": 26585269.0,
+      "step": 1360
+    },
+    {
+      "entropy": 3.7373417228460313,
+      "epoch": 2.192,
+      "grad_norm": 0.019775390625,
+      "learning_rate": 1.918e-05,
+      "loss": 5.099,
+      "mean_token_accuracy": 0.29109739176928995,
+      "num_tokens": 26776318.0,
+      "step": 1370
+    },
+    {
+      "entropy": 3.7420630186796187,
+      "epoch": 2.208,
+      "grad_norm": 0.02490234375,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 5.1231,
+      "mean_token_accuracy": 0.28790645729750397,
+      "num_tokens": 26974420.0,
+      "step": 1380
+    },
+    {
+      "entropy": 3.735623452067375,
+      "epoch": 2.224,
+      "grad_norm": 0.0157470703125,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 5.1347,
+      "mean_token_accuracy": 0.2884402872994542,
+      "num_tokens": 27167577.0,
+      "step": 1390
+    },
+    {
+      "entropy": 3.7288005337119103,
+      "epoch": 2.24,
+      "grad_norm": 0.01324462890625,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 5.0568,
+      "mean_token_accuracy": 0.2936669984832406,
+      "num_tokens": 27364189.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 27364189.0,
+      "eval_biology_runtime": 47.642,
+      "eval_biology_samples_per_second": 10.495,
+      "eval_biology_steps_per_second": 2.624,
+      "step": 1400
+    },
+    {
+      "entropy": 3.6691079929471018,
+      "epoch": 2.2560000000000002,
+      "grad_norm": 0.0191650390625,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 5.0664,
+      "mean_token_accuracy": 0.2951836409047246,
+      "num_tokens": 27557387.0,
+      "step": 1410
+    },
+    {
+      "entropy": 3.7611682564020157,
+      "epoch": 2.2720000000000002,
+      "grad_norm": 0.021728515625,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 5.152,
+      "mean_token_accuracy": 0.28709178091958165,
+      "num_tokens": 27755725.0,
+      "step": 1420
+    },
+    {
+      "entropy": 3.778307482600212,
+      "epoch": 2.288,
+      "grad_norm": 0.0189208984375,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 5.1678,
+      "mean_token_accuracy": 0.28473026901483534,
+      "num_tokens": 27950694.0,
+      "step": 1430
+    },
+    {
+      "entropy": 3.7009392485022543,
+      "epoch": 2.304,
+      "grad_norm": 0.016845703125,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 5.0617,
+      "mean_token_accuracy": 0.29399854615330695,
+      "num_tokens": 28150719.0,
+      "step": 1440
+    },
+    {
+      "entropy": 3.709285947680473,
+      "epoch": 2.32,
+      "grad_norm": 0.01611328125,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 5.0994,
+      "mean_token_accuracy": 0.29222611617296934,
+      "num_tokens": 28348652.0,
+      "step": 1450
+    },
+    {
+      "entropy": 3.718844693899155,
+      "epoch": 2.336,
+      "grad_norm": 0.0157470703125,
+      "learning_rate": 1.898e-05,
+      "loss": 5.09,
+      "mean_token_accuracy": 0.2933342828415334,
+      "num_tokens": 28542945.0,
+      "step": 1460
+    },
+    {
+      "entropy": 3.689019551873207,
+      "epoch": 2.352,
+      "grad_norm": 0.0216064453125,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 5.0641,
+      "mean_token_accuracy": 0.292084741499275,
+      "num_tokens": 28731927.0,
+      "step": 1470
+    },
+    {
+      "entropy": 3.686488929390907,
+      "epoch": 2.368,
+      "grad_norm": 0.01416015625,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 5.0355,
+      "mean_token_accuracy": 0.29083675481379034,
+      "num_tokens": 28929298.0,
+      "step": 1480
+    },
+    {
+      "entropy": 3.7178502827882767,
+      "epoch": 2.384,
+      "grad_norm": 0.0184326171875,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 5.1081,
+      "mean_token_accuracy": 0.29000213257968427,
+      "num_tokens": 29121142.0,
+      "step": 1490
+    },
+    {
+      "entropy": 3.709418612718582,
+      "epoch": 2.4,
+      "grad_norm": 0.013671875,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 5.0599,
+      "mean_token_accuracy": 0.29317397633567455,
+      "num_tokens": 29317919.0,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_biology_entropy": 3.665779479980469,
+      "eval_biology_loss": 5.049773216247559,
+      "eval_biology_mean_token_accuracy": 0.29773494935035705,
+      "eval_biology_num_tokens": 29317919.0,
+      "eval_biology_runtime": 47.6489,
+      "eval_biology_samples_per_second": 10.493,
+      "eval_biology_steps_per_second": 2.623,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 16,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.182272592862376e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aed5ae3f8efda159a1a46bcd744d079d1c322930322ae143f51aff11c274063
+size 6417