Upload folder using huggingface_hub

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-1000/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

checkpoint-1000/chat_template.jinja

@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

checkpoint-1000/config.json

@@ -0,0 +1,86 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3840,
+  "initializer_range": 0.02,
+  "intermediate_size": 15360,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": {
+    "factor": 8.0,
+    "rope_type": "linear"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 1024,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 262208
+}

checkpoint-1000/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 2,
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.56.1"
+}

checkpoint-1000/model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a4c14f38ffc9e23d873fa999eea1fc965dcedcec267e7a99f07c6fdbe8760d8
+size 4915892992

checkpoint-1000/model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0335ce336b31e38c581326acb77ff28330c2073764b1924ad7108d788b56ba68
+size 4931294472

checkpoint-1000/model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0d6a60cf6a1708be0613ab7c4d10457ad99b98c02e40ebd5f320fe02aa0d50c
+size 4931294528

checkpoint-1000/model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8b76dd5c3b1bb971354de059db455aac1c59fea624bf8a0289e37557ff43261
+size 4931294528

checkpoint-1000/model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d9c549d5bc84e50ecc0c62c8bf5edebd5036ac83bceabccf75b83689f58f370
+size 3822364808

checkpoint-1000/model.safetensors.index.json

@@ -0,0 +1,634 @@
+{
+  "metadata": {
+    "total_parameters": 11766034176,
+    "total_size": 23532068352
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.39.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.post_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.pre_feedforward_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.norm.weight": "model-00005-of-00005.safetensors"
+  }
+}

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3807720906b8ce0d6cdb5a1234d2eeebfa06b4ee459ad56f544f678293838513
+size 18355

checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acd74ad34e3c5060506b53a8a3fafbfb35d44065ee496b2abf3a0b9a3b93ba09
+size 14645

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6078e52773a379f026b8fbbbc5546ab7ed9418d05b713f8aff2a4f3c7e12f108
+size 1465

checkpoint-1000/special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

checkpoint-1000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

checkpoint-1000/trainer_state.json

@@ -0,0 +1,1254 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.8,
+  "eval_steps": 100,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.9169743306934834,
+      "epoch": 0.008,
+      "grad_norm": 0.10595703125,
+      "learning_rate": 1.8e-07,
+      "loss": 11.4093,
+      "mean_token_accuracy": 0.16113518364727497,
+      "num_tokens": 103172.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.943360574543476,
+      "epoch": 0.016,
+      "grad_norm": 0.11328125,
+      "learning_rate": 3.8e-07,
+      "loss": 11.3724,
+      "mean_token_accuracy": 0.15745336734689772,
+      "num_tokens": 195524.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.8853021122515201,
+      "epoch": 0.024,
+      "grad_norm": 0.11376953125,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 11.4536,
+      "mean_token_accuracy": 0.1584921860601753,
+      "num_tokens": 296999.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.9313011743128299,
+      "epoch": 0.032,
+      "grad_norm": 0.11962890625,
+      "learning_rate": 7.8e-07,
+      "loss": 11.4176,
+      "mean_token_accuracy": 0.15730738821439444,
+      "num_tokens": 390903.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.886887714266777,
+      "epoch": 0.04,
+      "grad_norm": 0.11083984375,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 11.362,
+      "mean_token_accuracy": 0.15966884773224593,
+      "num_tokens": 488481.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.8928033113479614,
+      "epoch": 0.048,
+      "grad_norm": 0.10791015625,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 11.4662,
+      "mean_token_accuracy": 0.1601271564140916,
+      "num_tokens": 589868.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.9126614801585675,
+      "epoch": 0.056,
+      "grad_norm": 0.11083984375,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 11.4393,
+      "mean_token_accuracy": 0.15678977984935044,
+      "num_tokens": 690754.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.8982849732041358,
+      "epoch": 0.064,
+      "grad_norm": 0.11279296875,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 11.428,
+      "mean_token_accuracy": 0.15833127587102352,
+      "num_tokens": 791190.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.9196412533521652,
+      "epoch": 0.072,
+      "grad_norm": 0.11181640625,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 11.3783,
+      "mean_token_accuracy": 0.1575094529427588,
+      "num_tokens": 890075.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.8977599568665027,
+      "epoch": 0.08,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 1.98e-06,
+      "loss": 11.3874,
+      "mean_token_accuracy": 0.16100947242230176,
+      "num_tokens": 989860.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 989860.0,
+      "eval_coding_runtime": 94.2996,
+      "eval_coding_samples_per_second": 5.302,
+      "eval_coding_steps_per_second": 2.651,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 989860.0,
+      "eval_biology_runtime": 41.3672,
+      "eval_biology_samples_per_second": 12.087,
+      "eval_biology_steps_per_second": 6.043,
+      "step": 100
+    },
+    {
+      "entropy": 1.9323325648903846,
+      "epoch": 0.088,
+      "grad_norm": 0.109375,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 11.43,
+      "mean_token_accuracy": 0.15889165913686157,
+      "num_tokens": 1087099.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.9131557896733284,
+      "epoch": 0.096,
+      "grad_norm": 0.1123046875,
+      "learning_rate": 2.38e-06,
+      "loss": 11.2832,
+      "mean_token_accuracy": 0.15686040292494,
+      "num_tokens": 1181777.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.8847123883664607,
+      "epoch": 0.104,
+      "grad_norm": 0.1181640625,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 11.5591,
+      "mean_token_accuracy": 0.15738505702465772,
+      "num_tokens": 1283881.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.871863854676485,
+      "epoch": 0.112,
+      "grad_norm": 0.10888671875,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 11.3822,
+      "mean_token_accuracy": 0.15778318694792687,
+      "num_tokens": 1385513.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.9166233658790588,
+      "epoch": 0.12,
+      "grad_norm": 0.115234375,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 11.3895,
+      "mean_token_accuracy": 0.15918795759789645,
+      "num_tokens": 1483667.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.902534269541502,
+      "epoch": 0.128,
+      "grad_norm": 0.10888671875,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 11.4745,
+      "mean_token_accuracy": 0.15784057211130859,
+      "num_tokens": 1582368.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.9250910609960556,
+      "epoch": 0.136,
+      "grad_norm": 0.1015625,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 11.3645,
+      "mean_token_accuracy": 0.15877335243858398,
+      "num_tokens": 1677060.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.888841236382723,
+      "epoch": 0.144,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 3.58e-06,
+      "loss": 11.3131,
+      "mean_token_accuracy": 0.15988174122758209,
+      "num_tokens": 1773764.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.923802337050438,
+      "epoch": 0.152,
+      "grad_norm": 0.10986328125,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 11.4321,
+      "mean_token_accuracy": 0.15674821990542115,
+      "num_tokens": 1873167.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.913605347275734,
+      "epoch": 0.16,
+      "grad_norm": 0.10693359375,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 11.5801,
+      "mean_token_accuracy": 0.1564559088088572,
+      "num_tokens": 1970077.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 1970077.0,
+      "eval_coding_runtime": 93.1599,
+      "eval_coding_samples_per_second": 5.367,
+      "eval_coding_steps_per_second": 2.684,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 1970077.0,
+      "eval_biology_runtime": 41.0689,
+      "eval_biology_samples_per_second": 12.175,
+      "eval_biology_steps_per_second": 6.087,
+      "step": 200
+    },
+    {
+      "entropy": 1.9281546145677566,
+      "epoch": 0.168,
+      "grad_norm": 0.11083984375,
+      "learning_rate": 4.18e-06,
+      "loss": 11.4462,
+      "mean_token_accuracy": 0.15564582953229547,
+      "num_tokens": 2066815.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.883149204403162,
+      "epoch": 0.176,
+      "grad_norm": 0.103515625,
+      "learning_rate": 4.38e-06,
+      "loss": 11.4588,
+      "mean_token_accuracy": 0.1579661637544632,
+      "num_tokens": 2168354.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.914576094597578,
+      "epoch": 0.184,
+      "grad_norm": 0.10400390625,
+      "learning_rate": 4.58e-06,
+      "loss": 11.3803,
+      "mean_token_accuracy": 0.15757576958276331,
+      "num_tokens": 2270617.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.9133810736238956,
+      "epoch": 0.192,
+      "grad_norm": 0.11083984375,
+      "learning_rate": 4.78e-06,
+      "loss": 11.5562,
+      "mean_token_accuracy": 0.15540659767575563,
+      "num_tokens": 2365822.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.9234349891543387,
+      "epoch": 0.2,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 11.5222,
+      "mean_token_accuracy": 0.15323229427449406,
+      "num_tokens": 2459876.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.9399859428405761,
+      "epoch": 0.208,
+      "grad_norm": 0.109375,
+      "learning_rate": 5.18e-06,
+      "loss": 11.428,
+      "mean_token_accuracy": 0.1552271238528192,
+      "num_tokens": 2558762.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.9029304042458535,
+      "epoch": 0.216,
+      "grad_norm": 0.10205078125,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 11.2925,
+      "mean_token_accuracy": 0.16210160069167615,
+      "num_tokens": 2654953.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.9189352929592132,
+      "epoch": 0.224,
+      "grad_norm": 0.09912109375,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 11.3849,
+      "mean_token_accuracy": 0.15769596104510128,
+      "num_tokens": 2755347.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.8914753369987012,
+      "epoch": 0.232,
+      "grad_norm": 0.1181640625,
+      "learning_rate": 5.78e-06,
+      "loss": 11.4031,
+      "mean_token_accuracy": 0.16200053002685308,
+      "num_tokens": 2852298.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.9092405900359153,
+      "epoch": 0.24,
+      "grad_norm": 0.10302734375,
+      "learning_rate": 5.98e-06,
+      "loss": 11.2327,
+      "mean_token_accuracy": 0.16136883310973643,
+      "num_tokens": 2947346.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 2947346.0,
+      "eval_coding_runtime": 93.3688,
+      "eval_coding_samples_per_second": 5.355,
+      "eval_coding_steps_per_second": 2.678,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 2947346.0,
+      "eval_biology_runtime": 41.7366,
+      "eval_biology_samples_per_second": 11.98,
+      "eval_biology_steps_per_second": 5.99,
+      "step": 300
+    },
+    {
+      "entropy": 1.9313343465328217,
+      "epoch": 0.248,
+      "grad_norm": 0.1123046875,
+      "learning_rate": 6.18e-06,
+      "loss": 11.5425,
+      "mean_token_accuracy": 0.15317737096920608,
+      "num_tokens": 3043128.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.9211706645786761,
+      "epoch": 0.256,
+      "grad_norm": 0.1083984375,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 11.4729,
+      "mean_token_accuracy": 0.1547949264757335,
+      "num_tokens": 3139957.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.923046988248825,
+      "epoch": 0.264,
+      "grad_norm": 0.10693359375,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 11.4159,
+      "mean_token_accuracy": 0.15573964624200015,
+      "num_tokens": 3232159.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.8901807874441148,
+      "epoch": 0.272,
+      "grad_norm": 0.1083984375,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 11.3543,
+      "mean_token_accuracy": 0.1617777702398598,
+      "num_tokens": 3335951.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.8801799044013023,
+      "epoch": 0.28,
+      "grad_norm": 0.10498046875,
+      "learning_rate": 6.98e-06,
+      "loss": 11.4879,
+      "mean_token_accuracy": 0.15775063186883925,
+      "num_tokens": 3437213.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.9043901436030866,
+      "epoch": 0.288,
+      "grad_norm": 0.11474609375,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 11.5366,
+      "mean_token_accuracy": 0.15751364254392683,
+      "num_tokens": 3539731.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.9588028222322464,
+      "epoch": 0.296,
+      "grad_norm": 0.1064453125,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 11.3894,
+      "mean_token_accuracy": 0.153670167690143,
+      "num_tokens": 3633448.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.881439681351185,
+      "epoch": 0.304,
+      "grad_norm": 0.1064453125,
+      "learning_rate": 7.58e-06,
+      "loss": 11.3016,
+      "mean_token_accuracy": 0.16050010859034955,
+      "num_tokens": 3733488.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.9542286798357964,
+      "epoch": 0.312,
+      "grad_norm": 0.103515625,
+      "learning_rate": 7.78e-06,
+      "loss": 11.273,
+      "mean_token_accuracy": 0.15929239662364125,
+      "num_tokens": 3825212.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.9287514887750148,
+      "epoch": 0.32,
+      "grad_norm": 0.10107421875,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 11.3054,
+      "mean_token_accuracy": 0.1604186396114528,
+      "num_tokens": 3920545.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 3920545.0,
+      "eval_coding_runtime": 92.8995,
+      "eval_coding_samples_per_second": 5.382,
+      "eval_coding_steps_per_second": 2.691,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 3920545.0,
+      "eval_biology_runtime": 41.088,
+      "eval_biology_samples_per_second": 12.169,
+      "eval_biology_steps_per_second": 6.085,
+      "step": 400
+    },
+    {
+      "entropy": 1.9058279685676098,
+      "epoch": 0.328,
+      "grad_norm": 0.11181640625,
+      "learning_rate": 8.18e-06,
+      "loss": 11.3842,
+      "mean_token_accuracy": 0.15871414174325765,
+      "num_tokens": 4016673.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.930207794904709,
+      "epoch": 0.336,
+      "grad_norm": 0.10888671875,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 11.3961,
+      "mean_token_accuracy": 0.1579543233383447,
+      "num_tokens": 4114077.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.9419952422380446,
+      "epoch": 0.344,
+      "grad_norm": 0.111328125,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 11.362,
+      "mean_token_accuracy": 0.15992560391314328,
+      "num_tokens": 4210403.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.913541816174984,
+      "epoch": 0.352,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 8.78e-06,
+      "loss": 11.3323,
+      "mean_token_accuracy": 0.1633904448710382,
+      "num_tokens": 4306949.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.913567177206278,
+      "epoch": 0.36,
+      "grad_norm": 0.1240234375,
+      "learning_rate": 8.98e-06,
+      "loss": 11.458,
+      "mean_token_accuracy": 0.15837323614396154,
+      "num_tokens": 4406104.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.931741751730442,
+      "epoch": 0.368,
+      "grad_norm": 0.1025390625,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 11.2873,
+      "mean_token_accuracy": 0.15801281332969666,
+      "num_tokens": 4504001.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.904297123849392,
+      "epoch": 0.376,
+      "grad_norm": 0.1025390625,
+      "learning_rate": 9.38e-06,
+      "loss": 11.3813,
+      "mean_token_accuracy": 0.155983364302665,
+      "num_tokens": 4597559.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.9029462151229382,
+      "epoch": 0.384,
+      "grad_norm": 0.09716796875,
+      "learning_rate": 9.58e-06,
+      "loss": 11.3823,
+      "mean_token_accuracy": 0.15874490085989237,
+      "num_tokens": 4693812.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.9272562205791473,
+      "epoch": 0.392,
+      "grad_norm": 0.11474609375,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 11.2937,
+      "mean_token_accuracy": 0.1593953504692763,
+      "num_tokens": 4790965.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.9103233270347117,
+      "epoch": 0.4,
+      "grad_norm": 0.1083984375,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 11.3472,
+      "mean_token_accuracy": 0.15859601702541112,
+      "num_tokens": 4887094.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 4887094.0,
+      "eval_coding_runtime": 93.1337,
+      "eval_coding_samples_per_second": 5.369,
+      "eval_coding_steps_per_second": 2.684,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 4887094.0,
+      "eval_biology_runtime": 41.2135,
+      "eval_biology_samples_per_second": 12.132,
+      "eval_biology_steps_per_second": 6.066,
+      "step": 500
+    },
+    {
+      "entropy": 1.9290625013411045,
+      "epoch": 0.408,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 1.018e-05,
+      "loss": 11.375,
+      "mean_token_accuracy": 0.16000252263620496,
+      "num_tokens": 4985394.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.9234229408204555,
+      "epoch": 0.416,
+      "grad_norm": 0.10595703125,
+      "learning_rate": 1.038e-05,
+      "loss": 11.4479,
+      "mean_token_accuracy": 0.1574494892731309,
+      "num_tokens": 5085369.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.9453741960227489,
+      "epoch": 0.424,
+      "grad_norm": 0.1162109375,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 11.3764,
+      "mean_token_accuracy": 0.15493024131283165,
+      "num_tokens": 5176083.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.9170791164040566,
+      "epoch": 0.432,
+      "grad_norm": 0.10498046875,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 11.4612,
+      "mean_token_accuracy": 0.15664362437091767,
+      "num_tokens": 5271275.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.9397698909044265,
+      "epoch": 0.44,
+      "grad_norm": 0.1044921875,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 11.2801,
+      "mean_token_accuracy": 0.15725393719039857,
+      "num_tokens": 5362624.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.9190262392163278,
+      "epoch": 0.448,
+      "grad_norm": 0.10205078125,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 11.4585,
+      "mean_token_accuracy": 0.157243634108454,
+      "num_tokens": 5460559.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.9073573514819144,
+      "epoch": 0.456,
+      "grad_norm": 0.10791015625,
+      "learning_rate": 1.138e-05,
+      "loss": 11.4459,
+      "mean_token_accuracy": 0.15579957109875978,
+      "num_tokens": 5558471.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.92078455388546,
+      "epoch": 0.464,
+      "grad_norm": 0.10693359375,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 11.3866,
+      "mean_token_accuracy": 0.15736165479756892,
+      "num_tokens": 5653809.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.9094278007745742,
+      "epoch": 0.472,
+      "grad_norm": 0.10595703125,
+      "learning_rate": 1.178e-05,
+      "loss": 11.4047,
+      "mean_token_accuracy": 0.15574069027788937,
+      "num_tokens": 5751529.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.9597051613032819,
+      "epoch": 0.48,
+      "grad_norm": 0.1181640625,
+      "learning_rate": 1.198e-05,
+      "loss": 11.4724,
+      "mean_token_accuracy": 0.1529582162387669,
+      "num_tokens": 5850176.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 5850176.0,
+      "eval_coding_runtime": 93.1046,
+      "eval_coding_samples_per_second": 5.37,
+      "eval_coding_steps_per_second": 2.685,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 5850176.0,
+      "eval_biology_runtime": 41.2971,
+      "eval_biology_samples_per_second": 12.107,
+      "eval_biology_steps_per_second": 6.054,
+      "step": 600
+    },
+    {
+      "entropy": 1.8962870091199875,
+      "epoch": 0.488,
+      "grad_norm": 0.10302734375,
+      "learning_rate": 1.218e-05,
+      "loss": 11.3305,
+      "mean_token_accuracy": 0.16129443161189555,
+      "num_tokens": 5948898.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.9090802267193794,
+      "epoch": 0.496,
+      "grad_norm": 0.10009765625,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 11.2714,
+      "mean_token_accuracy": 0.16270390092395245,
+      "num_tokens": 6046503.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.9413189493119716,
+      "epoch": 0.504,
+      "grad_norm": 0.1064453125,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 11.348,
+      "mean_token_accuracy": 0.15744470418430864,
+      "num_tokens": 6138649.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.9004696995019912,
+      "epoch": 0.512,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 11.5757,
+      "mean_token_accuracy": 0.15675477078184485,
+      "num_tokens": 6240456.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.9349555149674416,
+      "epoch": 0.52,
+      "grad_norm": 0.109375,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 11.3673,
+      "mean_token_accuracy": 0.15686554382555187,
+      "num_tokens": 6333958.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.9110501252114773,
+      "epoch": 0.528,
+      "grad_norm": 0.111328125,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 11.3261,
+      "mean_token_accuracy": 0.15933070508763195,
+      "num_tokens": 6430555.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.89276999309659,
+      "epoch": 0.536,
+      "grad_norm": 0.1103515625,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 11.3362,
+      "mean_token_accuracy": 0.16168827014043927,
+      "num_tokens": 6533033.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.9395132802426815,
+      "epoch": 0.544,
+      "grad_norm": 0.11083984375,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 11.4779,
+      "mean_token_accuracy": 0.15581823773682119,
+      "num_tokens": 6626006.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.941649568080902,
+      "epoch": 0.552,
+      "grad_norm": 0.10986328125,
+      "learning_rate": 1.378e-05,
+      "loss": 11.4785,
+      "mean_token_accuracy": 0.15422701863572003,
+      "num_tokens": 6719864.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.8964682638645172,
+      "epoch": 0.56,
+      "grad_norm": 0.109375,
+      "learning_rate": 1.398e-05,
+      "loss": 11.3283,
+      "mean_token_accuracy": 0.16160819460637868,
+      "num_tokens": 6820754.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 6820754.0,
+      "eval_coding_runtime": 93.0261,
+      "eval_coding_samples_per_second": 5.375,
+      "eval_coding_steps_per_second": 2.687,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 6820754.0,
+      "eval_biology_runtime": 40.9428,
+      "eval_biology_samples_per_second": 12.212,
+      "eval_biology_steps_per_second": 6.106,
+      "step": 700
+    },
+    {
+      "entropy": 1.8812049485743045,
+      "epoch": 0.568,
+      "grad_norm": 0.10986328125,
+      "learning_rate": 1.418e-05,
+      "loss": 11.4292,
+      "mean_token_accuracy": 0.15740878535434605,
+      "num_tokens": 6924942.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.9126729860901832,
+      "epoch": 0.576,
+      "grad_norm": 0.10205078125,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 11.4497,
+      "mean_token_accuracy": 0.15626165526919067,
+      "num_tokens": 7021844.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.9178711868822576,
+      "epoch": 0.584,
+      "grad_norm": 0.1083984375,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 11.4406,
+      "mean_token_accuracy": 0.1565072405152023,
+      "num_tokens": 7118443.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.899195620045066,
+      "epoch": 0.592,
+      "grad_norm": 0.10498046875,
+      "learning_rate": 1.478e-05,
+      "loss": 11.3891,
+      "mean_token_accuracy": 0.15970171936787664,
+      "num_tokens": 7213951.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.8711984746158123,
+      "epoch": 0.6,
+      "grad_norm": 0.107421875,
+      "learning_rate": 1.498e-05,
+      "loss": 11.6155,
+      "mean_token_accuracy": 0.1571272831875831,
+      "num_tokens": 7317984.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.8990586549043655,
+      "epoch": 0.608,
+      "grad_norm": 0.10888671875,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 11.5369,
+      "mean_token_accuracy": 0.15493298289366067,
+      "num_tokens": 7416773.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.8844982825219632,
+      "epoch": 0.616,
+      "grad_norm": 0.11376953125,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 11.4661,
+      "mean_token_accuracy": 0.15754226935096086,
+      "num_tokens": 7514031.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.9250130340456963,
+      "epoch": 0.624,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 11.5539,
+      "mean_token_accuracy": 0.15620456612668931,
+      "num_tokens": 7612843.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.9027365200221538,
+      "epoch": 0.632,
+      "grad_norm": 0.11572265625,
+      "learning_rate": 1.578e-05,
+      "loss": 11.397,
+      "mean_token_accuracy": 0.15837502111680807,
+      "num_tokens": 7710332.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.9388741664588451,
+      "epoch": 0.64,
+      "grad_norm": 0.1015625,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 11.2848,
+      "mean_token_accuracy": 0.15811930843628944,
+      "num_tokens": 7801633.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 7801633.0,
+      "eval_coding_runtime": 92.9264,
+      "eval_coding_samples_per_second": 5.381,
+      "eval_coding_steps_per_second": 2.69,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 7801633.0,
+      "eval_biology_runtime": 40.9144,
+      "eval_biology_samples_per_second": 12.221,
+      "eval_biology_steps_per_second": 6.11,
+      "step": 800
+    },
+    {
+      "entropy": 1.8989893589168787,
+      "epoch": 0.648,
+      "grad_norm": 0.1123046875,
+      "learning_rate": 1.618e-05,
+      "loss": 11.4501,
+      "mean_token_accuracy": 0.1592996684834361,
+      "num_tokens": 7899038.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.9489813253283501,
+      "epoch": 0.656,
+      "grad_norm": 0.1103515625,
+      "learning_rate": 1.638e-05,
+      "loss": 11.6214,
+      "mean_token_accuracy": 0.15152352401055397,
+      "num_tokens": 7995843.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.9073718406260014,
+      "epoch": 0.664,
+      "grad_norm": 0.109375,
+      "learning_rate": 1.658e-05,
+      "loss": 11.4378,
+      "mean_token_accuracy": 0.1551885347813368,
+      "num_tokens": 8092148.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.9346605040133,
+      "epoch": 0.672,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 11.4991,
+      "mean_token_accuracy": 0.15528175202198327,
+      "num_tokens": 8183103.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.8954717583954335,
+      "epoch": 0.68,
+      "grad_norm": 0.11328125,
+      "learning_rate": 1.698e-05,
+      "loss": 11.5802,
+      "mean_token_accuracy": 0.15582030937075614,
+      "num_tokens": 8283708.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.8708562321960926,
+      "epoch": 0.688,
+      "grad_norm": 0.10986328125,
+      "learning_rate": 1.718e-05,
+      "loss": 11.3021,
+      "mean_token_accuracy": 0.16033690557815133,
+      "num_tokens": 8385976.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.940383419394493,
+      "epoch": 0.696,
+      "grad_norm": 0.11376953125,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 11.5672,
+      "mean_token_accuracy": 0.15030423626303674,
+      "num_tokens": 8480263.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.8808147184550763,
+      "epoch": 0.704,
+      "grad_norm": 0.10986328125,
+      "learning_rate": 1.758e-05,
+      "loss": 11.3381,
+      "mean_token_accuracy": 0.16111928690224886,
+      "num_tokens": 8578431.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.8877504222095012,
+      "epoch": 0.712,
+      "grad_norm": 0.10791015625,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 11.3877,
+      "mean_token_accuracy": 0.1602990841027349,
+      "num_tokens": 8679830.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.8855501756072044,
+      "epoch": 0.72,
+      "grad_norm": 0.11279296875,
+      "learning_rate": 1.798e-05,
+      "loss": 11.5192,
+      "mean_token_accuracy": 0.1597036702558398,
+      "num_tokens": 8781342.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 8781342.0,
+      "eval_coding_runtime": 92.7951,
+      "eval_coding_samples_per_second": 5.388,
+      "eval_coding_steps_per_second": 2.694,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 8781342.0,
+      "eval_biology_runtime": 40.8386,
+      "eval_biology_samples_per_second": 12.243,
+      "eval_biology_steps_per_second": 6.122,
+      "step": 900
+    },
+    {
+      "entropy": 1.8865461751818657,
+      "epoch": 0.728,
+      "grad_norm": 0.1083984375,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 11.4372,
+      "mean_token_accuracy": 0.15840516053140163,
+      "num_tokens": 8876240.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.8957439251244068,
+      "epoch": 0.736,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 11.4042,
+      "mean_token_accuracy": 0.16179822930134832,
+      "num_tokens": 8977918.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.9178364366292953,
+      "epoch": 0.744,
+      "grad_norm": 0.11279296875,
+      "learning_rate": 1.858e-05,
+      "loss": 11.3294,
+      "mean_token_accuracy": 0.15907879420556129,
+      "num_tokens": 9073936.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.8959111988544464,
+      "epoch": 0.752,
+      "grad_norm": 0.11328125,
+      "learning_rate": 1.878e-05,
+      "loss": 11.5083,
+      "mean_token_accuracy": 0.1573775229975581,
+      "num_tokens": 9169322.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.9043195970356463,
+      "epoch": 0.76,
+      "grad_norm": 0.10595703125,
+      "learning_rate": 1.898e-05,
+      "loss": 11.4335,
+      "mean_token_accuracy": 0.15969336559064687,
+      "num_tokens": 9267416.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.8812848784029483,
+      "epoch": 0.768,
+      "grad_norm": 0.1044921875,
+      "learning_rate": 1.918e-05,
+      "loss": 11.4766,
+      "mean_token_accuracy": 0.16011672262102367,
+      "num_tokens": 9368141.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.9347192905843258,
+      "epoch": 0.776,
+      "grad_norm": 0.1171875,
+      "learning_rate": 1.938e-05,
+      "loss": 11.4781,
+      "mean_token_accuracy": 0.15778247057460248,
+      "num_tokens": 9463691.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.8791780702769756,
+      "epoch": 0.784,
+      "grad_norm": 0.11376953125,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 11.3971,
+      "mean_token_accuracy": 0.16152906292118133,
+      "num_tokens": 9565236.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.9060623526573182,
+      "epoch": 0.792,
+      "grad_norm": 0.11474609375,
+      "learning_rate": 1.978e-05,
+      "loss": 11.3989,
+      "mean_token_accuracy": 0.15776223079301416,
+      "num_tokens": 9664805.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.9234035909175873,
+      "epoch": 0.8,
+      "grad_norm": 0.109375,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 11.363,
+      "mean_token_accuracy": 0.15743429348804056,
+      "num_tokens": 9761227.0,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_coding_entropy": 1.000721924304962,
+      "eval_coding_loss": 8.267828941345215,
+      "eval_coding_mean_token_accuracy": 0.2908177390098572,
+      "eval_coding_num_tokens": 9761227.0,
+      "eval_coding_runtime": 92.8952,
+      "eval_coding_samples_per_second": 5.382,
+      "eval_coding_steps_per_second": 2.691,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 1.8729813146591185,
+      "eval_biology_loss": 11.396076202392578,
+      "eval_biology_mean_token_accuracy": 0.1593981314599514,
+      "eval_biology_num_tokens": 9761227.0,
+      "eval_biology_runtime": 40.8879,
+      "eval_biology_samples_per_second": 12.229,
+      "eval_biology_steps_per_second": 6.114,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.372499771845745e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cf30dd8c29e5358e9f34b5ea066dcaeb173898ab05b9087697aafc563ccddea
+size 6481