Upload folder using huggingface_hub

.gitattributes

@@ -37,3 +37,4 @@ checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-2500/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-2500/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-2500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-2500/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-2500/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-2500/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e62478432b521c96d94f7737cb25e8368191634c767560becb7ce25af779d70
+size 4962221464

checkpoint-2500/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e1573c9b86b6384e87988d9710ea1a304358f94119246b921723dd3e79285c4
+size 3670322200

checkpoint-2500/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-2500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:805736be15f3f26e6a3e35c83f9b2386d4f7e03c9c4511eebc21c917a4ea7162
+size 7927918951

checkpoint-2500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bbd07b04584d51483a537ebe86fdb5e065d175fef715f69db1afc368ee4945d
+size 14645

checkpoint-2500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb41608c74f0601f51aba42ba070368d58f306fd9096a73c5fc7061ed921c210
+size 1465

checkpoint-2500/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-2500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-2500/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-2500/trainer_state.json

@@ -0,0 +1,3084 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 100,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.049582312093116344,
+      "epoch": 0.016,
+      "grad_norm": 0.40234375,
+      "learning_rate": 7.2e-07,
+      "loss": 0.0306,
+      "mean_token_accuracy": 0.9906190823763609,
+      "num_tokens": 282849.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.05075581485871226,
+      "epoch": 0.032,
+      "grad_norm": 0.3828125,
+      "learning_rate": 1.52e-06,
+      "loss": 0.0311,
+      "mean_token_accuracy": 0.990446624532342,
+      "num_tokens": 561726.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.049523813289124516,
+      "epoch": 0.048,
+      "grad_norm": 0.388671875,
+      "learning_rate": 2.3200000000000002e-06,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.9905388820916414,
+      "num_tokens": 842092.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.04796491010347381,
+      "epoch": 0.064,
+      "grad_norm": 0.4453125,
+      "learning_rate": 3.12e-06,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9908089976757765,
+      "num_tokens": 1133349.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.04753222216386348,
+      "epoch": 0.08,
+      "grad_norm": 0.5859375,
+      "learning_rate": 3.920000000000001e-06,
+      "loss": 0.0301,
+      "mean_token_accuracy": 0.9906909823417663,
+      "num_tokens": 1422340.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.04738440909422934,
+      "epoch": 0.096,
+      "grad_norm": 0.51953125,
+      "learning_rate": 4.7200000000000005e-06,
+      "loss": 0.0303,
+      "mean_token_accuracy": 0.9907543085515499,
+      "num_tokens": 1711821.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.04916054802015424,
+      "epoch": 0.112,
+      "grad_norm": 0.478515625,
+      "learning_rate": 5.5200000000000005e-06,
+      "loss": 0.0314,
+      "mean_token_accuracy": 0.990179405733943,
+      "num_tokens": 1991855.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.04721315624192357,
+      "epoch": 0.128,
+      "grad_norm": 0.439453125,
+      "learning_rate": 6.3200000000000005e-06,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.990847859159112,
+      "num_tokens": 2288720.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.05030787743162364,
+      "epoch": 0.144,
+      "grad_norm": 0.44140625,
+      "learning_rate": 7.1200000000000004e-06,
+      "loss": 0.0323,
+      "mean_token_accuracy": 0.9902282193303108,
+      "num_tokens": 2564352.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.05056912547443062,
+      "epoch": 0.16,
+      "grad_norm": 0.41796875,
+      "learning_rate": 7.92e-06,
+      "loss": 0.0327,
+      "mean_token_accuracy": 0.9901061341166496,
+      "num_tokens": 2841876.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_chemistry_entropy": 0.24093767249584197,
+      "eval_chemistry_loss": 1.4594624042510986,
+      "eval_chemistry_mean_token_accuracy": 0.7775518426895142,
+      "eval_chemistry_num_tokens": 2841876.0,
+      "eval_chemistry_runtime": 34.757,
+      "eval_chemistry_samples_per_second": 14.386,
+      "eval_chemistry_steps_per_second": 3.596,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_math_entropy": 0.18010181015729904,
+      "eval_math_loss": 1.3218717575073242,
+      "eval_math_mean_token_accuracy": 0.8153265738487243,
+      "eval_math_num_tokens": 2841876.0,
+      "eval_math_runtime": 35.7695,
+      "eval_math_samples_per_second": 13.978,
+      "eval_math_steps_per_second": 3.495,
+      "step": 100
+    },
+    {
+      "entropy": 0.04988773693330586,
+      "epoch": 0.176,
+      "grad_norm": 0.431640625,
+      "learning_rate": 8.720000000000001e-06,
+      "loss": 0.0322,
+      "mean_token_accuracy": 0.9902508158236742,
+      "num_tokens": 3129237.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.05084227197803557,
+      "epoch": 0.192,
+      "grad_norm": 0.45703125,
+      "learning_rate": 9.52e-06,
+      "loss": 0.0329,
+      "mean_token_accuracy": 0.9902640093117953,
+      "num_tokens": 3414875.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.052573437918908894,
+      "epoch": 0.208,
+      "grad_norm": 0.423828125,
+      "learning_rate": 1.0320000000000001e-05,
+      "loss": 0.0339,
+      "mean_token_accuracy": 0.9898341007530689,
+      "num_tokens": 3695623.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.05318723081145436,
+      "epoch": 0.224,
+      "grad_norm": 0.5859375,
+      "learning_rate": 1.1120000000000002e-05,
+      "loss": 0.0345,
+      "mean_token_accuracy": 0.9897902756929398,
+      "num_tokens": 3977666.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.052078232262283565,
+      "epoch": 0.24,
+      "grad_norm": 0.70703125,
+      "learning_rate": 1.1920000000000001e-05,
+      "loss": 0.0346,
+      "mean_token_accuracy": 0.9899859059602022,
+      "num_tokens": 4269287.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.05714732928900048,
+      "epoch": 0.256,
+      "grad_norm": 1.265625,
+      "learning_rate": 1.2720000000000002e-05,
+      "loss": 0.0369,
+      "mean_token_accuracy": 0.9894224800169468,
+      "num_tokens": 4558632.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.0642960250377655,
+      "epoch": 0.272,
+      "grad_norm": 1.34375,
+      "learning_rate": 1.3520000000000003e-05,
+      "loss": 0.0424,
+      "mean_token_accuracy": 0.9880315110087394,
+      "num_tokens": 4839270.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.07618674396071583,
+      "epoch": 0.288,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.432e-05,
+      "loss": 0.0515,
+      "mean_token_accuracy": 0.9856673073023557,
+      "num_tokens": 5125447.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.08972589983604848,
+      "epoch": 0.304,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.5120000000000001e-05,
+      "loss": 0.0616,
+      "mean_token_accuracy": 0.98250596486032,
+      "num_tokens": 5415925.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.09321089587174355,
+      "epoch": 0.32,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.5920000000000003e-05,
+      "loss": 0.0662,
+      "mean_token_accuracy": 0.9811959642916918,
+      "num_tokens": 5710178.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_chemistry_entropy": 0.26300008833408356,
+      "eval_chemistry_loss": 1.3655120134353638,
+      "eval_chemistry_mean_token_accuracy": 0.7764006156921387,
+      "eval_chemistry_num_tokens": 5710178.0,
+      "eval_chemistry_runtime": 34.6925,
+      "eval_chemistry_samples_per_second": 14.412,
+      "eval_chemistry_steps_per_second": 3.603,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_math_entropy": 0.21227243894338607,
+      "eval_math_loss": 1.2128866910934448,
+      "eval_math_mean_token_accuracy": 0.8105128321647644,
+      "eval_math_num_tokens": 5710178.0,
+      "eval_math_runtime": 35.7511,
+      "eval_math_samples_per_second": 13.986,
+      "eval_math_steps_per_second": 3.496,
+      "step": 200
+    },
+    {
+      "entropy": 0.1054203346837312,
+      "epoch": 0.336,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.672e-05,
+      "loss": 0.0748,
+      "mean_token_accuracy": 0.9779953259974719,
+      "num_tokens": 5993843.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.10738181113265455,
+      "epoch": 0.352,
+      "grad_norm": 2.84375,
+      "learning_rate": 1.752e-05,
+      "loss": 0.077,
+      "mean_token_accuracy": 0.9772336103022099,
+      "num_tokens": 6279137.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.11388949756510555,
+      "epoch": 0.368,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.832e-05,
+      "loss": 0.082,
+      "mean_token_accuracy": 0.9750573400408029,
+      "num_tokens": 6562387.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.11353842569515109,
+      "epoch": 0.384,
+      "grad_norm": 2.5,
+      "learning_rate": 1.912e-05,
+      "loss": 0.0844,
+      "mean_token_accuracy": 0.9745409067720174,
+      "num_tokens": 6851520.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.12373326132073999,
+      "epoch": 0.4,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.9920000000000002e-05,
+      "loss": 0.0929,
+      "mean_token_accuracy": 0.9715707816183567,
+      "num_tokens": 7138932.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.1284570125862956,
+      "epoch": 0.416,
+      "grad_norm": 2.8125,
+      "learning_rate": 1.9920000000000002e-05,
+      "loss": 0.0971,
+      "mean_token_accuracy": 0.9700299393385649,
+      "num_tokens": 7420720.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.1309962844941765,
+      "epoch": 0.432,
+      "grad_norm": 3.0,
+      "learning_rate": 1.9831111111111113e-05,
+      "loss": 0.1024,
+      "mean_token_accuracy": 0.9681163311004639,
+      "num_tokens": 7695518.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.1339765153825283,
+      "epoch": 0.448,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.9742222222222223e-05,
+      "loss": 0.1012,
+      "mean_token_accuracy": 0.9680692821741104,
+      "num_tokens": 7983543.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.13085799617692828,
+      "epoch": 0.464,
+      "grad_norm": 2.625,
+      "learning_rate": 1.9653333333333334e-05,
+      "loss": 0.0988,
+      "mean_token_accuracy": 0.968861211463809,
+      "num_tokens": 8271787.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.13468673978932202,
+      "epoch": 0.48,
+      "grad_norm": 2.796875,
+      "learning_rate": 1.9564444444444444e-05,
+      "loss": 0.1021,
+      "mean_token_accuracy": 0.9679039020091296,
+      "num_tokens": 8556170.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_chemistry_entropy": 0.2733586312532425,
+      "eval_chemistry_loss": 1.3263864517211914,
+      "eval_chemistry_mean_token_accuracy": 0.7758925561904907,
+      "eval_chemistry_num_tokens": 8556170.0,
+      "eval_chemistry_runtime": 34.7028,
+      "eval_chemistry_samples_per_second": 14.408,
+      "eval_chemistry_steps_per_second": 3.602,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_math_entropy": 0.2326348875761032,
+      "eval_math_loss": 1.1490864753723145,
+      "eval_math_mean_token_accuracy": 0.8080902843475342,
+      "eval_math_num_tokens": 8556170.0,
+      "eval_math_runtime": 35.7382,
+      "eval_math_samples_per_second": 13.991,
+      "eval_math_steps_per_second": 3.498,
+      "step": 300
+    },
+    {
+      "entropy": 0.13333711712621152,
+      "epoch": 0.496,
+      "grad_norm": 3.140625,
+      "learning_rate": 1.9475555555555558e-05,
+      "loss": 0.1,
+      "mean_token_accuracy": 0.9678985115140677,
+      "num_tokens": 8840590.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.13105960306711495,
+      "epoch": 0.512,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.938666666666667e-05,
+      "loss": 0.0989,
+      "mean_token_accuracy": 0.9688469890505076,
+      "num_tokens": 9131903.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.12777495444752276,
+      "epoch": 0.528,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.929777777777778e-05,
+      "loss": 0.0957,
+      "mean_token_accuracy": 0.9697885576635599,
+      "num_tokens": 9429519.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.12928434419445695,
+      "epoch": 0.544,
+      "grad_norm": 2.84375,
+      "learning_rate": 1.920888888888889e-05,
+      "loss": 0.0975,
+      "mean_token_accuracy": 0.9691570393741131,
+      "num_tokens": 9718247.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.1348258295096457,
+      "epoch": 0.56,
+      "grad_norm": 2.953125,
+      "learning_rate": 1.912e-05,
+      "loss": 0.1009,
+      "mean_token_accuracy": 0.9687925592064858,
+      "num_tokens": 9991100.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.13326651249080895,
+      "epoch": 0.576,
+      "grad_norm": 3.03125,
+      "learning_rate": 1.9031111111111114e-05,
+      "loss": 0.0989,
+      "mean_token_accuracy": 0.9687243364751339,
+      "num_tokens": 10276090.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.13166287271305918,
+      "epoch": 0.592,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.8942222222222224e-05,
+      "loss": 0.0985,
+      "mean_token_accuracy": 0.9695811975747347,
+      "num_tokens": 10566762.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.13076401995494963,
+      "epoch": 0.608,
+      "grad_norm": 2.75,
+      "learning_rate": 1.8853333333333335e-05,
+      "loss": 0.0961,
+      "mean_token_accuracy": 0.9704167168587446,
+      "num_tokens": 10857002.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.13047253522090613,
+      "epoch": 0.624,
+      "grad_norm": 2.71875,
+      "learning_rate": 1.8764444444444445e-05,
+      "loss": 0.0967,
+      "mean_token_accuracy": 0.9699896775186062,
+      "num_tokens": 11147326.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.12835560292005538,
+      "epoch": 0.64,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.867555555555556e-05,
+      "loss": 0.0947,
+      "mean_token_accuracy": 0.9701224651187659,
+      "num_tokens": 11437710.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_chemistry_entropy": 0.26942539191246034,
+      "eval_chemistry_loss": 1.3352558612823486,
+      "eval_chemistry_mean_token_accuracy": 0.7756590819358826,
+      "eval_chemistry_num_tokens": 11437710.0,
+      "eval_chemistry_runtime": 34.7257,
+      "eval_chemistry_samples_per_second": 14.399,
+      "eval_chemistry_steps_per_second": 3.6,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_math_entropy": 0.22599203491210937,
+      "eval_math_loss": 1.1578930616378784,
+      "eval_math_mean_token_accuracy": 0.8086360969543457,
+      "eval_math_num_tokens": 11437710.0,
+      "eval_math_runtime": 35.7376,
+      "eval_math_samples_per_second": 13.991,
+      "eval_math_steps_per_second": 3.498,
+      "step": 400
+    },
+    {
+      "entropy": 0.12649631341919304,
+      "epoch": 0.656,
+      "grad_norm": 2.796875,
+      "learning_rate": 1.858666666666667e-05,
+      "loss": 0.095,
+      "mean_token_accuracy": 0.9704382967203855,
+      "num_tokens": 11727268.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.12803141153417527,
+      "epoch": 0.672,
+      "grad_norm": 2.828125,
+      "learning_rate": 1.849777777777778e-05,
+      "loss": 0.0939,
+      "mean_token_accuracy": 0.9711931616067886,
+      "num_tokens": 12020478.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.13106235223822296,
+      "epoch": 0.688,
+      "grad_norm": 3.109375,
+      "learning_rate": 1.840888888888889e-05,
+      "loss": 0.0975,
+      "mean_token_accuracy": 0.9697022248059511,
+      "num_tokens": 12300380.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.12510091881267726,
+      "epoch": 0.704,
+      "grad_norm": 2.90625,
+      "learning_rate": 1.832e-05,
+      "loss": 0.0918,
+      "mean_token_accuracy": 0.9710133023560047,
+      "num_tokens": 12591366.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.1280051540117711,
+      "epoch": 0.72,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.823111111111111e-05,
+      "loss": 0.0932,
+      "mean_token_accuracy": 0.9706473097205162,
+      "num_tokens": 12869618.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.1262424793560058,
+      "epoch": 0.736,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.814222222222222e-05,
+      "loss": 0.0911,
+      "mean_token_accuracy": 0.9721917398273945,
+      "num_tokens": 13156311.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.1231601320207119,
+      "epoch": 0.752,
+      "grad_norm": 2.609375,
+      "learning_rate": 1.8053333333333332e-05,
+      "loss": 0.0899,
+      "mean_token_accuracy": 0.9719577796757222,
+      "num_tokens": 13438911.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.12007905230857432,
+      "epoch": 0.768,
+      "grad_norm": 2.796875,
+      "learning_rate": 1.7964444444444446e-05,
+      "loss": 0.0877,
+      "mean_token_accuracy": 0.9723689883947373,
+      "num_tokens": 13735397.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.1257410950958729,
+      "epoch": 0.784,
+      "grad_norm": 2.75,
+      "learning_rate": 1.7875555555555556e-05,
+      "loss": 0.0898,
+      "mean_token_accuracy": 0.9727259453386068,
+      "num_tokens": 14024497.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.12430990342982114,
+      "epoch": 0.8,
+      "grad_norm": 2.828125,
+      "learning_rate": 1.7786666666666667e-05,
+      "loss": 0.0894,
+      "mean_token_accuracy": 0.9722163360565901,
+      "num_tokens": 14316427.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_chemistry_entropy": 0.2670133001804352,
+      "eval_chemistry_loss": 1.3451496362686157,
+      "eval_chemistry_mean_token_accuracy": 0.7761107358932495,
+      "eval_chemistry_num_tokens": 14316427.0,
+      "eval_chemistry_runtime": 34.7622,
+      "eval_chemistry_samples_per_second": 14.383,
+      "eval_chemistry_steps_per_second": 3.596,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_math_entropy": 0.223143694460392,
+      "eval_math_loss": 1.1661651134490967,
+      "eval_math_mean_token_accuracy": 0.8089712624549865,
+      "eval_math_num_tokens": 14316427.0,
+      "eval_math_runtime": 35.4897,
+      "eval_math_samples_per_second": 14.089,
+      "eval_math_steps_per_second": 3.522,
+      "step": 500
+    },
+    {
+      "entropy": 0.12252137046307325,
+      "epoch": 0.816,
+      "grad_norm": 2.9375,
+      "learning_rate": 1.7697777777777777e-05,
+      "loss": 0.0878,
+      "mean_token_accuracy": 0.9729147344827652,
+      "num_tokens": 14607136.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.12182142208330334,
+      "epoch": 0.832,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.760888888888889e-05,
+      "loss": 0.0867,
+      "mean_token_accuracy": 0.973680478706956,
+      "num_tokens": 14894084.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.1252059457823634,
+      "epoch": 0.848,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.752e-05,
+      "loss": 0.0886,
+      "mean_token_accuracy": 0.9728491563349962,
+      "num_tokens": 15177039.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.12408317467197776,
+      "epoch": 0.864,
+      "grad_norm": 2.828125,
+      "learning_rate": 1.7431111111111112e-05,
+      "loss": 0.0873,
+      "mean_token_accuracy": 0.9731103349477053,
+      "num_tokens": 15463393.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.12185636130161584,
+      "epoch": 0.88,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.7342222222222226e-05,
+      "loss": 0.0864,
+      "mean_token_accuracy": 0.9735086120665073,
+      "num_tokens": 15747943.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.12166528352536261,
+      "epoch": 0.896,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.7253333333333336e-05,
+      "loss": 0.0861,
+      "mean_token_accuracy": 0.9738527856767177,
+      "num_tokens": 16027931.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.12183338818140328,
+      "epoch": 0.912,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.7164444444444447e-05,
+      "loss": 0.0849,
+      "mean_token_accuracy": 0.9736077032983304,
+      "num_tokens": 16317127.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.11985594518482685,
+      "epoch": 0.928,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.7075555555555557e-05,
+      "loss": 0.0845,
+      "mean_token_accuracy": 0.9738747924566269,
+      "num_tokens": 16598757.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.12089673141017557,
+      "epoch": 0.944,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.6986666666666668e-05,
+      "loss": 0.0853,
+      "mean_token_accuracy": 0.974429277703166,
+      "num_tokens": 16882236.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.117778305336833,
+      "epoch": 0.96,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.6897777777777778e-05,
+      "loss": 0.0818,
+      "mean_token_accuracy": 0.9753529716283083,
+      "num_tokens": 17169495.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_chemistry_entropy": 0.2685954006910324,
+      "eval_chemistry_loss": 1.3377221822738647,
+      "eval_chemistry_mean_token_accuracy": 0.7758646550178527,
+      "eval_chemistry_num_tokens": 17169495.0,
+      "eval_chemistry_runtime": 34.7493,
+      "eval_chemistry_samples_per_second": 14.389,
+      "eval_chemistry_steps_per_second": 3.597,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_math_entropy": 0.21977827924489976,
+      "eval_math_loss": 1.166742205619812,
+      "eval_math_mean_token_accuracy": 0.8097256078720093,
+      "eval_math_num_tokens": 17169495.0,
+      "eval_math_runtime": 35.8222,
+      "eval_math_samples_per_second": 13.958,
+      "eval_math_steps_per_second": 3.489,
+      "step": 600
+    },
+    {
+      "entropy": 0.11901968852616847,
+      "epoch": 0.976,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.680888888888889e-05,
+      "loss": 0.0819,
+      "mean_token_accuracy": 0.9747418876737356,
+      "num_tokens": 17454147.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.11983131496235729,
+      "epoch": 0.992,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.672e-05,
+      "loss": 0.0825,
+      "mean_token_accuracy": 0.9750407934188843,
+      "num_tokens": 17742596.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.10401241672225296,
+      "epoch": 1.008,
+      "grad_norm": 1.8515625,
+      "learning_rate": 1.6631111111111113e-05,
+      "loss": 0.0702,
+      "mean_token_accuracy": 0.9788199871778488,
+      "num_tokens": 18033540.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.08530816063284874,
+      "epoch": 1.024,
+      "grad_norm": 1.8203125,
+      "learning_rate": 1.6542222222222223e-05,
+      "loss": 0.059,
+      "mean_token_accuracy": 0.982539826259017,
+      "num_tokens": 18316346.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.08670682646334171,
+      "epoch": 1.04,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.6453333333333334e-05,
+      "loss": 0.0605,
+      "mean_token_accuracy": 0.9820570342242718,
+      "num_tokens": 18595230.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.08672361080534756,
+      "epoch": 1.056,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.6364444444444444e-05,
+      "loss": 0.0606,
+      "mean_token_accuracy": 0.9823420338332653,
+      "num_tokens": 18885547.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.08972970175091177,
+      "epoch": 1.072,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.6275555555555558e-05,
+      "loss": 0.0619,
+      "mean_token_accuracy": 0.9819000914692879,
+      "num_tokens": 19169083.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.08532073406968266,
+      "epoch": 1.088,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.618666666666667e-05,
+      "loss": 0.0586,
+      "mean_token_accuracy": 0.9825958292931318,
+      "num_tokens": 19461465.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.08605421562679112,
+      "epoch": 1.104,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.609777777777778e-05,
+      "loss": 0.0594,
+      "mean_token_accuracy": 0.9826772827655077,
+      "num_tokens": 19755027.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.084381258697249,
+      "epoch": 1.12,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.600888888888889e-05,
+      "loss": 0.0584,
+      "mean_token_accuracy": 0.9828958626836538,
+      "num_tokens": 20043293.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_chemistry_entropy": 0.2597039748430252,
+      "eval_chemistry_loss": 1.3768759965896606,
+      "eval_chemistry_mean_token_accuracy": 0.776004008769989,
+      "eval_chemistry_num_tokens": 20043293.0,
+      "eval_chemistry_runtime": 34.7425,
+      "eval_chemistry_samples_per_second": 14.392,
+      "eval_chemistry_steps_per_second": 3.598,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_math_entropy": 0.2076559336781502,
+      "eval_math_loss": 1.2019093036651611,
+      "eval_math_mean_token_accuracy": 0.8111752443313599,
+      "eval_math_num_tokens": 20043293.0,
+      "eval_math_runtime": 35.7886,
+      "eval_math_samples_per_second": 13.971,
+      "eval_math_steps_per_second": 3.493,
+      "step": 700
+    },
+    {
+      "entropy": 0.09029958308674395,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.5920000000000003e-05,
+      "loss": 0.0646,
+      "mean_token_accuracy": 0.9808140520006419,
+      "num_tokens": 20323010.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.08711256731767207,
+      "epoch": 1.152,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.5831111111111114e-05,
+      "loss": 0.0607,
+      "mean_token_accuracy": 0.9819911230355501,
+      "num_tokens": 20613266.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.08424482597038149,
+      "epoch": 1.168,
+      "grad_norm": 1.8828125,
+      "learning_rate": 1.5742222222222224e-05,
+      "loss": 0.0583,
+      "mean_token_accuracy": 0.9828614890575409,
+      "num_tokens": 20910404.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.086951145529747,
+      "epoch": 1.184,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.5653333333333335e-05,
+      "loss": 0.0596,
+      "mean_token_accuracy": 0.9822407346218824,
+      "num_tokens": 21200197.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.08553398705553264,
+      "epoch": 1.2,
+      "grad_norm": 1.8515625,
+      "learning_rate": 1.5564444444444445e-05,
+      "loss": 0.0595,
+      "mean_token_accuracy": 0.9823605198413133,
+      "num_tokens": 21496441.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.08706241557374597,
+      "epoch": 1.216,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.5475555555555556e-05,
+      "loss": 0.0604,
+      "mean_token_accuracy": 0.9823201231658458,
+      "num_tokens": 21778030.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.08789820885285735,
+      "epoch": 1.232,
+      "grad_norm": 2.0,
+      "learning_rate": 1.5386666666666666e-05,
+      "loss": 0.0619,
+      "mean_token_accuracy": 0.9814675074070692,
+      "num_tokens": 22060851.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.08552719466388226,
+      "epoch": 1.248,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.5297777777777777e-05,
+      "loss": 0.0594,
+      "mean_token_accuracy": 0.9823314778506755,
+      "num_tokens": 22348738.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.08782941014505923,
+      "epoch": 1.264,
+      "grad_norm": 1.7578125,
+      "learning_rate": 1.520888888888889e-05,
+      "loss": 0.0609,
+      "mean_token_accuracy": 0.9820897091180086,
+      "num_tokens": 22635039.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.08776575655210764,
+      "epoch": 1.28,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.5120000000000001e-05,
+      "loss": 0.062,
+      "mean_token_accuracy": 0.9818732112646102,
+      "num_tokens": 22915190.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_chemistry_entropy": 0.262844596862793,
+      "eval_chemistry_loss": 1.3627640008926392,
+      "eval_chemistry_mean_token_accuracy": 0.7757310657501221,
+      "eval_chemistry_num_tokens": 22915190.0,
+      "eval_chemistry_runtime": 34.756,
+      "eval_chemistry_samples_per_second": 14.386,
+      "eval_chemistry_steps_per_second": 3.596,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_math_entropy": 0.20891815394163132,
+      "eval_math_loss": 1.1918283700942993,
+      "eval_math_mean_token_accuracy": 0.8114760699272155,
+      "eval_math_num_tokens": 22915190.0,
+      "eval_math_runtime": 35.7894,
+      "eval_math_samples_per_second": 13.971,
+      "eval_math_steps_per_second": 3.493,
+      "step": 800
+    },
+    {
+      "entropy": 0.08705279752612113,
+      "epoch": 1.296,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.5031111111111111e-05,
+      "loss": 0.0603,
+      "mean_token_accuracy": 0.982403326779604,
+      "num_tokens": 23194947.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.08580190755892544,
+      "epoch": 1.312,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.4942222222222222e-05,
+      "loss": 0.058,
+      "mean_token_accuracy": 0.9832350175827742,
+      "num_tokens": 23483496.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.08680400990415364,
+      "epoch": 1.328,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.4853333333333336e-05,
+      "loss": 0.0601,
+      "mean_token_accuracy": 0.9825425524264574,
+      "num_tokens": 23762589.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.08539955569431186,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 1.875,
+      "learning_rate": 1.4764444444444446e-05,
+      "loss": 0.0582,
+      "mean_token_accuracy": 0.983034698292613,
+      "num_tokens": 24041497.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.08369267631787806,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 1.8046875,
+      "learning_rate": 1.4675555555555557e-05,
+      "loss": 0.0594,
+      "mean_token_accuracy": 0.9825323965400458,
+      "num_tokens": 24330591.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.0880416841711849,
+      "epoch": 1.376,
+      "grad_norm": 1.890625,
+      "learning_rate": 1.4586666666666667e-05,
+      "loss": 0.0595,
+      "mean_token_accuracy": 0.982681792974472,
+      "num_tokens": 24611420.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.08504764605313539,
+      "epoch": 1.392,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.449777777777778e-05,
+      "loss": 0.0584,
+      "mean_token_accuracy": 0.9826866809278727,
+      "num_tokens": 24894694.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.083847014605999,
+      "epoch": 1.408,
+      "grad_norm": 2.0,
+      "learning_rate": 1.440888888888889e-05,
+      "loss": 0.057,
+      "mean_token_accuracy": 0.9832074366509914,
+      "num_tokens": 25184946.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.08376108810771257,
+      "epoch": 1.424,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.432e-05,
+      "loss": 0.0582,
+      "mean_token_accuracy": 0.9830348044633865,
+      "num_tokens": 25474746.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.0831807145383209,
+      "epoch": 1.44,
+      "grad_norm": 2.125,
+      "learning_rate": 1.4231111111111114e-05,
+      "loss": 0.0573,
+      "mean_token_accuracy": 0.9832527209073305,
+      "num_tokens": 25758550.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_chemistry_entropy": 0.2588958297967911,
+      "eval_chemistry_loss": 1.3800510168075562,
+      "eval_chemistry_mean_token_accuracy": 0.7761167316436768,
+      "eval_chemistry_num_tokens": 25758550.0,
+      "eval_chemistry_runtime": 34.746,
+      "eval_chemistry_samples_per_second": 14.39,
+      "eval_chemistry_steps_per_second": 3.598,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_math_entropy": 0.20648883867263795,
+      "eval_math_loss": 1.1997566223144531,
+      "eval_math_mean_token_accuracy": 0.8119877796173096,
+      "eval_math_num_tokens": 25758550.0,
+      "eval_math_runtime": 35.7996,
+      "eval_math_samples_per_second": 13.967,
+      "eval_math_steps_per_second": 3.492,
+      "step": 900
+    },
+    {
+      "entropy": 0.08563981833867729,
+      "epoch": 1.456,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.4142222222222224e-05,
+      "loss": 0.0572,
+      "mean_token_accuracy": 0.9828862383961677,
+      "num_tokens": 26039862.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.0852160410489887,
+      "epoch": 1.472,
+      "grad_norm": 1.8125,
+      "learning_rate": 1.4053333333333335e-05,
+      "loss": 0.0589,
+      "mean_token_accuracy": 0.9824321672320366,
+      "num_tokens": 26318666.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.08569062508177012,
+      "epoch": 1.488,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.3964444444444445e-05,
+      "loss": 0.0585,
+      "mean_token_accuracy": 0.9830207195132971,
+      "num_tokens": 26594867.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.08032872283365577,
+      "epoch": 1.504,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.3875555555555557e-05,
+      "loss": 0.0554,
+      "mean_token_accuracy": 0.9839188922196627,
+      "num_tokens": 26887233.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.0836705738445744,
+      "epoch": 1.52,
+      "grad_norm": 1.6015625,
+      "learning_rate": 1.3786666666666668e-05,
+      "loss": 0.0568,
+      "mean_token_accuracy": 0.9833197060972452,
+      "num_tokens": 27177448.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.08208613845054061,
+      "epoch": 1.536,
+      "grad_norm": 1.65625,
+      "learning_rate": 1.3697777777777778e-05,
+      "loss": 0.0553,
+      "mean_token_accuracy": 0.9837426465004683,
+      "num_tokens": 27467857.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.08092109204735606,
+      "epoch": 1.552,
+      "grad_norm": 1.8203125,
+      "learning_rate": 1.3608888888888889e-05,
+      "loss": 0.0554,
+      "mean_token_accuracy": 0.9836731895804405,
+      "num_tokens": 27755138.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.07861132090911269,
+      "epoch": 1.568,
+      "grad_norm": 1.7734375,
+      "learning_rate": 1.3520000000000003e-05,
+      "loss": 0.0537,
+      "mean_token_accuracy": 0.9842242918908596,
+      "num_tokens": 28049363.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.08059575033839791,
+      "epoch": 1.584,
+      "grad_norm": 1.9453125,
+      "learning_rate": 1.3431111111111113e-05,
+      "loss": 0.0548,
+      "mean_token_accuracy": 0.9839956350624561,
+      "num_tokens": 28337890.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.08031310953665524,
+      "epoch": 1.6,
+      "grad_norm": 1.84375,
+      "learning_rate": 1.3342222222222224e-05,
+      "loss": 0.0544,
+      "mean_token_accuracy": 0.983878904953599,
+      "num_tokens": 28626068.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_chemistry_entropy": 0.2602477210760117,
+      "eval_chemistry_loss": 1.3820487260818481,
+      "eval_chemistry_mean_token_accuracy": 0.7756126351356506,
+      "eval_chemistry_num_tokens": 28626068.0,
+      "eval_chemistry_runtime": 34.7493,
+      "eval_chemistry_samples_per_second": 14.389,
+      "eval_chemistry_steps_per_second": 3.597,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_math_entropy": 0.20556720286607744,
+      "eval_math_loss": 1.207412600517273,
+      "eval_math_mean_token_accuracy": 0.8115635991096497,
+      "eval_math_num_tokens": 28626068.0,
+      "eval_math_runtime": 35.4675,
+      "eval_math_samples_per_second": 14.097,
+      "eval_math_steps_per_second": 3.524,
+      "step": 1000
+    },
+    {
+      "entropy": 0.08080549829173833,
+      "epoch": 1.616,
+      "grad_norm": 1.828125,
+      "learning_rate": 1.3253333333333334e-05,
+      "loss": 0.0542,
+      "mean_token_accuracy": 0.9843609642237425,
+      "num_tokens": 28913208.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.08292383486405015,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 1.7734375,
+      "learning_rate": 1.3164444444444446e-05,
+      "loss": 0.0575,
+      "mean_token_accuracy": 0.9831736672669649,
+      "num_tokens": 29188917.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.07944189475383609,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 1.8203125,
+      "learning_rate": 1.3075555555555557e-05,
+      "loss": 0.0531,
+      "mean_token_accuracy": 0.9844527587294578,
+      "num_tokens": 29477292.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.08005691035650671,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 1.828125,
+      "learning_rate": 1.2986666666666667e-05,
+      "loss": 0.0532,
+      "mean_token_accuracy": 0.9843937639147043,
+      "num_tokens": 29772480.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.07776331030763686,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 1.828125,
+      "learning_rate": 1.2897777777777778e-05,
+      "loss": 0.053,
+      "mean_token_accuracy": 0.9847775120288134,
+      "num_tokens": 30061007.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.0810230050003156,
+      "epoch": 1.696,
+      "grad_norm": 1.6875,
+      "learning_rate": 1.2808888888888891e-05,
+      "loss": 0.0541,
+      "mean_token_accuracy": 0.984524767100811,
+      "num_tokens": 30344415.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.0787459987681359,
+      "epoch": 1.712,
+      "grad_norm": 1.7421875,
+      "learning_rate": 1.2720000000000002e-05,
+      "loss": 0.0534,
+      "mean_token_accuracy": 0.9842950701713562,
+      "num_tokens": 30640891.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.0789328703423962,
+      "epoch": 1.728,
+      "grad_norm": 1.9765625,
+      "learning_rate": 1.2631111111111112e-05,
+      "loss": 0.0531,
+      "mean_token_accuracy": 0.9845080733299255,
+      "num_tokens": 30921295.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.0788962548598647,
+      "epoch": 1.744,
+      "grad_norm": 1.828125,
+      "learning_rate": 1.2542222222222223e-05,
+      "loss": 0.0534,
+      "mean_token_accuracy": 0.9843788214027882,
+      "num_tokens": 31206375.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.08012886063661426,
+      "epoch": 1.76,
+      "grad_norm": 1.9453125,
+      "learning_rate": 1.2453333333333335e-05,
+      "loss": 0.0536,
+      "mean_token_accuracy": 0.9843036912381649,
+      "num_tokens": 31491856.0,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_chemistry_entropy": 0.2553063322305679,
+      "eval_chemistry_loss": 1.3884915113449097,
+      "eval_chemistry_mean_token_accuracy": 0.7761442866325379,
+      "eval_chemistry_num_tokens": 31491856.0,
+      "eval_chemistry_runtime": 34.7224,
+      "eval_chemistry_samples_per_second": 14.4,
+      "eval_chemistry_steps_per_second": 3.6,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_math_entropy": 0.20213844722509383,
+      "eval_math_loss": 1.2131967544555664,
+      "eval_math_mean_token_accuracy": 0.811974012374878,
+      "eval_math_num_tokens": 31491856.0,
+      "eval_math_runtime": 35.7761,
+      "eval_math_samples_per_second": 13.976,
+      "eval_math_steps_per_second": 3.494,
+      "step": 1100
+    },
+    {
+      "entropy": 0.07542885933071375,
+      "epoch": 1.776,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.2364444444444445e-05,
+      "loss": 0.051,
+      "mean_token_accuracy": 0.9852005202323199,
+      "num_tokens": 31782829.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.07717105301562696,
+      "epoch": 1.792,
+      "grad_norm": 1.6640625,
+      "learning_rate": 1.2275555555555556e-05,
+      "loss": 0.0519,
+      "mean_token_accuracy": 0.9847183488309383,
+      "num_tokens": 32070187.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.07923196398187429,
+      "epoch": 1.808,
+      "grad_norm": 1.640625,
+      "learning_rate": 1.2186666666666666e-05,
+      "loss": 0.0524,
+      "mean_token_accuracy": 0.9846949424594641,
+      "num_tokens": 32355072.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.07861853984650224,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 1.6640625,
+      "learning_rate": 1.209777777777778e-05,
+      "loss": 0.0531,
+      "mean_token_accuracy": 0.9844680838286877,
+      "num_tokens": 32634013.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.07815332850441337,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 1.609375,
+      "learning_rate": 1.200888888888889e-05,
+      "loss": 0.0524,
+      "mean_token_accuracy": 0.9846453677862883,
+      "num_tokens": 32917222.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.0771151206921786,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 1.75,
+      "learning_rate": 1.1920000000000001e-05,
+      "loss": 0.0515,
+      "mean_token_accuracy": 0.9850007418543101,
+      "num_tokens": 33207147.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.07652715451549738,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 1.640625,
+      "learning_rate": 1.1831111111111112e-05,
+      "loss": 0.0519,
+      "mean_token_accuracy": 0.9849152982234954,
+      "num_tokens": 33496572.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.07685239620041102,
+      "epoch": 1.888,
+      "grad_norm": 1.7421875,
+      "learning_rate": 1.1742222222222224e-05,
+      "loss": 0.0511,
+      "mean_token_accuracy": 0.9850867554545403,
+      "num_tokens": 33775788.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.07776038108859211,
+      "epoch": 1.904,
+      "grad_norm": 1.7890625,
+      "learning_rate": 1.1653333333333334e-05,
+      "loss": 0.0519,
+      "mean_token_accuracy": 0.984935637190938,
+      "num_tokens": 34052322.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.07653295549098402,
+      "epoch": 1.92,
+      "grad_norm": 1.4609375,
+      "learning_rate": 1.1564444444444445e-05,
+      "loss": 0.051,
+      "mean_token_accuracy": 0.9852206721901894,
+      "num_tokens": 34331468.0,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_chemistry_entropy": 0.2549130402803421,
+      "eval_chemistry_loss": 1.392774224281311,
+      "eval_chemistry_mean_token_accuracy": 0.7761725673675537,
+      "eval_chemistry_num_tokens": 34331468.0,
+      "eval_chemistry_runtime": 34.7035,
+      "eval_chemistry_samples_per_second": 14.408,
+      "eval_chemistry_steps_per_second": 3.602,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_math_entropy": 0.19969486129283906,
+      "eval_math_loss": 1.219051480293274,
+      "eval_math_mean_token_accuracy": 0.8130026779174805,
+      "eval_math_num_tokens": 34331468.0,
+      "eval_math_runtime": 35.7413,
+      "eval_math_samples_per_second": 13.989,
+      "eval_math_steps_per_second": 3.497,
+      "step": 1200
+    },
+    {
+      "entropy": 0.07580579959321768,
+      "epoch": 1.936,
+      "grad_norm": 1.8515625,
+      "learning_rate": 1.1475555555555557e-05,
+      "loss": 0.0515,
+      "mean_token_accuracy": 0.9851022522896529,
+      "num_tokens": 34612743.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.07751624898519367,
+      "epoch": 1.952,
+      "grad_norm": 1.6328125,
+      "learning_rate": 1.1386666666666669e-05,
+      "loss": 0.0513,
+      "mean_token_accuracy": 0.9850031793117523,
+      "num_tokens": 34891115.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.07615254581905902,
+      "epoch": 1.968,
+      "grad_norm": 1.796875,
+      "learning_rate": 1.129777777777778e-05,
+      "loss": 0.0506,
+      "mean_token_accuracy": 0.9852058101445437,
+      "num_tokens": 35179505.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.07313250387087464,
+      "epoch": 1.984,
+      "grad_norm": 1.6171875,
+      "learning_rate": 1.120888888888889e-05,
+      "loss": 0.0488,
+      "mean_token_accuracy": 0.9859140012413263,
+      "num_tokens": 35472596.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.0761550744296983,
+      "epoch": 2.0,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.1120000000000002e-05,
+      "loss": 0.0504,
+      "mean_token_accuracy": 0.9853185825049877,
+      "num_tokens": 35772848.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.056211807671934365,
+      "epoch": 2.016,
+      "grad_norm": 0.69921875,
+      "learning_rate": 1.1031111111111112e-05,
+      "loss": 0.0348,
+      "mean_token_accuracy": 0.9902306616306304,
+      "num_tokens": 36060330.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.05393635639920831,
+      "epoch": 2.032,
+      "grad_norm": 0.80078125,
+      "learning_rate": 1.0942222222222223e-05,
+      "loss": 0.0347,
+      "mean_token_accuracy": 0.989858265221119,
+      "num_tokens": 36343894.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.05242305672727525,
+      "epoch": 2.048,
+      "grad_norm": 0.82421875,
+      "learning_rate": 1.0853333333333333e-05,
+      "loss": 0.0336,
+      "mean_token_accuracy": 0.9902104564011097,
+      "num_tokens": 36636410.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.054460834711790085,
+      "epoch": 2.064,
+      "grad_norm": 0.7109375,
+      "learning_rate": 1.0764444444444445e-05,
+      "loss": 0.0339,
+      "mean_token_accuracy": 0.989987451210618,
+      "num_tokens": 36925382.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.05415360638871789,
+      "epoch": 2.08,
+      "grad_norm": 0.84375,
+      "learning_rate": 1.0675555555555558e-05,
+      "loss": 0.0345,
+      "mean_token_accuracy": 0.9898186340928078,
+      "num_tokens": 37208672.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_chemistry_entropy": 0.24465568888187408,
+      "eval_chemistry_loss": 1.4367998838424683,
+      "eval_chemistry_mean_token_accuracy": 0.7765562734603881,
+      "eval_chemistry_num_tokens": 37208672.0,
+      "eval_chemistry_runtime": 34.6804,
+      "eval_chemistry_samples_per_second": 14.417,
+      "eval_chemistry_steps_per_second": 3.604,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_math_entropy": 0.1864414402246475,
+      "eval_math_loss": 1.2687431573867798,
+      "eval_math_mean_token_accuracy": 0.814744426727295,
+      "eval_math_num_tokens": 37208672.0,
+      "eval_math_runtime": 35.7331,
+      "eval_math_samples_per_second": 13.993,
+      "eval_math_steps_per_second": 3.498,
+      "step": 1300
+    },
+    {
+      "entropy": 0.053227876662276685,
+      "epoch": 2.096,
+      "grad_norm": 0.7265625,
+      "learning_rate": 1.0586666666666668e-05,
+      "loss": 0.0347,
+      "mean_token_accuracy": 0.989788818731904,
+      "num_tokens": 37491954.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.05289595597423613,
+      "epoch": 2.112,
+      "grad_norm": 0.71875,
+      "learning_rate": 1.0497777777777779e-05,
+      "loss": 0.0336,
+      "mean_token_accuracy": 0.9899541862308979,
+      "num_tokens": 37787033.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.05384734331164509,
+      "epoch": 2.128,
+      "grad_norm": 0.64453125,
+      "learning_rate": 1.040888888888889e-05,
+      "loss": 0.0347,
+      "mean_token_accuracy": 0.9895545594394207,
+      "num_tokens": 38068955.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.052285452221985905,
+      "epoch": 2.144,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.0320000000000001e-05,
+      "loss": 0.034,
+      "mean_token_accuracy": 0.9899950183928012,
+      "num_tokens": 38359914.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.053004868398420515,
+      "epoch": 2.16,
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.0231111111111112e-05,
+      "loss": 0.0343,
+      "mean_token_accuracy": 0.9898937761783599,
+      "num_tokens": 38649684.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.05320889897411689,
+      "epoch": 2.176,
+      "grad_norm": 0.5703125,
+      "learning_rate": 1.0142222222222222e-05,
+      "loss": 0.0342,
+      "mean_token_accuracy": 0.9898546732962131,
+      "num_tokens": 38938155.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.05214473644737154,
+      "epoch": 2.192,
+      "grad_norm": 0.61328125,
+      "learning_rate": 1.0053333333333334e-05,
+      "loss": 0.0341,
+      "mean_token_accuracy": 0.9901005830615759,
+      "num_tokens": 39228800.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.053439665515907106,
+      "epoch": 2.208,
+      "grad_norm": 0.6171875,
+      "learning_rate": 9.964444444444445e-06,
+      "loss": 0.0343,
+      "mean_token_accuracy": 0.989708437025547,
+      "num_tokens": 39513118.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.052510200347751376,
+      "epoch": 2.224,
+      "grad_norm": 0.7734375,
+      "learning_rate": 9.875555555555557e-06,
+      "loss": 0.0338,
+      "mean_token_accuracy": 0.9900678258389235,
+      "num_tokens": 39804212.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.05290528133045882,
+      "epoch": 2.24,
+      "grad_norm": 0.64453125,
+      "learning_rate": 9.786666666666667e-06,
+      "loss": 0.0344,
+      "mean_token_accuracy": 0.9897066332399845,
+      "num_tokens": 40092012.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_chemistry_entropy": 0.24463952136039735,
+      "eval_chemistry_loss": 1.4411935806274414,
+      "eval_chemistry_mean_token_accuracy": 0.7765587282180786,
+      "eval_chemistry_num_tokens": 40092012.0,
+      "eval_chemistry_runtime": 34.6754,
+      "eval_chemistry_samples_per_second": 14.419,
+      "eval_chemistry_steps_per_second": 3.605,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_math_entropy": 0.18511718076467515,
+      "eval_math_loss": 1.2752846479415894,
+      "eval_math_mean_token_accuracy": 0.8149898409843445,
+      "eval_math_num_tokens": 40092012.0,
+      "eval_math_runtime": 35.713,
+      "eval_math_samples_per_second": 14.0,
+      "eval_math_steps_per_second": 3.5,
+      "step": 1400
+    },
+    {
+      "entropy": 0.05300161670893431,
+      "epoch": 2.2560000000000002,
+      "grad_norm": 0.80859375,
+      "learning_rate": 9.697777777777778e-06,
+      "loss": 0.0346,
+      "mean_token_accuracy": 0.9895025338977576,
+      "num_tokens": 40380069.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.054382482054643334,
+      "epoch": 2.2720000000000002,
+      "grad_norm": 0.8203125,
+      "learning_rate": 9.60888888888889e-06,
+      "loss": 0.0358,
+      "mean_token_accuracy": 0.9892595294862986,
+      "num_tokens": 40655903.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.05288956081494689,
+      "epoch": 2.288,
+      "grad_norm": 0.8359375,
+      "learning_rate": 9.52e-06,
+      "loss": 0.0338,
+      "mean_token_accuracy": 0.9897118430584669,
+      "num_tokens": 40944791.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.052792912826407704,
+      "epoch": 2.304,
+      "grad_norm": 0.7578125,
+      "learning_rate": 9.431111111111112e-06,
+      "loss": 0.0346,
+      "mean_token_accuracy": 0.989751597866416,
+      "num_tokens": 41227146.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.053943155938759445,
+      "epoch": 2.32,
+      "grad_norm": 0.7890625,
+      "learning_rate": 9.342222222222223e-06,
+      "loss": 0.0351,
+      "mean_token_accuracy": 0.9896157145500183,
+      "num_tokens": 41509322.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.05290579678257927,
+      "epoch": 2.336,
+      "grad_norm": 0.7109375,
+      "learning_rate": 9.253333333333333e-06,
+      "loss": 0.0349,
+      "mean_token_accuracy": 0.9897964365780354,
+      "num_tokens": 41792799.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.05324005091097206,
+      "epoch": 2.352,
+      "grad_norm": 0.69140625,
+      "learning_rate": 9.164444444444446e-06,
+      "loss": 0.0346,
+      "mean_token_accuracy": 0.989767849072814,
+      "num_tokens": 42083080.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.052101924282032995,
+      "epoch": 2.368,
+      "grad_norm": 0.60546875,
+      "learning_rate": 9.075555555555556e-06,
+      "loss": 0.0345,
+      "mean_token_accuracy": 0.9897124897688627,
+      "num_tokens": 42368442.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.05193082329351455,
+      "epoch": 2.384,
+      "grad_norm": 0.77734375,
+      "learning_rate": 8.986666666666666e-06,
+      "loss": 0.0337,
+      "mean_token_accuracy": 0.99007040373981,
+      "num_tokens": 42663926.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.05398317989893257,
+      "epoch": 2.4,
+      "grad_norm": 0.8046875,
+      "learning_rate": 8.897777777777779e-06,
+      "loss": 0.035,
+      "mean_token_accuracy": 0.989552128314972,
+      "num_tokens": 42946351.0,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_chemistry_entropy": 0.2435439591407776,
+      "eval_chemistry_loss": 1.4468469619750977,
+      "eval_chemistry_mean_token_accuracy": 0.7765617952346802,
+      "eval_chemistry_num_tokens": 42946351.0,
+      "eval_chemistry_runtime": 34.6793,
+      "eval_chemistry_samples_per_second": 14.418,
+      "eval_chemistry_steps_per_second": 3.604,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_math_entropy": 0.18436058980226516,
+      "eval_math_loss": 1.2825661897659302,
+      "eval_math_mean_token_accuracy": 0.8150607142448425,
+      "eval_math_num_tokens": 42946351.0,
+      "eval_math_runtime": 35.4927,
+      "eval_math_samples_per_second": 14.087,
+      "eval_math_steps_per_second": 3.522,
+      "step": 1500
+    },
+    {
+      "entropy": 0.05408466912340373,
+      "epoch": 2.416,
+      "grad_norm": 0.9296875,
+      "learning_rate": 8.80888888888889e-06,
+      "loss": 0.0349,
+      "mean_token_accuracy": 0.9895286500453949,
+      "num_tokens": 43227782.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.052965849777683614,
+      "epoch": 2.432,
+      "grad_norm": 1.0625,
+      "learning_rate": 8.720000000000001e-06,
+      "loss": 0.0353,
+      "mean_token_accuracy": 0.9892889097332954,
+      "num_tokens": 43511939.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.051178892608731985,
+      "epoch": 2.448,
+      "grad_norm": 0.6875,
+      "learning_rate": 8.631111111111112e-06,
+      "loss": 0.0329,
+      "mean_token_accuracy": 0.9901369903236628,
+      "num_tokens": 43811005.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.05202908001374453,
+      "epoch": 2.464,
+      "grad_norm": 0.72265625,
+      "learning_rate": 8.542222222222222e-06,
+      "loss": 0.0339,
+      "mean_token_accuracy": 0.9899697575718165,
+      "num_tokens": 44106223.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.05558811663649976,
+      "epoch": 2.48,
+      "grad_norm": 0.76171875,
+      "learning_rate": 8.453333333333334e-06,
+      "loss": 0.0361,
+      "mean_token_accuracy": 0.9893935658037663,
+      "num_tokens": 44379095.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.05247082086279988,
+      "epoch": 2.496,
+      "grad_norm": 0.69921875,
+      "learning_rate": 8.364444444444445e-06,
+      "loss": 0.0346,
+      "mean_token_accuracy": 0.9898388765752315,
+      "num_tokens": 44670316.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.05363391661085189,
+      "epoch": 2.512,
+      "grad_norm": 0.81640625,
+      "learning_rate": 8.275555555555557e-06,
+      "loss": 0.0352,
+      "mean_token_accuracy": 0.9896232932806015,
+      "num_tokens": 44952877.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.05272995062405243,
+      "epoch": 2.528,
+      "grad_norm": 0.68359375,
+      "learning_rate": 8.186666666666667e-06,
+      "loss": 0.0345,
+      "mean_token_accuracy": 0.9896573759615421,
+      "num_tokens": 45237653.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.05119302978273481,
+      "epoch": 2.544,
+      "grad_norm": 0.75390625,
+      "learning_rate": 8.09777777777778e-06,
+      "loss": 0.0341,
+      "mean_token_accuracy": 0.9897664703428746,
+      "num_tokens": 45524866.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.05278779338113963,
+      "epoch": 2.56,
+      "grad_norm": 0.6796875,
+      "learning_rate": 8.00888888888889e-06,
+      "loss": 0.0356,
+      "mean_token_accuracy": 0.9893198136240244,
+      "num_tokens": 45803783.0,
+      "step": 1600
+    },
+    {
+      "epoch": 2.56,
+      "eval_chemistry_entropy": 0.24356829929351806,
+      "eval_chemistry_loss": 1.449495792388916,
+      "eval_chemistry_mean_token_accuracy": 0.7762749667167663,
+      "eval_chemistry_num_tokens": 45803783.0,
+      "eval_chemistry_runtime": 34.6626,
+      "eval_chemistry_samples_per_second": 14.425,
+      "eval_chemistry_steps_per_second": 3.606,
+      "step": 1600
+    },
+    {
+      "epoch": 2.56,
+      "eval_math_entropy": 0.18325595593452454,
+      "eval_math_loss": 1.2849191427230835,
+      "eval_math_mean_token_accuracy": 0.8152834696769714,
+      "eval_math_num_tokens": 45803783.0,
+      "eval_math_runtime": 35.702,
+      "eval_math_samples_per_second": 14.005,
+      "eval_math_steps_per_second": 3.501,
+      "step": 1600
+    },
+    {
+      "entropy": 0.05134590875823051,
+      "epoch": 2.576,
+      "grad_norm": 0.7265625,
+      "learning_rate": 7.92e-06,
+      "loss": 0.0345,
+      "mean_token_accuracy": 0.9896847944706678,
+      "num_tokens": 46090713.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.05221337205730379,
+      "epoch": 2.592,
+      "grad_norm": 0.7109375,
+      "learning_rate": 7.831111111111111e-06,
+      "loss": 0.0344,
+      "mean_token_accuracy": 0.9897220823913813,
+      "num_tokens": 46384084.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.0540282168192789,
+      "epoch": 2.608,
+      "grad_norm": 0.79296875,
+      "learning_rate": 7.742222222222223e-06,
+      "loss": 0.0349,
+      "mean_token_accuracy": 0.9895514253526926,
+      "num_tokens": 46667552.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.05048422114923597,
+      "epoch": 2.624,
+      "grad_norm": 0.70703125,
+      "learning_rate": 7.653333333333333e-06,
+      "loss": 0.0331,
+      "mean_token_accuracy": 0.989990346878767,
+      "num_tokens": 46961124.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.05211569468956441,
+      "epoch": 2.64,
+      "grad_norm": 0.6796875,
+      "learning_rate": 7.564444444444446e-06,
+      "loss": 0.0348,
+      "mean_token_accuracy": 0.9895898640155792,
+      "num_tokens": 47245326.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.05267454255372286,
+      "epoch": 2.656,
+      "grad_norm": 0.68359375,
+      "learning_rate": 7.475555555555556e-06,
+      "loss": 0.0348,
+      "mean_token_accuracy": 0.989424579590559,
+      "num_tokens": 47527347.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.05313762115547434,
+      "epoch": 2.672,
+      "grad_norm": 0.95703125,
+      "learning_rate": 7.386666666666667e-06,
+      "loss": 0.0354,
+      "mean_token_accuracy": 0.9894337601959705,
+      "num_tokens": 47808167.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.05315034419763833,
+      "epoch": 2.6879999999999997,
+      "grad_norm": 0.62890625,
+      "learning_rate": 7.297777777777778e-06,
+      "loss": 0.0349,
+      "mean_token_accuracy": 0.9893308505415916,
+      "num_tokens": 48092035.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.050815221259836105,
+      "epoch": 2.7039999999999997,
+      "grad_norm": 0.73828125,
+      "learning_rate": 7.20888888888889e-06,
+      "loss": 0.0335,
+      "mean_token_accuracy": 0.9900240320712328,
+      "num_tokens": 48386231.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.05376783083193004,
+      "epoch": 2.7199999999999998,
+      "grad_norm": 0.76953125,
+      "learning_rate": 7.1200000000000004e-06,
+      "loss": 0.036,
+      "mean_token_accuracy": 0.9892230879515409,
+      "num_tokens": 48663262.0,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "eval_chemistry_entropy": 0.24340871345996856,
+      "eval_chemistry_loss": 1.45062255859375,
+      "eval_chemistry_mean_token_accuracy": 0.7765913362503052,
+      "eval_chemistry_num_tokens": 48663262.0,
+      "eval_chemistry_runtime": 34.6595,
+      "eval_chemistry_samples_per_second": 14.426,
+      "eval_chemistry_steps_per_second": 3.607,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "eval_math_entropy": 0.18342817682027818,
+      "eval_math_loss": 1.286737322807312,
+      "eval_math_mean_token_accuracy": 0.8153419017791748,
+      "eval_math_num_tokens": 48663262.0,
+      "eval_math_runtime": 35.697,
+      "eval_math_samples_per_second": 14.007,
+      "eval_math_steps_per_second": 3.502,
+      "step": 1700
+    },
+    {
+      "entropy": 0.05238793781027198,
+      "epoch": 2.7359999999999998,
+      "grad_norm": 0.73046875,
+      "learning_rate": 7.031111111111112e-06,
+      "loss": 0.0343,
+      "mean_token_accuracy": 0.9897935677319765,
+      "num_tokens": 48949456.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.05296056693186983,
+      "epoch": 2.752,
+      "grad_norm": 0.75390625,
+      "learning_rate": 6.942222222222222e-06,
+      "loss": 0.0345,
+      "mean_token_accuracy": 0.9895687259733676,
+      "num_tokens": 49235751.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.051222407980822025,
+      "epoch": 2.768,
+      "grad_norm": 0.62109375,
+      "learning_rate": 6.853333333333334e-06,
+      "loss": 0.0341,
+      "mean_token_accuracy": 0.9898298654705286,
+      "num_tokens": 49524628.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.05459372613113374,
+      "epoch": 2.784,
+      "grad_norm": 0.69140625,
+      "learning_rate": 6.764444444444445e-06,
+      "loss": 0.0358,
+      "mean_token_accuracy": 0.9893500994890928,
+      "num_tokens": 49799604.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.05316627144347876,
+      "epoch": 2.8,
+      "grad_norm": 0.578125,
+      "learning_rate": 6.675555555555556e-06,
+      "loss": 0.0348,
+      "mean_token_accuracy": 0.9896351356059313,
+      "num_tokens": 50079307.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.05220057033002377,
+      "epoch": 2.816,
+      "grad_norm": 0.6796875,
+      "learning_rate": 6.5866666666666666e-06,
+      "loss": 0.0341,
+      "mean_token_accuracy": 0.9897652886807918,
+      "num_tokens": 50372216.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.05233512815320864,
+      "epoch": 2.832,
+      "grad_norm": 0.6875,
+      "learning_rate": 6.497777777777779e-06,
+      "loss": 0.0343,
+      "mean_token_accuracy": 0.9895316615700722,
+      "num_tokens": 50657211.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.0521509436191991,
+      "epoch": 2.848,
+      "grad_norm": 0.640625,
+      "learning_rate": 6.408888888888889e-06,
+      "loss": 0.0341,
+      "mean_token_accuracy": 0.989656625688076,
+      "num_tokens": 50946503.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.05292182555422187,
+      "epoch": 2.864,
+      "grad_norm": 0.81640625,
+      "learning_rate": 6.3200000000000005e-06,
+      "loss": 0.0348,
+      "mean_token_accuracy": 0.9893360774964094,
+      "num_tokens": 51229298.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.05223811571486294,
+      "epoch": 2.88,
+      "grad_norm": 0.57421875,
+      "learning_rate": 6.231111111111111e-06,
+      "loss": 0.0348,
+      "mean_token_accuracy": 0.989601906761527,
+      "num_tokens": 51509622.0,
+      "step": 1800
+    },
+    {
+      "epoch": 2.88,
+      "eval_chemistry_entropy": 0.24320586144924164,
+      "eval_chemistry_loss": 1.4516633749008179,
+      "eval_chemistry_mean_token_accuracy": 0.776555242061615,
+      "eval_chemistry_num_tokens": 51509622.0,
+      "eval_chemistry_runtime": 34.6698,
+      "eval_chemistry_samples_per_second": 14.422,
+      "eval_chemistry_steps_per_second": 3.605,
+      "step": 1800
+    },
+    {
+      "epoch": 2.88,
+      "eval_math_entropy": 0.18338554829359055,
+      "eval_math_loss": 1.28714120388031,
+      "eval_math_mean_token_accuracy": 0.8153921165466309,
+      "eval_math_num_tokens": 51509622.0,
+      "eval_math_runtime": 35.7007,
+      "eval_math_samples_per_second": 14.005,
+      "eval_math_steps_per_second": 3.501,
+      "step": 1800
+    },
+    {
+      "entropy": 0.053149703633971514,
+      "epoch": 2.896,
+      "grad_norm": 0.56640625,
+      "learning_rate": 6.142222222222223e-06,
+      "loss": 0.0348,
+      "mean_token_accuracy": 0.9895217914134264,
+      "num_tokens": 51790856.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.05247251965338364,
+      "epoch": 2.912,
+      "grad_norm": 0.67578125,
+      "learning_rate": 6.0533333333333335e-06,
+      "loss": 0.0344,
+      "mean_token_accuracy": 0.9898622412234545,
+      "num_tokens": 52074393.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.05151027952088043,
+      "epoch": 2.928,
+      "grad_norm": 0.76953125,
+      "learning_rate": 5.964444444444445e-06,
+      "loss": 0.034,
+      "mean_token_accuracy": 0.9896991837769746,
+      "num_tokens": 52366467.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.051106683211401105,
+      "epoch": 2.944,
+      "grad_norm": 0.56640625,
+      "learning_rate": 5.875555555555556e-06,
+      "loss": 0.0341,
+      "mean_token_accuracy": 0.9898365925997495,
+      "num_tokens": 52657863.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.0515750922029838,
+      "epoch": 2.96,
+      "grad_norm": 0.7734375,
+      "learning_rate": 5.7866666666666674e-06,
+      "loss": 0.0343,
+      "mean_token_accuracy": 0.9897553384304046,
+      "num_tokens": 52947183.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.05394409007858485,
+      "epoch": 2.976,
+      "grad_norm": 0.734375,
+      "learning_rate": 5.697777777777779e-06,
+      "loss": 0.035,
+      "mean_token_accuracy": 0.989257312566042,
+      "num_tokens": 53226984.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.051608621410559864,
+      "epoch": 2.992,
+      "grad_norm": 0.66796875,
+      "learning_rate": 5.608888888888889e-06,
+      "loss": 0.0339,
+      "mean_token_accuracy": 0.9897487934678793,
+      "num_tokens": 53515204.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.050469014700502156,
+      "epoch": 3.008,
+      "grad_norm": 0.30078125,
+      "learning_rate": 5.5200000000000005e-06,
+      "loss": 0.0328,
+      "mean_token_accuracy": 0.9901252970099449,
+      "num_tokens": 53800592.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.04760575578548014,
+      "epoch": 3.024,
+      "grad_norm": 0.263671875,
+      "learning_rate": 5.431111111111112e-06,
+      "loss": 0.0301,
+      "mean_token_accuracy": 0.9910080127418042,
+      "num_tokens": 54092017.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.04652785587823018,
+      "epoch": 3.04,
+      "grad_norm": 0.279296875,
+      "learning_rate": 5.342222222222223e-06,
+      "loss": 0.0301,
+      "mean_token_accuracy": 0.9910020593553781,
+      "num_tokens": 54380158.0,
+      "step": 1900
+    },
+    {
+      "epoch": 3.04,
+      "eval_chemistry_entropy": 0.24179766488075255,
+      "eval_chemistry_loss": 1.4621965885162354,
+      "eval_chemistry_mean_token_accuracy": 0.7764029130935669,
+      "eval_chemistry_num_tokens": 54380158.0,
+      "eval_chemistry_runtime": 34.6754,
+      "eval_chemistry_samples_per_second": 14.419,
+      "eval_chemistry_steps_per_second": 3.605,
+      "step": 1900
+    },
+    {
+      "epoch": 3.04,
+      "eval_math_entropy": 0.18078194934129715,
+      "eval_math_loss": 1.2997863292694092,
+      "eval_math_mean_token_accuracy": 0.8158343658447266,
+      "eval_math_num_tokens": 54380158.0,
+      "eval_math_runtime": 35.7053,
+      "eval_math_samples_per_second": 14.004,
+      "eval_math_steps_per_second": 3.501,
+      "step": 1900
+    },
+    {
+      "entropy": 0.04787755131255835,
+      "epoch": 3.056,
+      "grad_norm": 0.3671875,
+      "learning_rate": 5.2533333333333336e-06,
+      "loss": 0.0313,
+      "mean_token_accuracy": 0.9905039332807064,
+      "num_tokens": 54658093.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.04592872656648979,
+      "epoch": 3.072,
+      "grad_norm": 0.30859375,
+      "learning_rate": 5.164444444444445e-06,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9909734159708024,
+      "num_tokens": 54949938.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.04627904891967773,
+      "epoch": 3.088,
+      "grad_norm": 0.33984375,
+      "learning_rate": 5.075555555555556e-06,
+      "loss": 0.0305,
+      "mean_token_accuracy": 0.9907639529556036,
+      "num_tokens": 55234643.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.047028734674677254,
+      "epoch": 3.104,
+      "grad_norm": 0.3828125,
+      "learning_rate": 4.986666666666667e-06,
+      "loss": 0.0307,
+      "mean_token_accuracy": 0.9904829286038875,
+      "num_tokens": 55519679.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.04583920445293188,
+      "epoch": 3.12,
+      "grad_norm": 0.3203125,
+      "learning_rate": 4.897777777777778e-06,
+      "loss": 0.0297,
+      "mean_token_accuracy": 0.9909133236855269,
+      "num_tokens": 55812397.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.046013057266827674,
+      "epoch": 3.136,
+      "grad_norm": 0.94140625,
+      "learning_rate": 4.808888888888889e-06,
+      "loss": 0.03,
+      "mean_token_accuracy": 0.9908631198108196,
+      "num_tokens": 56101550.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.04605525400256738,
+      "epoch": 3.152,
+      "grad_norm": 0.384765625,
+      "learning_rate": 4.7200000000000005e-06,
+      "loss": 0.0304,
+      "mean_token_accuracy": 0.9907284393906594,
+      "num_tokens": 56387662.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.045735701138619335,
+      "epoch": 3.168,
+      "grad_norm": 0.369140625,
+      "learning_rate": 4.631111111111111e-06,
+      "loss": 0.0305,
+      "mean_token_accuracy": 0.990792403742671,
+      "num_tokens": 56673426.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.04651033441768959,
+      "epoch": 3.184,
+      "grad_norm": 0.408203125,
+      "learning_rate": 4.542222222222223e-06,
+      "loss": 0.0308,
+      "mean_token_accuracy": 0.9905624877661466,
+      "num_tokens": 56957044.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.046006057260092346,
+      "epoch": 3.2,
+      "grad_norm": 0.390625,
+      "learning_rate": 4.453333333333334e-06,
+      "loss": 0.0298,
+      "mean_token_accuracy": 0.990779859945178,
+      "num_tokens": 57250097.0,
+      "step": 2000
+    },
+    {
+      "epoch": 3.2,
+      "eval_chemistry_entropy": 0.24037811040878296,
+      "eval_chemistry_loss": 1.4743051528930664,
+      "eval_chemistry_mean_token_accuracy": 0.7761513013839721,
+      "eval_chemistry_num_tokens": 57250097.0,
+      "eval_chemistry_runtime": 34.67,
+      "eval_chemistry_samples_per_second": 14.422,
+      "eval_chemistry_steps_per_second": 3.605,
+      "step": 2000
+    },
+    {
+      "epoch": 3.2,
+      "eval_math_entropy": 0.17890368604660034,
+      "eval_math_loss": 1.3124377727508545,
+      "eval_math_mean_token_accuracy": 0.8159590802192688,
+      "eval_math_num_tokens": 57250097.0,
+      "eval_math_runtime": 35.7969,
+      "eval_math_samples_per_second": 13.968,
+      "eval_math_steps_per_second": 3.492,
+      "step": 2000
+    },
+    {
+      "entropy": 0.04777187714353204,
+      "epoch": 3.216,
+      "grad_norm": 0.376953125,
+      "learning_rate": 4.364444444444445e-06,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.9904848050326109,
+      "num_tokens": 57532026.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.048222763324156405,
+      "epoch": 3.232,
+      "grad_norm": 0.359375,
+      "learning_rate": 4.275555555555556e-06,
+      "loss": 0.031,
+      "mean_token_accuracy": 0.9903705444186925,
+      "num_tokens": 57814853.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.047401591949164865,
+      "epoch": 3.248,
+      "grad_norm": 0.3515625,
+      "learning_rate": 4.1866666666666675e-06,
+      "loss": 0.0312,
+      "mean_token_accuracy": 0.990451006963849,
+      "num_tokens": 58094190.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.04756687419721857,
+      "epoch": 3.2640000000000002,
+      "grad_norm": 0.400390625,
+      "learning_rate": 4.097777777777778e-06,
+      "loss": 0.0312,
+      "mean_token_accuracy": 0.9902511730790138,
+      "num_tokens": 58374378.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.0473020819015801,
+      "epoch": 3.2800000000000002,
+      "grad_norm": 0.33203125,
+      "learning_rate": 4.008888888888889e-06,
+      "loss": 0.0317,
+      "mean_token_accuracy": 0.9904257394373417,
+      "num_tokens": 58652789.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.048701566038653256,
+      "epoch": 3.296,
+      "grad_norm": 0.35546875,
+      "learning_rate": 3.920000000000001e-06,
+      "loss": 0.0315,
+      "mean_token_accuracy": 0.9902492504566908,
+      "num_tokens": 58930014.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.04785212180577218,
+      "epoch": 3.312,
+      "grad_norm": 0.43359375,
+      "learning_rate": 3.831111111111112e-06,
+      "loss": 0.0311,
+      "mean_token_accuracy": 0.990452916175127,
+      "num_tokens": 59210270.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.046990638901479545,
+      "epoch": 3.328,
+      "grad_norm": 0.35546875,
+      "learning_rate": 3.7422222222222228e-06,
+      "loss": 0.0306,
+      "mean_token_accuracy": 0.9905503377318382,
+      "num_tokens": 59496408.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.047202743543311956,
+      "epoch": 3.344,
+      "grad_norm": 0.3515625,
+      "learning_rate": 3.6533333333333336e-06,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.9904990542680026,
+      "num_tokens": 59779722.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.047031561320181936,
+      "epoch": 3.36,
+      "grad_norm": 0.41796875,
+      "learning_rate": 3.564444444444445e-06,
+      "loss": 0.031,
+      "mean_token_accuracy": 0.9906260650604963,
+      "num_tokens": 60061562.0,
+      "step": 2100
+    },
+    {
+      "epoch": 3.36,
+      "eval_chemistry_entropy": 0.23970958662033082,
+      "eval_chemistry_loss": 1.4770554304122925,
+      "eval_chemistry_mean_token_accuracy": 0.7761442346572875,
+      "eval_chemistry_num_tokens": 60061562.0,
+      "eval_chemistry_runtime": 34.7404,
+      "eval_chemistry_samples_per_second": 14.392,
+      "eval_chemistry_steps_per_second": 3.598,
+      "step": 2100
+    },
+    {
+      "epoch": 3.36,
+      "eval_math_entropy": 0.17825451320409774,
+      "eval_math_loss": 1.3161224126815796,
+      "eval_math_mean_token_accuracy": 0.8157565836906433,
+      "eval_math_num_tokens": 60061562.0,
+      "eval_math_runtime": 35.7593,
+      "eval_math_samples_per_second": 13.982,
+      "eval_math_steps_per_second": 3.496,
+      "step": 2100
+    },
+    {
+      "entropy": 0.04683599259005859,
+      "epoch": 3.376,
+      "grad_norm": 0.39453125,
+      "learning_rate": 3.475555555555556e-06,
+      "loss": 0.0308,
+      "mean_token_accuracy": 0.9906395886093378,
+      "num_tokens": 60346574.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.045522975514177236,
+      "epoch": 3.392,
+      "grad_norm": 0.349609375,
+      "learning_rate": 3.386666666666667e-06,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9908528294414282,
+      "num_tokens": 60636820.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.04630873181158677,
+      "epoch": 3.408,
+      "grad_norm": 0.34375,
+      "learning_rate": 3.297777777777778e-06,
+      "loss": 0.0307,
+      "mean_token_accuracy": 0.9906216621398926,
+      "num_tokens": 60922033.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.046549882076215,
+      "epoch": 3.424,
+      "grad_norm": 0.345703125,
+      "learning_rate": 3.2088888888888893e-06,
+      "loss": 0.0304,
+      "mean_token_accuracy": 0.990608799085021,
+      "num_tokens": 61209156.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.046345280681271105,
+      "epoch": 3.44,
+      "grad_norm": 0.376953125,
+      "learning_rate": 3.12e-06,
+      "loss": 0.031,
+      "mean_token_accuracy": 0.9905180517584086,
+      "num_tokens": 61491228.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.04777171619934961,
+      "epoch": 3.456,
+      "grad_norm": 0.361328125,
+      "learning_rate": 3.0311111111111115e-06,
+      "loss": 0.0311,
+      "mean_token_accuracy": 0.9902653150260449,
+      "num_tokens": 61773033.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.046266474935691804,
+      "epoch": 3.472,
+      "grad_norm": 0.3203125,
+      "learning_rate": 2.9422222222222224e-06,
+      "loss": 0.0303,
+      "mean_token_accuracy": 0.9906618434935808,
+      "num_tokens": 62061802.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.046397614572197196,
+      "epoch": 3.488,
+      "grad_norm": 0.333984375,
+      "learning_rate": 2.8533333333333337e-06,
+      "loss": 0.0306,
+      "mean_token_accuracy": 0.9905564673244953,
+      "num_tokens": 62347490.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.046488759107887745,
+      "epoch": 3.504,
+      "grad_norm": 0.380859375,
+      "learning_rate": 2.7644444444444446e-06,
+      "loss": 0.0302,
+      "mean_token_accuracy": 0.990698029473424,
+      "num_tokens": 62636811.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.04671492046909407,
+      "epoch": 3.52,
+      "grad_norm": 0.3828125,
+      "learning_rate": 2.675555555555556e-06,
+      "loss": 0.0303,
+      "mean_token_accuracy": 0.9906057629734277,
+      "num_tokens": 62928536.0,
+      "step": 2200
+    },
+    {
+      "epoch": 3.52,
+      "eval_chemistry_entropy": 0.2400316128730774,
+      "eval_chemistry_loss": 1.4769163131713867,
+      "eval_chemistry_mean_token_accuracy": 0.7761477336883545,
+      "eval_chemistry_num_tokens": 62928536.0,
+      "eval_chemistry_runtime": 34.7677,
+      "eval_chemistry_samples_per_second": 14.381,
+      "eval_chemistry_steps_per_second": 3.595,
+      "step": 2200
+    },
+    {
+      "epoch": 3.52,
+      "eval_math_entropy": 0.1784315680861473,
+      "eval_math_loss": 1.3170243501663208,
+      "eval_math_mean_token_accuracy": 0.8158855767250061,
+      "eval_math_num_tokens": 62928536.0,
+      "eval_math_runtime": 35.7757,
+      "eval_math_samples_per_second": 13.976,
+      "eval_math_steps_per_second": 3.494,
+      "step": 2200
+    },
+    {
+      "entropy": 0.0479715758934617,
+      "epoch": 3.536,
+      "grad_norm": 0.357421875,
+      "learning_rate": 2.5866666666666667e-06,
+      "loss": 0.0307,
+      "mean_token_accuracy": 0.9903990484774112,
+      "num_tokens": 63212496.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.045799953246023505,
+      "epoch": 3.552,
+      "grad_norm": 0.63671875,
+      "learning_rate": 2.497777777777778e-06,
+      "loss": 0.0298,
+      "mean_token_accuracy": 0.9906595416367054,
+      "num_tokens": 63506921.0,
+      "step": 2220
+    },
+    {
+      "entropy": 0.045416490186471495,
+      "epoch": 3.568,
+      "grad_norm": 0.32421875,
+      "learning_rate": 2.4088888888888894e-06,
+      "loss": 0.0297,
+      "mean_token_accuracy": 0.9906731244176626,
+      "num_tokens": 63803208.0,
+      "step": 2230
+    },
+    {
+      "entropy": 0.04842924257973209,
+      "epoch": 3.584,
+      "grad_norm": 0.345703125,
+      "learning_rate": 2.3200000000000002e-06,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.990395812317729,
+      "num_tokens": 64084867.0,
+      "step": 2240
+    },
+    {
+      "entropy": 0.04583321294048801,
+      "epoch": 3.6,
+      "grad_norm": 0.36328125,
+      "learning_rate": 2.2311111111111115e-06,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9907201394438744,
+      "num_tokens": 64377046.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.046151836856734005,
+      "epoch": 3.616,
+      "grad_norm": 0.314453125,
+      "learning_rate": 2.1422222222222224e-06,
+      "loss": 0.0306,
+      "mean_token_accuracy": 0.9905206307768821,
+      "num_tokens": 64664301.0,
+      "step": 2260
+    },
+    {
+      "entropy": 0.04715829082997516,
+      "epoch": 3.632,
+      "grad_norm": 0.359375,
+      "learning_rate": 2.0533333333333337e-06,
+      "loss": 0.0311,
+      "mean_token_accuracy": 0.9905262131243944,
+      "num_tokens": 64946172.0,
+      "step": 2270
+    },
+    {
+      "entropy": 0.04714928624453023,
+      "epoch": 3.648,
+      "grad_norm": 0.318359375,
+      "learning_rate": 1.9644444444444446e-06,
+      "loss": 0.0308,
+      "mean_token_accuracy": 0.9905759919434786,
+      "num_tokens": 65230709.0,
+      "step": 2280
+    },
+    {
+      "entropy": 0.04715377010870725,
+      "epoch": 3.664,
+      "grad_norm": 0.369140625,
+      "learning_rate": 1.8755555555555557e-06,
+      "loss": 0.031,
+      "mean_token_accuracy": 0.9906590826809406,
+      "num_tokens": 65515738.0,
+      "step": 2290
+    },
+    {
+      "entropy": 0.046317026717588305,
+      "epoch": 3.68,
+      "grad_norm": 0.361328125,
+      "learning_rate": 1.7866666666666668e-06,
+      "loss": 0.0306,
+      "mean_token_accuracy": 0.9905350394546986,
+      "num_tokens": 65802680.0,
+      "step": 2300
+    },
+    {
+      "epoch": 3.68,
+      "eval_chemistry_entropy": 0.23951965999603272,
+      "eval_chemistry_loss": 1.4780032634735107,
+      "eval_chemistry_mean_token_accuracy": 0.7760953726768494,
+      "eval_chemistry_num_tokens": 65802680.0,
+      "eval_chemistry_runtime": 34.7483,
+      "eval_chemistry_samples_per_second": 14.389,
+      "eval_chemistry_steps_per_second": 3.597,
+      "step": 2300
+    },
+    {
+      "epoch": 3.68,
+      "eval_math_entropy": 0.17792051881551743,
+      "eval_math_loss": 1.3179256916046143,
+      "eval_math_mean_token_accuracy": 0.8158724884986878,
+      "eval_math_num_tokens": 65802680.0,
+      "eval_math_runtime": 35.7661,
+      "eval_math_samples_per_second": 13.98,
+      "eval_math_steps_per_second": 3.495,
+      "step": 2300
+    },
+    {
+      "entropy": 0.04655458262423053,
+      "epoch": 3.6959999999999997,
+      "grad_norm": 0.3203125,
+      "learning_rate": 1.6977777777777779e-06,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.9906194817274809,
+      "num_tokens": 66089104.0,
+      "step": 2310
+    },
+    {
+      "entropy": 0.047844100883230566,
+      "epoch": 3.7119999999999997,
+      "grad_norm": 0.337890625,
+      "learning_rate": 1.608888888888889e-06,
+      "loss": 0.0312,
+      "mean_token_accuracy": 0.9902839906513691,
+      "num_tokens": 66369895.0,
+      "step": 2320
+    },
+    {
+      "entropy": 0.04698739227605984,
+      "epoch": 3.7279999999999998,
+      "grad_norm": 0.380859375,
+      "learning_rate": 1.52e-06,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.9904122680425644,
+      "num_tokens": 66653165.0,
+      "step": 2330
+    },
+    {
+      "entropy": 0.04657714320346713,
+      "epoch": 3.7439999999999998,
+      "grad_norm": 0.400390625,
+      "learning_rate": 1.4311111111111111e-06,
+      "loss": 0.0304,
+      "mean_token_accuracy": 0.9906352493911982,
+      "num_tokens": 66941476.0,
+      "step": 2340
+    },
+    {
+      "entropy": 0.04702814534539357,
+      "epoch": 3.76,
+      "grad_norm": 0.3515625,
+      "learning_rate": 1.3422222222222222e-06,
+      "loss": 0.0308,
+      "mean_token_accuracy": 0.9905426375567913,
+      "num_tokens": 67225380.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.04688780263531953,
+      "epoch": 3.776,
+      "grad_norm": 0.3515625,
+      "learning_rate": 1.2533333333333333e-06,
+      "loss": 0.0302,
+      "mean_token_accuracy": 0.9906635366380214,
+      "num_tokens": 67516641.0,
+      "step": 2360
+    },
+    {
+      "entropy": 0.04620065385242924,
+      "epoch": 3.792,
+      "grad_norm": 0.37890625,
+      "learning_rate": 1.1644444444444446e-06,
+      "loss": 0.0302,
+      "mean_token_accuracy": 0.9907540038228035,
+      "num_tokens": 67808009.0,
+      "step": 2370
+    },
+    {
+      "entropy": 0.0462893542018719,
+      "epoch": 3.808,
+      "grad_norm": 0.412109375,
+      "learning_rate": 1.0755555555555557e-06,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.9905878853052854,
+      "num_tokens": 68092996.0,
+      "step": 2380
+    },
+    {
+      "entropy": 0.04656721533974632,
+      "epoch": 3.824,
+      "grad_norm": 0.34375,
+      "learning_rate": 9.866666666666668e-07,
+      "loss": 0.0305,
+      "mean_token_accuracy": 0.990664380043745,
+      "num_tokens": 68380631.0,
+      "step": 2390
+    },
+    {
+      "entropy": 0.047634795191697776,
+      "epoch": 3.84,
+      "grad_norm": 0.34375,
+      "learning_rate": 8.977777777777778e-07,
+      "loss": 0.0307,
+      "mean_token_accuracy": 0.9904828619211912,
+      "num_tokens": 68666505.0,
+      "step": 2400
+    },
+    {
+      "epoch": 3.84,
+      "eval_chemistry_entropy": 0.23970548629760743,
+      "eval_chemistry_loss": 1.4779794216156006,
+      "eval_chemistry_mean_token_accuracy": 0.7760384821891785,
+      "eval_chemistry_num_tokens": 68666505.0,
+      "eval_chemistry_runtime": 34.7298,
+      "eval_chemistry_samples_per_second": 14.397,
+      "eval_chemistry_steps_per_second": 3.599,
+      "step": 2400
+    },
+    {
+      "epoch": 3.84,
+      "eval_math_entropy": 0.17805879640579222,
+      "eval_math_loss": 1.318205714225769,
+      "eval_math_mean_token_accuracy": 0.8158385133743287,
+      "eval_math_num_tokens": 68666505.0,
+      "eval_math_runtime": 35.7836,
+      "eval_math_samples_per_second": 13.973,
+      "eval_math_steps_per_second": 3.493,
+      "step": 2400
+    },
+    {
+      "entropy": 0.046251483936794104,
+      "epoch": 3.856,
+      "grad_norm": 0.353515625,
+      "learning_rate": 8.088888888888889e-07,
+      "loss": 0.0308,
+      "mean_token_accuracy": 0.9906993404030799,
+      "num_tokens": 68951258.0,
+      "step": 2410
+    },
+    {
+      "entropy": 0.0466153520043008,
+      "epoch": 3.872,
+      "grad_norm": 0.333984375,
+      "learning_rate": 7.2e-07,
+      "loss": 0.0302,
+      "mean_token_accuracy": 0.9907167844474316,
+      "num_tokens": 69243705.0,
+      "step": 2420
+    },
+    {
+      "entropy": 0.04767106171930209,
+      "epoch": 3.888,
+      "grad_norm": 0.345703125,
+      "learning_rate": 6.311111111111112e-07,
+      "loss": 0.0307,
+      "mean_token_accuracy": 0.9904027633368969,
+      "num_tokens": 69529393.0,
+      "step": 2430
+    },
+    {
+      "entropy": 0.04571640334324911,
+      "epoch": 3.904,
+      "grad_norm": 0.447265625,
+      "learning_rate": 5.422222222222223e-07,
+      "loss": 0.0303,
+      "mean_token_accuracy": 0.9909489165991545,
+      "num_tokens": 69820950.0,
+      "step": 2440
+    },
+    {
+      "entropy": 0.04731314091477543,
+      "epoch": 3.92,
+      "grad_norm": 0.36328125,
+      "learning_rate": 4.533333333333334e-07,
+      "loss": 0.0314,
+      "mean_token_accuracy": 0.9903879094868898,
+      "num_tokens": 70097736.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.046360219467896965,
+      "epoch": 3.936,
+      "grad_norm": 0.392578125,
+      "learning_rate": 3.644444444444445e-07,
+      "loss": 0.0297,
+      "mean_token_accuracy": 0.9907674510031939,
+      "num_tokens": 70395123.0,
+      "step": 2460
+    },
+    {
+      "entropy": 0.04712863607564941,
+      "epoch": 3.952,
+      "grad_norm": 0.3046875,
+      "learning_rate": 2.7555555555555555e-07,
+      "loss": 0.031,
+      "mean_token_accuracy": 0.9905471954494715,
+      "num_tokens": 70678153.0,
+      "step": 2470
+    },
+    {
+      "entropy": 0.04746697030495852,
+      "epoch": 3.968,
+      "grad_norm": 0.34765625,
+      "learning_rate": 1.866666666666667e-07,
+      "loss": 0.031,
+      "mean_token_accuracy": 0.9904777407646179,
+      "num_tokens": 70963030.0,
+      "step": 2480
+    },
+    {
+      "entropy": 0.04502330997493118,
+      "epoch": 3.984,
+      "grad_norm": 0.32421875,
+      "learning_rate": 9.777777777777779e-08,
+      "loss": 0.0294,
+      "mean_token_accuracy": 0.991044569760561,
+      "num_tokens": 71264525.0,
+      "step": 2490
+    },
+    {
+      "entropy": 0.048340489657130094,
+      "epoch": 4.0,
+      "grad_norm": 0.361328125,
+      "learning_rate": 8.88888888888889e-09,
+      "loss": 0.0311,
+      "mean_token_accuracy": 0.9902615990489722,
+      "num_tokens": 71545696.0,
+      "step": 2500
+    },
+    {
+      "epoch": 4.0,
+      "eval_chemistry_entropy": 0.23966557669639588,
+      "eval_chemistry_loss": 1.4780405759811401,
+      "eval_chemistry_mean_token_accuracy": 0.7761346774101258,
+      "eval_chemistry_num_tokens": 71545696.0,
+      "eval_chemistry_runtime": 34.7178,
+      "eval_chemistry_samples_per_second": 14.402,
+      "eval_chemistry_steps_per_second": 3.6,
+      "step": 2500
+    },
+    {
+      "epoch": 4.0,
+      "eval_math_entropy": 0.17805184537172317,
+      "eval_math_loss": 1.3182107210159302,
+      "eval_math_mean_token_accuracy": 0.8157890677452088,
+      "eval_math_num_tokens": 71545696.0,
+      "eval_math_runtime": 35.4787,
+      "eval_math_samples_per_second": 14.093,
+      "eval_math_steps_per_second": 3.523,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.428120947106271e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb2f64a043856303a7c1fa64d5fad3c4b1c34f11ca082413bb2449fe0d779655
+size 6609