Upload folder using huggingface_hub

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-2000/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-2000/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-2000/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-2000/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-2000/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-2000/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4d8c15ddfedd920e4c7c932ac371b6f1b2fdc700f2d993b86ed3de38d0e63f6
+size 4962221464

checkpoint-2000/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:721a82ddf33b9dc3306d803624257f5585e18d5c6d833fcf6d74ac3e719e9bf6
+size 3670322200

checkpoint-2000/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f1d527b1d71e8b80bf87cd2230feef7167d643af257479bbea65a64b1c0caf5
+size 7927918951

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55295ab5d6868e8fe7d2a4ad01c244b707441ee5076c05d21ec765761ced364f
+size 14645

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d388275ed788b731428fee5fd20996cc5607181219fa17c40d7a13ba8962f4f
+size 1465

checkpoint-2000/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-2000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-2000/trainer_state.json

@@ -0,0 +1,2914 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.2,
+  "eval_steps": 100,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.1658192440867423,
+      "epoch": 0.016,
+      "grad_norm": 23.125,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 2.1704,
+      "mean_token_accuracy": 0.627803846821189,
+      "num_tokens": 282849.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2136684671044349,
+      "epoch": 0.032,
+      "grad_norm": 21.875,
+      "learning_rate": 6.333333333333334e-07,
+      "loss": 2.2125,
+      "mean_token_accuracy": 0.6216072155162692,
+      "num_tokens": 561726.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2173424981534482,
+      "epoch": 0.048,
+      "grad_norm": 19.875,
+      "learning_rate": 9.666666666666668e-07,
+      "loss": 2.1138,
+      "mean_token_accuracy": 0.633067087456584,
+      "num_tokens": 842092.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.3084764428436757,
+      "epoch": 0.064,
+      "grad_norm": 14.25,
+      "learning_rate": 1.3e-06,
+      "loss": 2.0259,
+      "mean_token_accuracy": 0.6363544659689069,
+      "num_tokens": 1133349.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.3902073439210654,
+      "epoch": 0.08,
+      "grad_norm": 11.5625,
+      "learning_rate": 1.6333333333333335e-06,
+      "loss": 1.869,
+      "mean_token_accuracy": 0.6566519955173135,
+      "num_tokens": 1422340.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.4069613303989172,
+      "epoch": 0.096,
+      "grad_norm": 7.9375,
+      "learning_rate": 1.9666666666666668e-06,
+      "loss": 1.6593,
+      "mean_token_accuracy": 0.6801599662750959,
+      "num_tokens": 1711821.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.3535694684833288,
+      "epoch": 0.112,
+      "grad_norm": 5.34375,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 1.4913,
+      "mean_token_accuracy": 0.7020411744713784,
+      "num_tokens": 1991855.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.2954536855220795,
+      "epoch": 0.128,
+      "grad_norm": 4.03125,
+      "learning_rate": 2.6333333333333332e-06,
+      "loss": 1.3759,
+      "mean_token_accuracy": 0.7201175257563591,
+      "num_tokens": 2288720.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.166874735802412,
+      "epoch": 0.144,
+      "grad_norm": 3.453125,
+      "learning_rate": 2.9666666666666673e-06,
+      "loss": 1.2892,
+      "mean_token_accuracy": 0.7321068961173296,
+      "num_tokens": 2564352.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.078620458394289,
+      "epoch": 0.16,
+      "grad_norm": 3.609375,
+      "learning_rate": 3.3000000000000006e-06,
+      "loss": 1.2108,
+      "mean_token_accuracy": 0.7476159445941448,
+      "num_tokens": 2841876.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 3.2154119491577147,
+      "eval_biology_loss": 4.707394123077393,
+      "eval_biology_mean_token_accuracy": 0.3904775631427765,
+      "eval_biology_num_tokens": 2841876.0,
+      "eval_biology_runtime": 27.3859,
+      "eval_biology_samples_per_second": 18.258,
+      "eval_biology_steps_per_second": 4.564,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_chemistry_entropy": 2.048550369262695,
+      "eval_chemistry_loss": 2.626492500305176,
+      "eval_chemistry_mean_token_accuracy": 0.5851920173168182,
+      "eval_chemistry_num_tokens": 2841876.0,
+      "eval_chemistry_runtime": 34.5202,
+      "eval_chemistry_samples_per_second": 14.484,
+      "eval_chemistry_steps_per_second": 3.621,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_math_entropy": 1.0965594487190247,
+      "eval_math_loss": 1.2351130247116089,
+      "eval_math_mean_token_accuracy": 0.7443663082122802,
+      "eval_math_num_tokens": 2841876.0,
+      "eval_math_runtime": 35.5634,
+      "eval_math_samples_per_second": 14.059,
+      "eval_math_steps_per_second": 3.515,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_physics_entropy": 1.5686066508293153,
+      "eval_physics_loss": 1.8286855220794678,
+      "eval_physics_mean_token_accuracy": 0.6651120615005494,
+      "eval_physics_num_tokens": 2841876.0,
+      "eval_physics_runtime": 41.2266,
+      "eval_physics_samples_per_second": 12.128,
+      "eval_physics_steps_per_second": 3.032,
+      "step": 100
+    },
+    {
+      "entropy": 1.0461613606661557,
+      "epoch": 0.176,
+      "grad_norm": 2.46875,
+      "learning_rate": 3.633333333333334e-06,
+      "loss": 1.1606,
+      "mean_token_accuracy": 0.7545761711895466,
+      "num_tokens": 3129237.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.034791724383831,
+      "epoch": 0.192,
+      "grad_norm": 3.5625,
+      "learning_rate": 3.966666666666667e-06,
+      "loss": 1.1654,
+      "mean_token_accuracy": 0.7535115238279104,
+      "num_tokens": 3414875.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.00306654907763,
+      "epoch": 0.208,
+      "grad_norm": 3.296875,
+      "learning_rate": 4.3e-06,
+      "loss": 1.1237,
+      "mean_token_accuracy": 0.7600865386426449,
+      "num_tokens": 3695623.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.9779280100017786,
+      "epoch": 0.224,
+      "grad_norm": 3.484375,
+      "learning_rate": 4.633333333333334e-06,
+      "loss": 1.0908,
+      "mean_token_accuracy": 0.7654250439256429,
+      "num_tokens": 3977666.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.93392180390656,
+      "epoch": 0.24,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.966666666666667e-06,
+      "loss": 1.0463,
+      "mean_token_accuracy": 0.7724226381629705,
+      "num_tokens": 4269287.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.9080729607492686,
+      "epoch": 0.256,
+      "grad_norm": 2.984375,
+      "learning_rate": 5.300000000000001e-06,
+      "loss": 1.0141,
+      "mean_token_accuracy": 0.7763364154845476,
+      "num_tokens": 4558632.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.9103192266076803,
+      "epoch": 0.272,
+      "grad_norm": 2.40625,
+      "learning_rate": 5.633333333333334e-06,
+      "loss": 1.0179,
+      "mean_token_accuracy": 0.7763371668756008,
+      "num_tokens": 4839270.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.8979891583323478,
+      "epoch": 0.288,
+      "grad_norm": 3.03125,
+      "learning_rate": 5.966666666666667e-06,
+      "loss": 1.0008,
+      "mean_token_accuracy": 0.7794784251600504,
+      "num_tokens": 5125447.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.8931992243975401,
+      "epoch": 0.304,
+      "grad_norm": 2.453125,
+      "learning_rate": 6.300000000000001e-06,
+      "loss": 0.9795,
+      "mean_token_accuracy": 0.7793796982616186,
+      "num_tokens": 5415925.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.8518329776823521,
+      "epoch": 0.32,
+      "grad_norm": 2.3125,
+      "learning_rate": 6.633333333333334e-06,
+      "loss": 0.9272,
+      "mean_token_accuracy": 0.7885121151804924,
+      "num_tokens": 5710178.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 3.132701642036438,
+      "eval_biology_loss": 3.9858410358428955,
+      "eval_biology_mean_token_accuracy": 0.43233425450325014,
+      "eval_biology_num_tokens": 5710178.0,
+      "eval_biology_runtime": 27.3486,
+      "eval_biology_samples_per_second": 18.282,
+      "eval_biology_steps_per_second": 4.571,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_chemistry_entropy": 1.7960433859825133,
+      "eval_chemistry_loss": 2.123067855834961,
+      "eval_chemistry_mean_token_accuracy": 0.6318115179538727,
+      "eval_chemistry_num_tokens": 5710178.0,
+      "eval_chemistry_runtime": 34.5331,
+      "eval_chemistry_samples_per_second": 14.479,
+      "eval_chemistry_steps_per_second": 3.62,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_math_entropy": 0.884596830368042,
+      "eval_math_loss": 0.9804532527923584,
+      "eval_math_mean_token_accuracy": 0.7791947345733643,
+      "eval_math_num_tokens": 5710178.0,
+      "eval_math_runtime": 35.5815,
+      "eval_math_samples_per_second": 14.052,
+      "eval_math_steps_per_second": 3.513,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_physics_entropy": 1.3077924380302428,
+      "eval_physics_loss": 1.4966135025024414,
+      "eval_physics_mean_token_accuracy": 0.7047412896156311,
+      "eval_physics_num_tokens": 5710178.0,
+      "eval_physics_runtime": 41.202,
+      "eval_physics_samples_per_second": 12.135,
+      "eval_physics_steps_per_second": 3.034,
+      "step": 200
+    },
+    {
+      "entropy": 0.8812895350158214,
+      "epoch": 0.336,
+      "grad_norm": 2.53125,
+      "learning_rate": 6.966666666666667e-06,
+      "loss": 0.9375,
+      "mean_token_accuracy": 0.7815663177520037,
+      "num_tokens": 5993843.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.8503613330423831,
+      "epoch": 0.352,
+      "grad_norm": 2.390625,
+      "learning_rate": 7.3e-06,
+      "loss": 0.8846,
+      "mean_token_accuracy": 0.7902019210159779,
+      "num_tokens": 6279137.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.8630260169506073,
+      "epoch": 0.368,
+      "grad_norm": 2.09375,
+      "learning_rate": 7.633333333333334e-06,
+      "loss": 0.8532,
+      "mean_token_accuracy": 0.7891123633831739,
+      "num_tokens": 6562387.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.8541328482329845,
+      "epoch": 0.384,
+      "grad_norm": 2.3125,
+      "learning_rate": 7.966666666666668e-06,
+      "loss": 0.8347,
+      "mean_token_accuracy": 0.7935563068836927,
+      "num_tokens": 6851520.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.8779759708791971,
+      "epoch": 0.4,
+      "grad_norm": 2.640625,
+      "learning_rate": 8.3e-06,
+      "loss": 0.8656,
+      "mean_token_accuracy": 0.7872439008206129,
+      "num_tokens": 7138932.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.8603043548762799,
+      "epoch": 0.416,
+      "grad_norm": 2.171875,
+      "learning_rate": 8.633333333333334e-06,
+      "loss": 0.8317,
+      "mean_token_accuracy": 0.7921371571719646,
+      "num_tokens": 7420720.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.8324572496116162,
+      "epoch": 0.432,
+      "grad_norm": 1.9921875,
+      "learning_rate": 8.966666666666667e-06,
+      "loss": 0.8089,
+      "mean_token_accuracy": 0.7964957885444164,
+      "num_tokens": 7695518.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.8152802981436252,
+      "epoch": 0.448,
+      "grad_norm": 2.484375,
+      "learning_rate": 9.3e-06,
+      "loss": 0.795,
+      "mean_token_accuracy": 0.7969364944845438,
+      "num_tokens": 7983543.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.8216956006363034,
+      "epoch": 0.464,
+      "grad_norm": 2.671875,
+      "learning_rate": 9.633333333333335e-06,
+      "loss": 0.8009,
+      "mean_token_accuracy": 0.7986288439482451,
+      "num_tokens": 8271787.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.8429995454847813,
+      "epoch": 0.48,
+      "grad_norm": 2.296875,
+      "learning_rate": 9.966666666666667e-06,
+      "loss": 0.8108,
+      "mean_token_accuracy": 0.7953756704926491,
+      "num_tokens": 8556170.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 3.1451055603027345,
+      "eval_biology_loss": 3.5734522342681885,
+      "eval_biology_mean_token_accuracy": 0.45723238396644594,
+      "eval_biology_num_tokens": 8556170.0,
+      "eval_biology_runtime": 27.3386,
+      "eval_biology_samples_per_second": 18.289,
+      "eval_biology_steps_per_second": 4.572,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_chemistry_entropy": 1.7087490386962891,
+      "eval_chemistry_loss": 1.8647722005844116,
+      "eval_chemistry_mean_token_accuracy": 0.653634866476059,
+      "eval_chemistry_num_tokens": 8556170.0,
+      "eval_chemistry_runtime": 34.5372,
+      "eval_chemistry_samples_per_second": 14.477,
+      "eval_chemistry_steps_per_second": 3.619,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_math_entropy": 0.822877546787262,
+      "eval_math_loss": 0.8222662806510925,
+      "eval_math_mean_token_accuracy": 0.7941151804924012,
+      "eval_math_num_tokens": 8556170.0,
+      "eval_math_runtime": 35.5822,
+      "eval_math_samples_per_second": 14.052,
+      "eval_math_steps_per_second": 3.513,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_physics_entropy": 1.2233833904266358,
+      "eval_physics_loss": 1.3149102926254272,
+      "eval_physics_mean_token_accuracy": 0.7227814173698426,
+      "eval_physics_num_tokens": 8556170.0,
+      "eval_physics_runtime": 41.203,
+      "eval_physics_samples_per_second": 12.135,
+      "eval_physics_steps_per_second": 3.034,
+      "step": 300
+    },
+    {
+      "entropy": 0.7992334196344018,
+      "epoch": 0.496,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.0300000000000001e-05,
+      "loss": 0.7744,
+      "mean_token_accuracy": 0.8018383506685496,
+      "num_tokens": 8840590.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.8067768542096019,
+      "epoch": 0.512,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.0633333333333334e-05,
+      "loss": 0.7726,
+      "mean_token_accuracy": 0.8011369973421096,
+      "num_tokens": 9131903.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.7787048129364849,
+      "epoch": 0.528,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.0966666666666668e-05,
+      "loss": 0.7553,
+      "mean_token_accuracy": 0.8060367163270712,
+      "num_tokens": 9429519.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.7824771545827389,
+      "epoch": 0.544,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.13e-05,
+      "loss": 0.769,
+      "mean_token_accuracy": 0.8051767688244581,
+      "num_tokens": 9718247.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.8219987377524376,
+      "epoch": 0.56,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.1633333333333334e-05,
+      "loss": 0.7938,
+      "mean_token_accuracy": 0.7962981123477221,
+      "num_tokens": 9991100.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.7830230491235852,
+      "epoch": 0.576,
+      "grad_norm": 2.25,
+      "learning_rate": 1.1966666666666668e-05,
+      "loss": 0.765,
+      "mean_token_accuracy": 0.8018256653100252,
+      "num_tokens": 10276090.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.7815432142466306,
+      "epoch": 0.592,
+      "grad_norm": 1.953125,
+      "learning_rate": 1.23e-05,
+      "loss": 0.7581,
+      "mean_token_accuracy": 0.8042603217065334,
+      "num_tokens": 10566762.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.7898051368072629,
+      "epoch": 0.608,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.2633333333333335e-05,
+      "loss": 0.7704,
+      "mean_token_accuracy": 0.8025864154100418,
+      "num_tokens": 10857002.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.7680038006976246,
+      "epoch": 0.624,
+      "grad_norm": 2.6875,
+      "learning_rate": 1.2966666666666667e-05,
+      "loss": 0.7434,
+      "mean_token_accuracy": 0.8053195260465145,
+      "num_tokens": 11147326.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.7495566725730896,
+      "epoch": 0.64,
+      "grad_norm": 1.7890625,
+      "learning_rate": 1.3300000000000001e-05,
+      "loss": 0.7341,
+      "mean_token_accuracy": 0.80861243493855,
+      "num_tokens": 11437710.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 3.0337429494857786,
+      "eval_biology_loss": 3.416069746017456,
+      "eval_biology_mean_token_accuracy": 0.4705715277194977,
+      "eval_biology_num_tokens": 11437710.0,
+      "eval_biology_runtime": 27.339,
+      "eval_biology_samples_per_second": 18.289,
+      "eval_biology_steps_per_second": 4.572,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_chemistry_entropy": 1.6220130553245544,
+      "eval_chemistry_loss": 1.7599753141403198,
+      "eval_chemistry_mean_token_accuracy": 0.6644220778942108,
+      "eval_chemistry_num_tokens": 11437710.0,
+      "eval_chemistry_runtime": 34.5278,
+      "eval_chemistry_samples_per_second": 14.481,
+      "eval_chemistry_steps_per_second": 3.62,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_math_entropy": 0.7739131915569305,
+      "eval_math_loss": 0.7716156244277954,
+      "eval_math_mean_token_accuracy": 0.8028618817329407,
+      "eval_math_num_tokens": 11437710.0,
+      "eval_math_runtime": 35.5569,
+      "eval_math_samples_per_second": 14.062,
+      "eval_math_steps_per_second": 3.515,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_physics_entropy": 1.1613539371490478,
+      "eval_physics_loss": 1.246579885482788,
+      "eval_physics_mean_token_accuracy": 0.7308829197883606,
+      "eval_physics_num_tokens": 11437710.0,
+      "eval_physics_runtime": 41.2074,
+      "eval_physics_samples_per_second": 12.134,
+      "eval_physics_steps_per_second": 3.033,
+      "step": 400
+    },
+    {
+      "entropy": 0.7411411795765162,
+      "epoch": 0.656,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.3633333333333334e-05,
+      "loss": 0.7267,
+      "mean_token_accuracy": 0.8106930069625378,
+      "num_tokens": 11727268.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.7629997652024031,
+      "epoch": 0.672,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.3966666666666668e-05,
+      "loss": 0.7428,
+      "mean_token_accuracy": 0.8056826919317246,
+      "num_tokens": 12020478.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.7495820479467511,
+      "epoch": 0.688,
+      "grad_norm": 1.6328125,
+      "learning_rate": 1.43e-05,
+      "loss": 0.7268,
+      "mean_token_accuracy": 0.808751605078578,
+      "num_tokens": 12300380.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.7387325355783105,
+      "epoch": 0.704,
+      "grad_norm": 1.8046875,
+      "learning_rate": 1.4633333333333334e-05,
+      "loss": 0.7183,
+      "mean_token_accuracy": 0.8098014373332262,
+      "num_tokens": 12591366.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.757083036750555,
+      "epoch": 0.72,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.4966666666666667e-05,
+      "loss": 0.7296,
+      "mean_token_accuracy": 0.8071023549884557,
+      "num_tokens": 12869618.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.735528401657939,
+      "epoch": 0.736,
+      "grad_norm": 1.9765625,
+      "learning_rate": 1.5300000000000003e-05,
+      "loss": 0.7217,
+      "mean_token_accuracy": 0.8109818566590548,
+      "num_tokens": 13156311.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.7283307375386358,
+      "epoch": 0.752,
+      "grad_norm": 1.7734375,
+      "learning_rate": 1.5633333333333335e-05,
+      "loss": 0.7009,
+      "mean_token_accuracy": 0.8137538563460112,
+      "num_tokens": 13438911.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.7037693921476602,
+      "epoch": 0.768,
+      "grad_norm": 1.46875,
+      "learning_rate": 1.5966666666666667e-05,
+      "loss": 0.6928,
+      "mean_token_accuracy": 0.8189146313816309,
+      "num_tokens": 13735397.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.7542859153822065,
+      "epoch": 0.784,
+      "grad_norm": 1.9765625,
+      "learning_rate": 1.63e-05,
+      "loss": 0.7311,
+      "mean_token_accuracy": 0.8085405450314284,
+      "num_tokens": 14024497.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.7433818062767387,
+      "epoch": 0.8,
+      "grad_norm": 1.84375,
+      "learning_rate": 1.6633333333333336e-05,
+      "loss": 0.7265,
+      "mean_token_accuracy": 0.808263574540615,
+      "num_tokens": 14316427.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 3.1283711318969725,
+      "eval_biology_loss": 3.32532000541687,
+      "eval_biology_mean_token_accuracy": 0.47950622129440307,
+      "eval_biology_num_tokens": 14316427.0,
+      "eval_biology_runtime": 27.3377,
+      "eval_biology_samples_per_second": 18.29,
+      "eval_biology_steps_per_second": 4.572,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_chemistry_entropy": 1.653478057384491,
+      "eval_chemistry_loss": 1.6739438772201538,
+      "eval_chemistry_mean_token_accuracy": 0.6748317039012909,
+      "eval_chemistry_num_tokens": 14316427.0,
+      "eval_chemistry_runtime": 34.2699,
+      "eval_chemistry_samples_per_second": 14.59,
+      "eval_chemistry_steps_per_second": 3.648,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_math_entropy": 0.7548661649227142,
+      "eval_math_loss": 0.7394739985466003,
+      "eval_math_mean_token_accuracy": 0.807350200176239,
+      "eval_math_num_tokens": 14316427.0,
+      "eval_math_runtime": 35.4413,
+      "eval_math_samples_per_second": 14.108,
+      "eval_math_steps_per_second": 3.527,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_physics_entropy": 1.1648940191268922,
+      "eval_physics_loss": 1.1956281661987305,
+      "eval_physics_mean_token_accuracy": 0.7380300326347351,
+      "eval_physics_num_tokens": 14316427.0,
+      "eval_physics_runtime": 41.1508,
+      "eval_physics_samples_per_second": 12.15,
+      "eval_physics_steps_per_second": 3.038,
+      "step": 500
+    },
+    {
+      "entropy": 0.7288682738319039,
+      "epoch": 0.816,
+      "grad_norm": 1.7578125,
+      "learning_rate": 1.6966666666666668e-05,
+      "loss": 0.7054,
+      "mean_token_accuracy": 0.8109466649591923,
+      "num_tokens": 14607136.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.7200716659426689,
+      "epoch": 0.832,
+      "grad_norm": 1.6484375,
+      "learning_rate": 1.73e-05,
+      "loss": 0.7025,
+      "mean_token_accuracy": 0.8143566310405731,
+      "num_tokens": 14894084.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.7481377905234694,
+      "epoch": 0.848,
+      "grad_norm": 1.734375,
+      "learning_rate": 1.7633333333333336e-05,
+      "loss": 0.7284,
+      "mean_token_accuracy": 0.8080315344035626,
+      "num_tokens": 15177039.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.737450024113059,
+      "epoch": 0.864,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.796666666666667e-05,
+      "loss": 0.7113,
+      "mean_token_accuracy": 0.810840192809701,
+      "num_tokens": 15463393.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.7388068171218037,
+      "epoch": 0.88,
+      "grad_norm": 1.78125,
+      "learning_rate": 1.83e-05,
+      "loss": 0.7171,
+      "mean_token_accuracy": 0.808526486903429,
+      "num_tokens": 15747943.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.7369815710932016,
+      "epoch": 0.896,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.8633333333333333e-05,
+      "loss": 0.7174,
+      "mean_token_accuracy": 0.8113453444093466,
+      "num_tokens": 16027931.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.7089779742062092,
+      "epoch": 0.912,
+      "grad_norm": 1.65625,
+      "learning_rate": 1.896666666666667e-05,
+      "loss": 0.6903,
+      "mean_token_accuracy": 0.8140273470431566,
+      "num_tokens": 16317127.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.7124684944748878,
+      "epoch": 0.928,
+      "grad_norm": 1.640625,
+      "learning_rate": 1.93e-05,
+      "loss": 0.6895,
+      "mean_token_accuracy": 0.8141853727400303,
+      "num_tokens": 16598757.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.726560864970088,
+      "epoch": 0.944,
+      "grad_norm": 4.875,
+      "learning_rate": 1.9633333333333334e-05,
+      "loss": 0.7127,
+      "mean_token_accuracy": 0.81165018863976,
+      "num_tokens": 16882236.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.7348319509997964,
+      "epoch": 0.96,
+      "grad_norm": 1.375,
+      "learning_rate": 1.9966666666666666e-05,
+      "loss": 0.7131,
+      "mean_token_accuracy": 0.8112556729465723,
+      "num_tokens": 17169495.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_biology_entropy": 2.6509881944656373,
+      "eval_biology_loss": 3.270949125289917,
+      "eval_biology_mean_token_accuracy": 0.48801903986930845,
+      "eval_biology_num_tokens": 17169495.0,
+      "eval_biology_runtime": 27.3701,
+      "eval_biology_samples_per_second": 18.268,
+      "eval_biology_steps_per_second": 4.567,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_chemistry_entropy": 1.4206414470672608,
+      "eval_chemistry_loss": 1.6406230926513672,
+      "eval_chemistry_mean_token_accuracy": 0.6812815477848053,
+      "eval_chemistry_num_tokens": 17169495.0,
+      "eval_chemistry_runtime": 34.5426,
+      "eval_chemistry_samples_per_second": 14.475,
+      "eval_chemistry_steps_per_second": 3.619,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_math_entropy": 0.7245740656852722,
+      "eval_math_loss": 0.7165825366973877,
+      "eval_math_mean_token_accuracy": 0.8110268330574035,
+      "eval_math_num_tokens": 17169495.0,
+      "eval_math_runtime": 35.5941,
+      "eval_math_samples_per_second": 14.047,
+      "eval_math_steps_per_second": 3.512,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_physics_entropy": 1.061232611656189,
+      "eval_physics_loss": 1.1630969047546387,
+      "eval_physics_mean_token_accuracy": 0.7436661815643311,
+      "eval_physics_num_tokens": 17169495.0,
+      "eval_physics_runtime": 41.2108,
+      "eval_physics_samples_per_second": 12.133,
+      "eval_physics_steps_per_second": 3.033,
+      "step": 600
+    },
+    {
+      "entropy": 0.7112372944131493,
+      "epoch": 0.976,
+      "grad_norm": 1.703125,
+      "learning_rate": 1.9966666666666666e-05,
+      "loss": 0.6916,
+      "mean_token_accuracy": 0.8145402394235134,
+      "num_tokens": 17454147.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.7206208346411586,
+      "epoch": 0.992,
+      "grad_norm": 1.46875,
+      "learning_rate": 1.992962962962963e-05,
+      "loss": 0.6952,
+      "mean_token_accuracy": 0.8113347049802542,
+      "num_tokens": 17742596.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.6954748228192329,
+      "epoch": 1.008,
+      "grad_norm": 1.3671875,
+      "learning_rate": 1.9892592592592593e-05,
+      "loss": 0.6756,
+      "mean_token_accuracy": 0.8164566665887832,
+      "num_tokens": 18033540.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.6619296694174409,
+      "epoch": 1.024,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.9855555555555558e-05,
+      "loss": 0.6421,
+      "mean_token_accuracy": 0.8223833762109279,
+      "num_tokens": 18316346.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.6639872753992677,
+      "epoch": 1.04,
+      "grad_norm": 1.515625,
+      "learning_rate": 1.981851851851852e-05,
+      "loss": 0.6426,
+      "mean_token_accuracy": 0.8235018502920866,
+      "num_tokens": 18595230.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.674530028924346,
+      "epoch": 1.056,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.9781481481481484e-05,
+      "loss": 0.6544,
+      "mean_token_accuracy": 0.8203453596681356,
+      "num_tokens": 18885547.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.6652203820645809,
+      "epoch": 1.072,
+      "grad_norm": 1.3828125,
+      "learning_rate": 1.9744444444444446e-05,
+      "loss": 0.6464,
+      "mean_token_accuracy": 0.8207233041524887,
+      "num_tokens": 19169083.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.6666320703923703,
+      "epoch": 1.088,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.9707407407407407e-05,
+      "loss": 0.64,
+      "mean_token_accuracy": 0.8223770081996917,
+      "num_tokens": 19461465.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.6598716391250491,
+      "epoch": 1.104,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.9670370370370372e-05,
+      "loss": 0.6438,
+      "mean_token_accuracy": 0.822533056512475,
+      "num_tokens": 19755027.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.6419721782207489,
+      "epoch": 1.12,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.9633333333333334e-05,
+      "loss": 0.6239,
+      "mean_token_accuracy": 0.8271979708224535,
+      "num_tokens": 20043293.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_biology_entropy": 2.5483399925231933,
+      "eval_biology_loss": 3.304119348526001,
+      "eval_biology_mean_token_accuracy": 0.4908475253582001,
+      "eval_biology_num_tokens": 20043293.0,
+      "eval_biology_runtime": 27.3695,
+      "eval_biology_samples_per_second": 18.268,
+      "eval_biology_steps_per_second": 4.567,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_chemistry_entropy": 1.323628324508667,
+      "eval_chemistry_loss": 1.6196691989898682,
+      "eval_chemistry_mean_token_accuracy": 0.685321212053299,
+      "eval_chemistry_num_tokens": 20043293.0,
+      "eval_chemistry_runtime": 34.5494,
+      "eval_chemistry_samples_per_second": 14.472,
+      "eval_chemistry_steps_per_second": 3.618,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_math_entropy": 0.6825333631038666,
+      "eval_math_loss": 0.7001223564147949,
+      "eval_math_mean_token_accuracy": 0.8137474083900451,
+      "eval_math_num_tokens": 20043293.0,
+      "eval_math_runtime": 35.5959,
+      "eval_math_samples_per_second": 14.047,
+      "eval_math_steps_per_second": 3.512,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_physics_entropy": 0.974976891040802,
+      "eval_physics_loss": 1.148239254951477,
+      "eval_physics_mean_token_accuracy": 0.7467325129508973,
+      "eval_physics_num_tokens": 20043293.0,
+      "eval_physics_runtime": 41.239,
+      "eval_physics_samples_per_second": 12.124,
+      "eval_physics_steps_per_second": 3.031,
+      "step": 700
+    },
+    {
+      "entropy": 0.647271628677845,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.95962962962963e-05,
+      "loss": 0.6354,
+      "mean_token_accuracy": 0.8242664244025946,
+      "num_tokens": 20323010.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.6585094593465328,
+      "epoch": 1.152,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.955925925925926e-05,
+      "loss": 0.6412,
+      "mean_token_accuracy": 0.8244892667979002,
+      "num_tokens": 20613266.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.6505060261115432,
+      "epoch": 1.168,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.9522222222222225e-05,
+      "loss": 0.6363,
+      "mean_token_accuracy": 0.8239100620150566,
+      "num_tokens": 20910404.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.6509549736976623,
+      "epoch": 1.184,
+      "grad_norm": 1.546875,
+      "learning_rate": 1.9485185185185187e-05,
+      "loss": 0.6324,
+      "mean_token_accuracy": 0.8239473849534988,
+      "num_tokens": 21200197.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.6359233345836401,
+      "epoch": 1.2,
+      "grad_norm": 1.265625,
+      "learning_rate": 1.944814814814815e-05,
+      "loss": 0.6195,
+      "mean_token_accuracy": 0.8269548691809178,
+      "num_tokens": 21496441.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.6566847365349531,
+      "epoch": 1.216,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.941111111111111e-05,
+      "loss": 0.6366,
+      "mean_token_accuracy": 0.8229359701275826,
+      "num_tokens": 21778030.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.6345798650756478,
+      "epoch": 1.232,
+      "grad_norm": 1.546875,
+      "learning_rate": 1.9374074074074075e-05,
+      "loss": 0.6178,
+      "mean_token_accuracy": 0.8286544498056173,
+      "num_tokens": 22060851.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.6317464917898178,
+      "epoch": 1.248,
+      "grad_norm": 1.359375,
+      "learning_rate": 1.933703703703704e-05,
+      "loss": 0.6122,
+      "mean_token_accuracy": 0.827758340910077,
+      "num_tokens": 22348738.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.6566873624920845,
+      "epoch": 1.264,
+      "grad_norm": 1.3515625,
+      "learning_rate": 1.93e-05,
+      "loss": 0.6359,
+      "mean_token_accuracy": 0.8226253062486648,
+      "num_tokens": 22635039.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.6513905024155975,
+      "epoch": 1.28,
+      "grad_norm": 1.375,
+      "learning_rate": 1.9262962962962967e-05,
+      "loss": 0.6332,
+      "mean_token_accuracy": 0.8246075186878443,
+      "num_tokens": 22915190.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_biology_entropy": 2.615177293777466,
+      "eval_biology_loss": 3.3023173809051514,
+      "eval_biology_mean_token_accuracy": 0.4925825412273407,
+      "eval_biology_num_tokens": 22915190.0,
+      "eval_biology_runtime": 27.3795,
+      "eval_biology_samples_per_second": 18.262,
+      "eval_biology_steps_per_second": 4.565,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_chemistry_entropy": 1.3414388194084168,
+      "eval_chemistry_loss": 1.5987354516983032,
+      "eval_chemistry_mean_token_accuracy": 0.6886345105171203,
+      "eval_chemistry_num_tokens": 22915190.0,
+      "eval_chemistry_runtime": 34.9278,
+      "eval_chemistry_samples_per_second": 14.315,
+      "eval_chemistry_steps_per_second": 3.579,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_math_entropy": 0.6710349791049958,
+      "eval_math_loss": 0.6838563084602356,
+      "eval_math_mean_token_accuracy": 0.8167732076644898,
+      "eval_math_num_tokens": 22915190.0,
+      "eval_math_runtime": 35.5835,
+      "eval_math_samples_per_second": 14.051,
+      "eval_math_steps_per_second": 3.513,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_physics_entropy": 0.9784445714950561,
+      "eval_physics_loss": 1.1236796379089355,
+      "eval_physics_mean_token_accuracy": 0.7502694492340087,
+      "eval_physics_num_tokens": 22915190.0,
+      "eval_physics_runtime": 41.2279,
+      "eval_physics_samples_per_second": 12.128,
+      "eval_physics_steps_per_second": 3.032,
+      "step": 800
+    },
+    {
+      "entropy": 0.6346945486962795,
+      "epoch": 1.296,
+      "grad_norm": 1.3515625,
+      "learning_rate": 1.9225925925925928e-05,
+      "loss": 0.6142,
+      "mean_token_accuracy": 0.827120078727603,
+      "num_tokens": 23194947.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.6482907500118017,
+      "epoch": 1.312,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.918888888888889e-05,
+      "loss": 0.6379,
+      "mean_token_accuracy": 0.824469443410635,
+      "num_tokens": 23483496.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.6572995545342565,
+      "epoch": 1.328,
+      "grad_norm": 1.46875,
+      "learning_rate": 1.915185185185185e-05,
+      "loss": 0.6402,
+      "mean_token_accuracy": 0.8227909743785858,
+      "num_tokens": 23762589.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.6324281422421336,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.9114814814814816e-05,
+      "loss": 0.6184,
+      "mean_token_accuracy": 0.8270029693841934,
+      "num_tokens": 24041497.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.6391168424859643,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 1.09375,
+      "learning_rate": 1.907777777777778e-05,
+      "loss": 0.623,
+      "mean_token_accuracy": 0.8274307977408171,
+      "num_tokens": 24330591.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.6829585919156671,
+      "epoch": 1.376,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.9040740740740743e-05,
+      "loss": 0.6631,
+      "mean_token_accuracy": 0.8163150053471326,
+      "num_tokens": 24611420.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.6386255823075772,
+      "epoch": 1.392,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.9003703703703704e-05,
+      "loss": 0.6189,
+      "mean_token_accuracy": 0.8260396342724562,
+      "num_tokens": 24894694.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.6312606217339635,
+      "epoch": 1.408,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.896666666666667e-05,
+      "loss": 0.6129,
+      "mean_token_accuracy": 0.8287516962736845,
+      "num_tokens": 25184946.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.6245067711919546,
+      "epoch": 1.424,
+      "grad_norm": 1.1328125,
+      "learning_rate": 1.892962962962963e-05,
+      "loss": 0.6122,
+      "mean_token_accuracy": 0.8291997779160738,
+      "num_tokens": 25474746.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.6473252691328526,
+      "epoch": 1.44,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.8892592592592592e-05,
+      "loss": 0.6279,
+      "mean_token_accuracy": 0.8257731795310974,
+      "num_tokens": 25758550.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_biology_entropy": 2.6763860845565794,
+      "eval_biology_loss": 3.173260450363159,
+      "eval_biology_mean_token_accuracy": 0.4976111261844635,
+      "eval_biology_num_tokens": 25758550.0,
+      "eval_biology_runtime": 27.37,
+      "eval_biology_samples_per_second": 18.268,
+      "eval_biology_steps_per_second": 4.567,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_chemistry_entropy": 1.3377286305427551,
+      "eval_chemistry_loss": 1.5585026741027832,
+      "eval_chemistry_mean_token_accuracy": 0.6910244026184083,
+      "eval_chemistry_num_tokens": 25758550.0,
+      "eval_chemistry_runtime": 34.5566,
+      "eval_chemistry_samples_per_second": 14.469,
+      "eval_chemistry_steps_per_second": 3.617,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_math_entropy": 0.6553519473075867,
+      "eval_math_loss": 0.6712566018104553,
+      "eval_math_mean_token_accuracy": 0.8193869161605835,
+      "eval_math_num_tokens": 25758550.0,
+      "eval_math_runtime": 35.5892,
+      "eval_math_samples_per_second": 14.049,
+      "eval_math_steps_per_second": 3.512,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_physics_entropy": 0.9619774799346924,
+      "eval_physics_loss": 1.1111396551132202,
+      "eval_physics_mean_token_accuracy": 0.7528104310035706,
+      "eval_physics_num_tokens": 25758550.0,
+      "eval_physics_runtime": 41.2096,
+      "eval_physics_samples_per_second": 12.133,
+      "eval_physics_steps_per_second": 3.033,
+      "step": 900
+    },
+    {
+      "entropy": 0.6286329736933112,
+      "epoch": 1.456,
+      "grad_norm": 1.203125,
+      "learning_rate": 1.8855555555555557e-05,
+      "loss": 0.6127,
+      "mean_token_accuracy": 0.8275392398238182,
+      "num_tokens": 26039862.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.6320978362113238,
+      "epoch": 1.472,
+      "grad_norm": 1.296875,
+      "learning_rate": 1.881851851851852e-05,
+      "loss": 0.622,
+      "mean_token_accuracy": 0.8266615178436041,
+      "num_tokens": 26318666.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.6575064631178975,
+      "epoch": 1.488,
+      "grad_norm": 1.2109375,
+      "learning_rate": 1.8781481481481484e-05,
+      "loss": 0.6406,
+      "mean_token_accuracy": 0.8220913629978895,
+      "num_tokens": 26594867.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.6149632008746266,
+      "epoch": 1.504,
+      "grad_norm": 1.5,
+      "learning_rate": 1.8744444444444445e-05,
+      "loss": 0.6043,
+      "mean_token_accuracy": 0.8317741062492132,
+      "num_tokens": 26887233.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.6513713056221604,
+      "epoch": 1.52,
+      "grad_norm": 1.1015625,
+      "learning_rate": 1.870740740740741e-05,
+      "loss": 0.6258,
+      "mean_token_accuracy": 0.8250748608261347,
+      "num_tokens": 27177448.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.629990316927433,
+      "epoch": 1.536,
+      "grad_norm": 1.09375,
+      "learning_rate": 1.8670370370370372e-05,
+      "loss": 0.621,
+      "mean_token_accuracy": 0.8269896287471056,
+      "num_tokens": 27467857.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.6225374899804592,
+      "epoch": 1.552,
+      "grad_norm": 1.3515625,
+      "learning_rate": 1.8633333333333333e-05,
+      "loss": 0.6104,
+      "mean_token_accuracy": 0.829260740801692,
+      "num_tokens": 27755138.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.6163769841194153,
+      "epoch": 1.568,
+      "grad_norm": 1.1640625,
+      "learning_rate": 1.85962962962963e-05,
+      "loss": 0.5994,
+      "mean_token_accuracy": 0.8334966115653515,
+      "num_tokens": 28049363.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.6230853283777833,
+      "epoch": 1.584,
+      "grad_norm": 1.3515625,
+      "learning_rate": 1.855925925925926e-05,
+      "loss": 0.6091,
+      "mean_token_accuracy": 0.8301577147096395,
+      "num_tokens": 28337890.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.6198741147294641,
+      "epoch": 1.6,
+      "grad_norm": 1.3125,
+      "learning_rate": 1.8522222222222225e-05,
+      "loss": 0.6074,
+      "mean_token_accuracy": 0.8279337525367737,
+      "num_tokens": 28626068.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_biology_entropy": 2.5461103229522704,
+      "eval_biology_loss": 3.16886568069458,
+      "eval_biology_mean_token_accuracy": 0.5016276073455811,
+      "eval_biology_num_tokens": 28626068.0,
+      "eval_biology_runtime": 27.3573,
+      "eval_biology_samples_per_second": 18.277,
+      "eval_biology_steps_per_second": 4.569,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_chemistry_entropy": 1.2698839874267578,
+      "eval_chemistry_loss": 1.5576449632644653,
+      "eval_chemistry_mean_token_accuracy": 0.6939039103984833,
+      "eval_chemistry_num_tokens": 28626068.0,
+      "eval_chemistry_runtime": 34.2793,
+      "eval_chemistry_samples_per_second": 14.586,
+      "eval_chemistry_steps_per_second": 3.647,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_math_entropy": 0.6368684613704682,
+      "eval_math_loss": 0.6604805588722229,
+      "eval_math_mean_token_accuracy": 0.8214335250854492,
+      "eval_math_num_tokens": 28626068.0,
+      "eval_math_runtime": 35.4518,
+      "eval_math_samples_per_second": 14.104,
+      "eval_math_steps_per_second": 3.526,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_physics_entropy": 0.9321202540397644,
+      "eval_physics_loss": 1.1082653999328613,
+      "eval_physics_mean_token_accuracy": 0.753868275642395,
+      "eval_physics_num_tokens": 28626068.0,
+      "eval_physics_runtime": 41.5098,
+      "eval_physics_samples_per_second": 12.045,
+      "eval_physics_steps_per_second": 3.011,
+      "step": 1000
+    },
+    {
+      "entropy": 0.6313193686306476,
+      "epoch": 1.616,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.8485185185185186e-05,
+      "loss": 0.622,
+      "mean_token_accuracy": 0.8274159673601389,
+      "num_tokens": 28913208.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.632500814460218,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.844814814814815e-05,
+      "loss": 0.6182,
+      "mean_token_accuracy": 0.8253830634057522,
+      "num_tokens": 29188917.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.6128746416419745,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 1.2421875,
+      "learning_rate": 1.8411111111111113e-05,
+      "loss": 0.6004,
+      "mean_token_accuracy": 0.8334595657885074,
+      "num_tokens": 29477292.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.6234495924785733,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.8374074074074074e-05,
+      "loss": 0.6058,
+      "mean_token_accuracy": 0.8289853416383266,
+      "num_tokens": 29772480.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.6295564426109195,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 1.1171875,
+      "learning_rate": 1.8337037037037036e-05,
+      "loss": 0.6169,
+      "mean_token_accuracy": 0.8286502953618765,
+      "num_tokens": 30061007.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.6352518420666456,
+      "epoch": 1.696,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.83e-05,
+      "loss": 0.6183,
+      "mean_token_accuracy": 0.8266870338469744,
+      "num_tokens": 30344415.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.6140534708276391,
+      "epoch": 1.712,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.8262962962962966e-05,
+      "loss": 0.6048,
+      "mean_token_accuracy": 0.8309987772256135,
+      "num_tokens": 30640891.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.6059238471090793,
+      "epoch": 1.728,
+      "grad_norm": 1.25,
+      "learning_rate": 1.8225925925925927e-05,
+      "loss": 0.5948,
+      "mean_token_accuracy": 0.8328473635017872,
+      "num_tokens": 30921295.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.6121155306696892,
+      "epoch": 1.744,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.8188888888888892e-05,
+      "loss": 0.6033,
+      "mean_token_accuracy": 0.8320579200983047,
+      "num_tokens": 31206375.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.6091686520725489,
+      "epoch": 1.76,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.8151851851851854e-05,
+      "loss": 0.5938,
+      "mean_token_accuracy": 0.8308170285075903,
+      "num_tokens": 31491856.0,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_biology_entropy": 2.5234148960113525,
+      "eval_biology_loss": 3.142578125,
+      "eval_biology_mean_token_accuracy": 0.5035465421676636,
+      "eval_biology_num_tokens": 31491856.0,
+      "eval_biology_runtime": 27.4009,
+      "eval_biology_samples_per_second": 18.248,
+      "eval_biology_steps_per_second": 4.562,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_chemistry_entropy": 1.282408926010132,
+      "eval_chemistry_loss": 1.5344536304473877,
+      "eval_chemistry_mean_token_accuracy": 0.6954309647083282,
+      "eval_chemistry_num_tokens": 31491856.0,
+      "eval_chemistry_runtime": 34.5999,
+      "eval_chemistry_samples_per_second": 14.451,
+      "eval_chemistry_steps_per_second": 3.613,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_math_entropy": 0.6422419800758362,
+      "eval_math_loss": 0.6516004800796509,
+      "eval_math_mean_token_accuracy": 0.8231606707572937,
+      "eval_math_num_tokens": 31491856.0,
+      "eval_math_runtime": 35.6246,
+      "eval_math_samples_per_second": 14.035,
+      "eval_math_steps_per_second": 3.509,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_physics_entropy": 0.9564714193344116,
+      "eval_physics_loss": 1.0839052200317383,
+      "eval_physics_mean_token_accuracy": 0.7559929075241089,
+      "eval_physics_num_tokens": 31491856.0,
+      "eval_physics_runtime": 41.27,
+      "eval_physics_samples_per_second": 12.115,
+      "eval_physics_steps_per_second": 3.029,
+      "step": 1100
+    },
+    {
+      "entropy": 0.6162398861721158,
+      "epoch": 1.776,
+      "grad_norm": 1.2578125,
+      "learning_rate": 1.8114814814814816e-05,
+      "loss": 0.5992,
+      "mean_token_accuracy": 0.8303060304373503,
+      "num_tokens": 31782829.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.6266137836501002,
+      "epoch": 1.792,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.8077777777777777e-05,
+      "loss": 0.6126,
+      "mean_token_accuracy": 0.8280655439943075,
+      "num_tokens": 32070187.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.6210170408710838,
+      "epoch": 1.808,
+      "grad_norm": 1.2109375,
+      "learning_rate": 1.8040740740740742e-05,
+      "loss": 0.6125,
+      "mean_token_accuracy": 0.8283982180058956,
+      "num_tokens": 32355072.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.5900590697303414,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 1.078125,
+      "learning_rate": 1.8003703703703707e-05,
+      "loss": 0.5742,
+      "mean_token_accuracy": 0.8367947399616241,
+      "num_tokens": 32634013.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.6116258937865495,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 1.3671875,
+      "learning_rate": 1.796666666666667e-05,
+      "loss": 0.602,
+      "mean_token_accuracy": 0.8321913931518793,
+      "num_tokens": 32917222.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.6234990337863564,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 1.203125,
+      "learning_rate": 1.792962962962963e-05,
+      "loss": 0.6033,
+      "mean_token_accuracy": 0.8300769094377756,
+      "num_tokens": 33207147.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.6067718509584665,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 1.296875,
+      "learning_rate": 1.7892592592592595e-05,
+      "loss": 0.5921,
+      "mean_token_accuracy": 0.8332997910678387,
+      "num_tokens": 33496572.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.6149971356615425,
+      "epoch": 1.888,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.7855555555555557e-05,
+      "loss": 0.6012,
+      "mean_token_accuracy": 0.8313036315143109,
+      "num_tokens": 33775788.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.6183292152360081,
+      "epoch": 1.904,
+      "grad_norm": 1.265625,
+      "learning_rate": 1.7818518518518518e-05,
+      "loss": 0.6066,
+      "mean_token_accuracy": 0.8297228269279003,
+      "num_tokens": 34052322.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.5967534257099032,
+      "epoch": 1.92,
+      "grad_norm": 1.171875,
+      "learning_rate": 1.7781481481481483e-05,
+      "loss": 0.5861,
+      "mean_token_accuracy": 0.8338323317468166,
+      "num_tokens": 34331468.0,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_biology_entropy": 2.5120252017974853,
+      "eval_biology_loss": 3.1175456047058105,
+      "eval_biology_mean_token_accuracy": 0.5043386931419372,
+      "eval_biology_num_tokens": 34331468.0,
+      "eval_biology_runtime": 27.3783,
+      "eval_biology_samples_per_second": 18.263,
+      "eval_biology_steps_per_second": 4.566,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_chemistry_entropy": 1.2727207522392272,
+      "eval_chemistry_loss": 1.542059063911438,
+      "eval_chemistry_mean_token_accuracy": 0.6960053942203521,
+      "eval_chemistry_num_tokens": 34331468.0,
+      "eval_chemistry_runtime": 34.5719,
+      "eval_chemistry_samples_per_second": 14.463,
+      "eval_chemistry_steps_per_second": 3.616,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_math_entropy": 0.6281100554466248,
+      "eval_math_loss": 0.646048367023468,
+      "eval_math_mean_token_accuracy": 0.8237533407211304,
+      "eval_math_num_tokens": 34331468.0,
+      "eval_math_runtime": 35.6158,
+      "eval_math_samples_per_second": 14.039,
+      "eval_math_steps_per_second": 3.51,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_physics_entropy": 0.9339615530967712,
+      "eval_physics_loss": 1.0827937126159668,
+      "eval_physics_mean_token_accuracy": 0.7571085038185119,
+      "eval_physics_num_tokens": 34331468.0,
+      "eval_physics_runtime": 41.2881,
+      "eval_physics_samples_per_second": 12.11,
+      "eval_physics_steps_per_second": 3.028,
+      "step": 1200
+    },
+    {
+      "entropy": 0.6118536571040749,
+      "epoch": 1.936,
+      "grad_norm": 1.34375,
+      "learning_rate": 1.7744444444444445e-05,
+      "loss": 0.6014,
+      "mean_token_accuracy": 0.8308535933494567,
+      "num_tokens": 34612743.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.6105628669261932,
+      "epoch": 1.952,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.770740740740741e-05,
+      "loss": 0.5904,
+      "mean_token_accuracy": 0.831901628896594,
+      "num_tokens": 34891115.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.6228807104751468,
+      "epoch": 1.968,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.767037037037037e-05,
+      "loss": 0.6172,
+      "mean_token_accuracy": 0.8282031897455454,
+      "num_tokens": 35179505.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.5993367858231068,
+      "epoch": 1.984,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.7633333333333336e-05,
+      "loss": 0.5872,
+      "mean_token_accuracy": 0.8343776896595955,
+      "num_tokens": 35472596.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.6047713309526443,
+      "epoch": 2.0,
+      "grad_norm": 1.140625,
+      "learning_rate": 1.7596296296296298e-05,
+      "loss": 0.5931,
+      "mean_token_accuracy": 0.8327744524925947,
+      "num_tokens": 35772848.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.5155709801241756,
+      "epoch": 2.016,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.755925925925926e-05,
+      "loss": 0.4906,
+      "mean_token_accuracy": 0.852144667878747,
+      "num_tokens": 36060330.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.4909618007019162,
+      "epoch": 2.032,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.7522222222222224e-05,
+      "loss": 0.4759,
+      "mean_token_accuracy": 0.8550532199442387,
+      "num_tokens": 36343894.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.47384012741968035,
+      "epoch": 2.048,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.7485185185185186e-05,
+      "loss": 0.4635,
+      "mean_token_accuracy": 0.8587454378604888,
+      "num_tokens": 36636410.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.47754389233887196,
+      "epoch": 2.064,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.744814814814815e-05,
+      "loss": 0.4613,
+      "mean_token_accuracy": 0.8579986277967692,
+      "num_tokens": 36925382.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.477533900924027,
+      "epoch": 2.08,
+      "grad_norm": 1.5,
+      "learning_rate": 1.7411111111111112e-05,
+      "loss": 0.4577,
+      "mean_token_accuracy": 0.8580481503158808,
+      "num_tokens": 37208672.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_biology_entropy": 1.9212858390808105,
+      "eval_biology_loss": 3.4324111938476562,
+      "eval_biology_mean_token_accuracy": 0.49684978103637695,
+      "eval_biology_num_tokens": 37208672.0,
+      "eval_biology_runtime": 27.3965,
+      "eval_biology_samples_per_second": 18.251,
+      "eval_biology_steps_per_second": 4.563,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_chemistry_entropy": 0.9143568122386933,
+      "eval_chemistry_loss": 1.7243843078613281,
+      "eval_chemistry_mean_token_accuracy": 0.6901392197608948,
+      "eval_chemistry_num_tokens": 37208672.0,
+      "eval_chemistry_runtime": 34.5719,
+      "eval_chemistry_samples_per_second": 14.463,
+      "eval_chemistry_steps_per_second": 3.616,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_math_entropy": 0.5000924124717713,
+      "eval_math_loss": 0.6863380074501038,
+      "eval_math_mean_token_accuracy": 0.8220298099517822,
+      "eval_math_num_tokens": 37208672.0,
+      "eval_math_runtime": 35.6225,
+      "eval_math_samples_per_second": 14.036,
+      "eval_math_steps_per_second": 3.509,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_physics_entropy": 0.6751700477600098,
+      "eval_physics_loss": 1.236175298690796,
+      "eval_physics_mean_token_accuracy": 0.7503512153625488,
+      "eval_physics_num_tokens": 37208672.0,
+      "eval_physics_runtime": 41.2659,
+      "eval_physics_samples_per_second": 12.117,
+      "eval_physics_steps_per_second": 3.029,
+      "step": 1300
+    },
+    {
+      "entropy": 0.4839732026681304,
+      "epoch": 2.096,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.7374074074074077e-05,
+      "loss": 0.4708,
+      "mean_token_accuracy": 0.8568592548370362,
+      "num_tokens": 37491954.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.4706313606351614,
+      "epoch": 2.112,
+      "grad_norm": 1.5,
+      "learning_rate": 1.733703703703704e-05,
+      "loss": 0.4554,
+      "mean_token_accuracy": 0.8569178070873023,
+      "num_tokens": 37787033.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.4742715910077095,
+      "epoch": 2.128,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.73e-05,
+      "loss": 0.4584,
+      "mean_token_accuracy": 0.859177602827549,
+      "num_tokens": 38068955.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.4703082088381052,
+      "epoch": 2.144,
+      "grad_norm": 1.46875,
+      "learning_rate": 1.7262962962962962e-05,
+      "loss": 0.4648,
+      "mean_token_accuracy": 0.8577755928039551,
+      "num_tokens": 38359914.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.4766748737543821,
+      "epoch": 2.16,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.7225925925925927e-05,
+      "loss": 0.4623,
+      "mean_token_accuracy": 0.8581593327224255,
+      "num_tokens": 38649684.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.45521042607724665,
+      "epoch": 2.176,
+      "grad_norm": 1.5546875,
+      "learning_rate": 1.7188888888888892e-05,
+      "loss": 0.4407,
+      "mean_token_accuracy": 0.8626933339983225,
+      "num_tokens": 38938155.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.4853563438169658,
+      "epoch": 2.192,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.7151851851851853e-05,
+      "loss": 0.4724,
+      "mean_token_accuracy": 0.8544307108968496,
+      "num_tokens": 39228800.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.4833542363718152,
+      "epoch": 2.208,
+      "grad_norm": 1.6875,
+      "learning_rate": 1.7114814814814815e-05,
+      "loss": 0.4735,
+      "mean_token_accuracy": 0.8565698899328709,
+      "num_tokens": 39513118.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.4737219920381904,
+      "epoch": 2.224,
+      "grad_norm": 1.6171875,
+      "learning_rate": 1.707777777777778e-05,
+      "loss": 0.4597,
+      "mean_token_accuracy": 0.8575448788702488,
+      "num_tokens": 39804212.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.4836106028407812,
+      "epoch": 2.24,
+      "grad_norm": 1.53125,
+      "learning_rate": 1.704074074074074e-05,
+      "loss": 0.4742,
+      "mean_token_accuracy": 0.8556978423148394,
+      "num_tokens": 40092012.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_biology_entropy": 1.9033697624206543,
+      "eval_biology_loss": 3.5103633403778076,
+      "eval_biology_mean_token_accuracy": 0.49410775113105776,
+      "eval_biology_num_tokens": 40092012.0,
+      "eval_biology_runtime": 27.3812,
+      "eval_biology_samples_per_second": 18.261,
+      "eval_biology_steps_per_second": 4.565,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_chemistry_entropy": 0.9143473811149597,
+      "eval_chemistry_loss": 1.774486780166626,
+      "eval_chemistry_mean_token_accuracy": 0.6876211338043213,
+      "eval_chemistry_num_tokens": 40092012.0,
+      "eval_chemistry_runtime": 34.5787,
+      "eval_chemistry_samples_per_second": 14.46,
+      "eval_chemistry_steps_per_second": 3.615,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_math_entropy": 0.5127572872638703,
+      "eval_math_loss": 0.6833173632621765,
+      "eval_math_mean_token_accuracy": 0.8215805702209472,
+      "eval_math_num_tokens": 40092012.0,
+      "eval_math_runtime": 35.6078,
+      "eval_math_samples_per_second": 14.042,
+      "eval_math_steps_per_second": 3.51,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_physics_entropy": 0.6786217405796051,
+      "eval_physics_loss": 1.2495317459106445,
+      "eval_physics_mean_token_accuracy": 0.7499923658370972,
+      "eval_physics_num_tokens": 40092012.0,
+      "eval_physics_runtime": 41.2591,
+      "eval_physics_samples_per_second": 12.119,
+      "eval_physics_steps_per_second": 3.03,
+      "step": 1400
+    },
+    {
+      "entropy": 0.4777224402874708,
+      "epoch": 2.2560000000000002,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.7003703703703703e-05,
+      "loss": 0.4646,
+      "mean_token_accuracy": 0.8567751668393612,
+      "num_tokens": 40380069.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.4836098331958055,
+      "epoch": 2.2720000000000002,
+      "grad_norm": 1.78125,
+      "learning_rate": 1.6966666666666668e-05,
+      "loss": 0.4704,
+      "mean_token_accuracy": 0.8550269123166799,
+      "num_tokens": 40655903.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.4833408955484629,
+      "epoch": 2.288,
+      "grad_norm": 1.6796875,
+      "learning_rate": 1.6929629629629633e-05,
+      "loss": 0.4684,
+      "mean_token_accuracy": 0.8551026821136475,
+      "num_tokens": 40944791.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.46882958598434926,
+      "epoch": 2.304,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.6892592592592594e-05,
+      "loss": 0.4568,
+      "mean_token_accuracy": 0.8597967244684697,
+      "num_tokens": 41227146.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.4883909322321415,
+      "epoch": 2.32,
+      "grad_norm": 1.609375,
+      "learning_rate": 1.6855555555555556e-05,
+      "loss": 0.4792,
+      "mean_token_accuracy": 0.8557040151208639,
+      "num_tokens": 41509322.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.4878495089709759,
+      "epoch": 2.336,
+      "grad_norm": 1.5,
+      "learning_rate": 1.681851851851852e-05,
+      "loss": 0.4688,
+      "mean_token_accuracy": 0.8561562273651362,
+      "num_tokens": 41792799.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.49036325346678494,
+      "epoch": 2.352,
+      "grad_norm": 1.609375,
+      "learning_rate": 1.6781481481481483e-05,
+      "loss": 0.4805,
+      "mean_token_accuracy": 0.8520781677216291,
+      "num_tokens": 42083080.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.4816764058545232,
+      "epoch": 2.368,
+      "grad_norm": 1.609375,
+      "learning_rate": 1.6744444444444444e-05,
+      "loss": 0.4683,
+      "mean_token_accuracy": 0.8558718893676996,
+      "num_tokens": 42368442.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.4973900321871042,
+      "epoch": 2.384,
+      "grad_norm": 1.7578125,
+      "learning_rate": 1.670740740740741e-05,
+      "loss": 0.4872,
+      "mean_token_accuracy": 0.8530533351004124,
+      "num_tokens": 42663926.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.4891781846061349,
+      "epoch": 2.4,
+      "grad_norm": 1.84375,
+      "learning_rate": 1.667037037037037e-05,
+      "loss": 0.48,
+      "mean_token_accuracy": 0.8547772366553545,
+      "num_tokens": 42946351.0,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_biology_entropy": 1.8815545105934144,
+      "eval_biology_loss": 3.5141749382019043,
+      "eval_biology_mean_token_accuracy": 0.4941290125846863,
+      "eval_biology_num_tokens": 42946351.0,
+      "eval_biology_runtime": 27.7037,
+      "eval_biology_samples_per_second": 18.048,
+      "eval_biology_steps_per_second": 4.512,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_chemistry_entropy": 0.9088821232318878,
+      "eval_chemistry_loss": 1.7617615461349487,
+      "eval_chemistry_mean_token_accuracy": 0.6893277273178101,
+      "eval_chemistry_num_tokens": 42946351.0,
+      "eval_chemistry_runtime": 34.2534,
+      "eval_chemistry_samples_per_second": 14.597,
+      "eval_chemistry_steps_per_second": 3.649,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_math_entropy": 0.5177790434360504,
+      "eval_math_loss": 0.6780233979225159,
+      "eval_math_mean_token_accuracy": 0.8222798390388488,
+      "eval_math_num_tokens": 42946351.0,
+      "eval_math_runtime": 35.4396,
+      "eval_math_samples_per_second": 14.109,
+      "eval_math_steps_per_second": 3.527,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_physics_entropy": 0.6826021068096161,
+      "eval_physics_loss": 1.249171257019043,
+      "eval_physics_mean_token_accuracy": 0.7502854156494141,
+      "eval_physics_num_tokens": 42946351.0,
+      "eval_physics_runtime": 41.1298,
+      "eval_physics_samples_per_second": 12.157,
+      "eval_physics_steps_per_second": 3.039,
+      "step": 1500
+    },
+    {
+      "entropy": 0.4981799952685833,
+      "epoch": 2.416,
+      "grad_norm": 1.6640625,
+      "learning_rate": 1.6633333333333336e-05,
+      "loss": 0.4852,
+      "mean_token_accuracy": 0.852097024768591,
+      "num_tokens": 43227782.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.4849033308215439,
+      "epoch": 2.432,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.6596296296296297e-05,
+      "loss": 0.4739,
+      "mean_token_accuracy": 0.8551602113991976,
+      "num_tokens": 43511939.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.47294504195451736,
+      "epoch": 2.448,
+      "grad_norm": 1.71875,
+      "learning_rate": 1.6559259259259262e-05,
+      "loss": 0.4611,
+      "mean_token_accuracy": 0.8589533522725106,
+      "num_tokens": 43811005.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.4894484682008624,
+      "epoch": 2.464,
+      "grad_norm": 1.640625,
+      "learning_rate": 1.6522222222222224e-05,
+      "loss": 0.4775,
+      "mean_token_accuracy": 0.8542467694729566,
+      "num_tokens": 44106223.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.5072572749108076,
+      "epoch": 2.48,
+      "grad_norm": 1.7421875,
+      "learning_rate": 1.6485185185185185e-05,
+      "loss": 0.4921,
+      "mean_token_accuracy": 0.8498393319547176,
+      "num_tokens": 44379095.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.48876257669180634,
+      "epoch": 2.496,
+      "grad_norm": 1.6015625,
+      "learning_rate": 1.644814814814815e-05,
+      "loss": 0.4786,
+      "mean_token_accuracy": 0.853264183551073,
+      "num_tokens": 44670316.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.4863064644858241,
+      "epoch": 2.512,
+      "grad_norm": 1.6875,
+      "learning_rate": 1.6411111111111112e-05,
+      "loss": 0.4743,
+      "mean_token_accuracy": 0.8554254125803709,
+      "num_tokens": 44952877.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.4791873342357576,
+      "epoch": 2.528,
+      "grad_norm": 1.546875,
+      "learning_rate": 1.6374074074074077e-05,
+      "loss": 0.4682,
+      "mean_token_accuracy": 0.8562641687691211,
+      "num_tokens": 45237653.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.47710453290492294,
+      "epoch": 2.544,
+      "grad_norm": 1.6875,
+      "learning_rate": 1.6337037037037038e-05,
+      "loss": 0.4678,
+      "mean_token_accuracy": 0.8573417123407125,
+      "num_tokens": 45524866.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.47768820971250536,
+      "epoch": 2.56,
+      "grad_norm": 1.5390625,
+      "learning_rate": 1.63e-05,
+      "loss": 0.4634,
+      "mean_token_accuracy": 0.8564421378076077,
+      "num_tokens": 45803783.0,
+      "step": 1600
+    },
+    {
+      "epoch": 2.56,
+      "eval_biology_entropy": 1.8816225261688233,
+      "eval_biology_loss": 3.5400381088256836,
+      "eval_biology_mean_token_accuracy": 0.4940107421875,
+      "eval_biology_num_tokens": 45803783.0,
+      "eval_biology_runtime": 27.3897,
+      "eval_biology_samples_per_second": 18.255,
+      "eval_biology_steps_per_second": 4.564,
+      "step": 1600
+    },
+    {
+      "epoch": 2.56,
+      "eval_chemistry_entropy": 0.9121307201385498,
+      "eval_chemistry_loss": 1.7674332857131958,
+      "eval_chemistry_mean_token_accuracy": 0.688238130569458,
+      "eval_chemistry_num_tokens": 45803783.0,
+      "eval_chemistry_runtime": 34.545,
+      "eval_chemistry_samples_per_second": 14.474,
+      "eval_chemistry_steps_per_second": 3.618,
+      "step": 1600
+    },
+    {
+      "epoch": 2.56,
+      "eval_math_entropy": 0.5144661903381348,
+      "eval_math_loss": 0.6768107414245605,
+      "eval_math_mean_token_accuracy": 0.822895619392395,
+      "eval_math_num_tokens": 45803783.0,
+      "eval_math_runtime": 35.5936,
+      "eval_math_samples_per_second": 14.047,
+      "eval_math_steps_per_second": 3.512,
+      "step": 1600
+    },
+    {
+      "epoch": 2.56,
+      "eval_physics_entropy": 0.6755557696819305,
+      "eval_physics_loss": 1.2565526962280273,
+      "eval_physics_mean_token_accuracy": 0.7500922141075135,
+      "eval_physics_num_tokens": 45803783.0,
+      "eval_physics_runtime": 41.2091,
+      "eval_physics_samples_per_second": 12.133,
+      "eval_physics_steps_per_second": 3.033,
+      "step": 1600
+    },
+    {
+      "entropy": 0.47620510812848804,
+      "epoch": 2.576,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.6262962962962965e-05,
+      "loss": 0.4654,
+      "mean_token_accuracy": 0.8572851110249757,
+      "num_tokens": 46090713.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.4747265296056867,
+      "epoch": 2.592,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.6225925925925926e-05,
+      "loss": 0.4626,
+      "mean_token_accuracy": 0.8579647820442915,
+      "num_tokens": 46384084.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.49363710451871157,
+      "epoch": 2.608,
+      "grad_norm": 1.7890625,
+      "learning_rate": 1.6188888888888888e-05,
+      "loss": 0.4833,
+      "mean_token_accuracy": 0.8523125596344471,
+      "num_tokens": 46667552.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.46127278869971633,
+      "epoch": 2.624,
+      "grad_norm": 1.5546875,
+      "learning_rate": 1.6151851851851853e-05,
+      "loss": 0.446,
+      "mean_token_accuracy": 0.862875821441412,
+      "num_tokens": 46961124.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.4852636460214853,
+      "epoch": 2.64,
+      "grad_norm": 1.7109375,
+      "learning_rate": 1.6114814814814818e-05,
+      "loss": 0.4779,
+      "mean_token_accuracy": 0.8543730188161135,
+      "num_tokens": 47245326.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.46447359696030616,
+      "epoch": 2.656,
+      "grad_norm": 1.703125,
+      "learning_rate": 1.607777777777778e-05,
+      "loss": 0.4534,
+      "mean_token_accuracy": 0.8590674869716167,
+      "num_tokens": 47527347.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.4888186492025852,
+      "epoch": 2.672,
+      "grad_norm": 1.90625,
+      "learning_rate": 1.604074074074074e-05,
+      "loss": 0.474,
+      "mean_token_accuracy": 0.8537934329360723,
+      "num_tokens": 47808167.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.4809805376455188,
+      "epoch": 2.6879999999999997,
+      "grad_norm": 1.5,
+      "learning_rate": 1.6003703703703706e-05,
+      "loss": 0.4698,
+      "mean_token_accuracy": 0.8555871646851301,
+      "num_tokens": 48092035.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.487162452749908,
+      "epoch": 2.7039999999999997,
+      "grad_norm": 1.8125,
+      "learning_rate": 1.5966666666666667e-05,
+      "loss": 0.4786,
+      "mean_token_accuracy": 0.8542944025248289,
+      "num_tokens": 48386231.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.4787469983100891,
+      "epoch": 2.7199999999999998,
+      "grad_norm": 1.65625,
+      "learning_rate": 1.592962962962963e-05,
+      "loss": 0.4671,
+      "mean_token_accuracy": 0.8569595035165548,
+      "num_tokens": 48663262.0,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "eval_biology_entropy": 1.8445427660942078,
+      "eval_biology_loss": 3.6288812160491943,
+      "eval_biology_mean_token_accuracy": 0.49425346159934996,
+      "eval_biology_num_tokens": 48663262.0,
+      "eval_biology_runtime": 27.3715,
+      "eval_biology_samples_per_second": 18.267,
+      "eval_biology_steps_per_second": 4.567,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "eval_chemistry_entropy": 0.8915166838169097,
+      "eval_chemistry_loss": 1.822845697402954,
+      "eval_chemistry_mean_token_accuracy": 0.6868290622234344,
+      "eval_chemistry_num_tokens": 48663262.0,
+      "eval_chemistry_runtime": 34.5521,
+      "eval_chemistry_samples_per_second": 14.471,
+      "eval_chemistry_steps_per_second": 3.618,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "eval_math_entropy": 0.5080059399604797,
+      "eval_math_loss": 0.6736173033714294,
+      "eval_math_mean_token_accuracy": 0.8239876780509948,
+      "eval_math_num_tokens": 48663262.0,
+      "eval_math_runtime": 35.5751,
+      "eval_math_samples_per_second": 14.055,
+      "eval_math_steps_per_second": 3.514,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "eval_physics_entropy": 0.6629427807331085,
+      "eval_physics_loss": 1.2736350297927856,
+      "eval_physics_mean_token_accuracy": 0.7491478958129882,
+      "eval_physics_num_tokens": 48663262.0,
+      "eval_physics_runtime": 41.2042,
+      "eval_physics_samples_per_second": 12.135,
+      "eval_physics_steps_per_second": 3.034,
+      "step": 1700
+    },
+    {
+      "entropy": 0.47265797313302754,
+      "epoch": 2.7359999999999998,
+      "grad_norm": 1.71875,
+      "learning_rate": 1.5892592592592594e-05,
+      "loss": 0.4625,
+      "mean_token_accuracy": 0.8578159011900425,
+      "num_tokens": 48949456.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.483623811788857,
+      "epoch": 2.752,
+      "grad_norm": 1.5390625,
+      "learning_rate": 1.585555555555556e-05,
+      "loss": 0.4656,
+      "mean_token_accuracy": 0.8569592870771885,
+      "num_tokens": 49235751.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.48072876017540694,
+      "epoch": 2.768,
+      "grad_norm": 1.546875,
+      "learning_rate": 1.581851851851852e-05,
+      "loss": 0.474,
+      "mean_token_accuracy": 0.8559926971793175,
+      "num_tokens": 49524628.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.4934543133713305,
+      "epoch": 2.784,
+      "grad_norm": 1.65625,
+      "learning_rate": 1.5781481481481482e-05,
+      "loss": 0.4834,
+      "mean_token_accuracy": 0.8518608968704939,
+      "num_tokens": 49799604.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.48038765527307986,
+      "epoch": 2.8,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.5744444444444447e-05,
+      "loss": 0.4667,
+      "mean_token_accuracy": 0.8575652431696653,
+      "num_tokens": 50079307.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.4796597076579928,
+      "epoch": 2.816,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.570740740740741e-05,
+      "loss": 0.4643,
+      "mean_token_accuracy": 0.856653056293726,
+      "num_tokens": 50372216.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.4651268219575286,
+      "epoch": 2.832,
+      "grad_norm": 1.703125,
+      "learning_rate": 1.567037037037037e-05,
+      "loss": 0.4592,
+      "mean_token_accuracy": 0.8590575397014618,
+      "num_tokens": 50657211.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.49336096495389936,
+      "epoch": 2.848,
+      "grad_norm": 1.6328125,
+      "learning_rate": 1.5633333333333335e-05,
+      "loss": 0.4821,
+      "mean_token_accuracy": 0.8544768530875444,
+      "num_tokens": 50946503.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.48163238521665336,
+      "epoch": 2.864,
+      "grad_norm": 1.71875,
+      "learning_rate": 1.55962962962963e-05,
+      "loss": 0.4705,
+      "mean_token_accuracy": 0.8557115413248539,
+      "num_tokens": 51229298.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.4813537947833538,
+      "epoch": 2.88,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.555925925925926e-05,
+      "loss": 0.4664,
+      "mean_token_accuracy": 0.8561726022511721,
+      "num_tokens": 51509622.0,
+      "step": 1800
+    },
+    {
+      "epoch": 2.88,
+      "eval_biology_entropy": 1.8386799283027648,
+      "eval_biology_loss": 3.5641400814056396,
+      "eval_biology_mean_token_accuracy": 0.4960144295692444,
+      "eval_biology_num_tokens": 51509622.0,
+      "eval_biology_runtime": 27.3699,
+      "eval_biology_samples_per_second": 18.268,
+      "eval_biology_steps_per_second": 4.567,
+      "step": 1800
+    },
+    {
+      "epoch": 2.88,
+      "eval_chemistry_entropy": 0.8844058077335357,
+      "eval_chemistry_loss": 1.7807621955871582,
+      "eval_chemistry_mean_token_accuracy": 0.6901232223510743,
+      "eval_chemistry_num_tokens": 51509622.0,
+      "eval_chemistry_runtime": 34.5388,
+      "eval_chemistry_samples_per_second": 14.476,
+      "eval_chemistry_steps_per_second": 3.619,
+      "step": 1800
+    },
+    {
+      "epoch": 2.88,
+      "eval_math_entropy": 0.5032263789176941,
+      "eval_math_loss": 0.6734446883201599,
+      "eval_math_mean_token_accuracy": 0.8245284442901611,
+      "eval_math_num_tokens": 51509622.0,
+      "eval_math_runtime": 35.5836,
+      "eval_math_samples_per_second": 14.051,
+      "eval_math_steps_per_second": 3.513,
+      "step": 1800
+    },
+    {
+      "epoch": 2.88,
+      "eval_physics_entropy": 0.6657945175170898,
+      "eval_physics_loss": 1.2609224319458008,
+      "eval_physics_mean_token_accuracy": 0.7508005146980286,
+      "eval_physics_num_tokens": 51509622.0,
+      "eval_physics_runtime": 41.2236,
+      "eval_physics_samples_per_second": 12.129,
+      "eval_physics_steps_per_second": 3.032,
+      "step": 1800
+    },
+    {
+      "entropy": 0.4714194001629949,
+      "epoch": 2.896,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.5522222222222223e-05,
+      "loss": 0.4589,
+      "mean_token_accuracy": 0.8592868953943252,
+      "num_tokens": 51790856.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.47873496301472185,
+      "epoch": 2.912,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.5485185185185188e-05,
+      "loss": 0.469,
+      "mean_token_accuracy": 0.8562558207660913,
+      "num_tokens": 52074393.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.4847739338874817,
+      "epoch": 2.928,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.544814814814815e-05,
+      "loss": 0.4763,
+      "mean_token_accuracy": 0.8563588079065084,
+      "num_tokens": 52366467.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.4752591371536255,
+      "epoch": 2.944,
+      "grad_norm": 1.578125,
+      "learning_rate": 1.541111111111111e-05,
+      "loss": 0.4654,
+      "mean_token_accuracy": 0.8569884821772575,
+      "num_tokens": 52657863.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.4828122748062015,
+      "epoch": 2.96,
+      "grad_norm": 1.6328125,
+      "learning_rate": 1.5374074074074076e-05,
+      "loss": 0.4697,
+      "mean_token_accuracy": 0.8553634613752366,
+      "num_tokens": 52947183.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.47997692283242943,
+      "epoch": 2.976,
+      "grad_norm": 1.5859375,
+      "learning_rate": 1.5337037037037038e-05,
+      "loss": 0.4654,
+      "mean_token_accuracy": 0.856503576785326,
+      "num_tokens": 53226984.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.47029617708176374,
+      "epoch": 2.992,
+      "grad_norm": 1.65625,
+      "learning_rate": 1.5300000000000003e-05,
+      "loss": 0.4614,
+      "mean_token_accuracy": 0.8578217685222626,
+      "num_tokens": 53515204.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.4411141573451459,
+      "epoch": 3.008,
+      "grad_norm": 2.0,
+      "learning_rate": 1.5262962962962964e-05,
+      "loss": 0.3984,
+      "mean_token_accuracy": 0.8765936587005854,
+      "num_tokens": 53800592.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.3218042753636837,
+      "epoch": 3.024,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.5225925925925926e-05,
+      "loss": 0.3121,
+      "mean_token_accuracy": 0.8983496509492397,
+      "num_tokens": 54092017.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.30874418197199704,
+      "epoch": 3.04,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.518888888888889e-05,
+      "loss": 0.2901,
+      "mean_token_accuracy": 0.9049881529062986,
+      "num_tokens": 54380158.0,
+      "step": 1900
+    },
+    {
+      "epoch": 3.04,
+      "eval_biology_entropy": 1.2401256990432739,
+      "eval_biology_loss": 4.647972106933594,
+      "eval_biology_mean_token_accuracy": 0.4716033492088318,
+      "eval_biology_num_tokens": 54380158.0,
+      "eval_biology_runtime": 27.373,
+      "eval_biology_samples_per_second": 18.266,
+      "eval_biology_steps_per_second": 4.567,
+      "step": 1900
+    },
+    {
+      "epoch": 3.04,
+      "eval_chemistry_entropy": 0.6346060464382172,
+      "eval_chemistry_loss": 2.3420917987823486,
+      "eval_chemistry_mean_token_accuracy": 0.6706257765293121,
+      "eval_chemistry_num_tokens": 54380158.0,
+      "eval_chemistry_runtime": 34.5329,
+      "eval_chemistry_samples_per_second": 14.479,
+      "eval_chemistry_steps_per_second": 3.62,
+      "step": 1900
+    },
+    {
+      "epoch": 3.04,
+      "eval_math_entropy": 0.376160995721817,
+      "eval_math_loss": 0.8153116703033447,
+      "eval_math_mean_token_accuracy": 0.8162390570640564,
+      "eval_math_num_tokens": 54380158.0,
+      "eval_math_runtime": 35.5884,
+      "eval_math_samples_per_second": 14.05,
+      "eval_math_steps_per_second": 3.512,
+      "step": 1900
+    },
+    {
+      "epoch": 3.04,
+      "eval_physics_entropy": 0.47869771099090574,
+      "eval_physics_loss": 1.6276943683624268,
+      "eval_physics_mean_token_accuracy": 0.736794306755066,
+      "eval_physics_num_tokens": 54380158.0,
+      "eval_physics_runtime": 41.2039,
+      "eval_physics_samples_per_second": 12.135,
+      "eval_physics_steps_per_second": 3.034,
+      "step": 1900
+    },
+    {
+      "entropy": 0.3143508082255721,
+      "epoch": 3.056,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.5151851851851854e-05,
+      "loss": 0.2876,
+      "mean_token_accuracy": 0.9053623665124177,
+      "num_tokens": 54658093.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.307541798427701,
+      "epoch": 3.072,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.5114814814814815e-05,
+      "loss": 0.2877,
+      "mean_token_accuracy": 0.9054018415510654,
+      "num_tokens": 54949938.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.2924814156256616,
+      "epoch": 3.088,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.507777777777778e-05,
+      "loss": 0.274,
+      "mean_token_accuracy": 0.9092987533658743,
+      "num_tokens": 55234643.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.3124914604704827,
+      "epoch": 3.104,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.5040740740740742e-05,
+      "loss": 0.292,
+      "mean_token_accuracy": 0.9039953570812941,
+      "num_tokens": 55519679.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.30089363884180786,
+      "epoch": 3.12,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.5003703703703705e-05,
+      "loss": 0.2837,
+      "mean_token_accuracy": 0.9076365385204553,
+      "num_tokens": 55812397.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.3052088193595409,
+      "epoch": 3.136,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.4966666666666667e-05,
+      "loss": 0.2869,
+      "mean_token_accuracy": 0.9053549755364656,
+      "num_tokens": 56101550.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.30253817411139605,
+      "epoch": 3.152,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.4929629629629632e-05,
+      "loss": 0.2866,
+      "mean_token_accuracy": 0.9054879754781723,
+      "num_tokens": 56387662.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.31735899141058327,
+      "epoch": 3.168,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.4892592592592593e-05,
+      "loss": 0.2998,
+      "mean_token_accuracy": 0.9025642845779658,
+      "num_tokens": 56673426.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.31278759371489284,
+      "epoch": 3.184,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.4855555555555557e-05,
+      "loss": 0.2903,
+      "mean_token_accuracy": 0.9030884895473719,
+      "num_tokens": 56957044.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.3017499660141766,
+      "epoch": 3.2,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.4818518518518518e-05,
+      "loss": 0.2857,
+      "mean_token_accuracy": 0.9056338388472795,
+      "num_tokens": 57250097.0,
+      "step": 2000
+    },
+    {
+      "epoch": 3.2,
+      "eval_biology_entropy": 1.203611235141754,
+      "eval_biology_loss": 4.602008819580078,
+      "eval_biology_mean_token_accuracy": 0.47562575507164,
+      "eval_biology_num_tokens": 57250097.0,
+      "eval_biology_runtime": 27.361,
+      "eval_biology_samples_per_second": 18.274,
+      "eval_biology_steps_per_second": 4.569,
+      "step": 2000
+    },
+    {
+      "epoch": 3.2,
+      "eval_chemistry_entropy": 0.6066193521022797,
+      "eval_chemistry_loss": 2.3166801929473877,
+      "eval_chemistry_mean_token_accuracy": 0.6736224148273467,
+      "eval_chemistry_num_tokens": 57250097.0,
+      "eval_chemistry_runtime": 34.2481,
+      "eval_chemistry_samples_per_second": 14.599,
+      "eval_chemistry_steps_per_second": 3.65,
+      "step": 2000
+    },
+    {
+      "epoch": 3.2,
+      "eval_math_entropy": 0.365049503326416,
+      "eval_math_loss": 0.8271200656890869,
+      "eval_math_mean_token_accuracy": 0.8170590615272522,
+      "eval_math_num_tokens": 57250097.0,
+      "eval_math_runtime": 35.4885,
+      "eval_math_samples_per_second": 14.089,
+      "eval_math_steps_per_second": 3.522,
+      "step": 2000
+    },
+    {
+      "epoch": 3.2,
+      "eval_physics_entropy": 0.4583213438987732,
+      "eval_physics_loss": 1.6291862726211548,
+      "eval_physics_mean_token_accuracy": 0.7387176752090454,
+      "eval_physics_num_tokens": 57250097.0,
+      "eval_physics_runtime": 41.1914,
+      "eval_physics_samples_per_second": 12.138,
+      "eval_physics_steps_per_second": 3.035,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 6000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.343629994565034e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098d035725d0ae59ab1f5028bf93c2ff6d354179dc0d5be4e7abf77a5c4f5bea
+size 6417