Upload folder using huggingface_hub

.gitattributes

@@ -37,3 +37,4 @@ checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-2500/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-2500/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-2500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-2500/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-2500/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-2500/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9688dfaf1bb711551831675a233e0eabd3fba8693b35cf8932578bc4efafc928
+size 4962221464

checkpoint-2500/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4904cf3a18943b477f1222c91081d6bc50f821903f3fb9d755e67543e02dfce5
+size 3670322200

checkpoint-2500/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-2500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e59c931a527779bac1d4ee04cef0b179d949dc78ccccfdc2fc4a0f8f2f4adfce
+size 1585596266

checkpoint-2500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca7edf23f0cef681f30485dc71955f6a675cc4e0dbf22e0ed44c7452b774c627
+size 14645

checkpoint-2500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd200143f139327b4d41bcb99ffeaa13b2acc11ded5b3dbab181a6ff2ce855d7
+size 1465

checkpoint-2500/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-2500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-2500/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-2500/trainer_state.json

@@ -0,0 +1,3084 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 7.9888,
+  "eval_steps": 100,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.5645898684859274,
+      "epoch": 0.032,
+      "grad_norm": 252.0,
+      "learning_rate": 1.8e-07,
+      "loss": 6.6746,
+      "mean_token_accuracy": 0.2171937441918999,
+      "num_tokens": 166758.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.578542186319828,
+      "epoch": 0.064,
+      "grad_norm": 258.0,
+      "learning_rate": 3.8e-07,
+      "loss": 6.5875,
+      "mean_token_accuracy": 0.21883576279506087,
+      "num_tokens": 333231.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.558652176707983,
+      "epoch": 0.096,
+      "grad_norm": 264.0,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 6.4857,
+      "mean_token_accuracy": 0.21997002325952053,
+      "num_tokens": 494697.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.633940643817186,
+      "epoch": 0.128,
+      "grad_norm": 246.0,
+      "learning_rate": 7.8e-07,
+      "loss": 6.3208,
+      "mean_token_accuracy": 0.21828346541151405,
+      "num_tokens": 665636.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.767303283512592,
+      "epoch": 0.16,
+      "grad_norm": 190.0,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 6.0534,
+      "mean_token_accuracy": 0.21722873956896366,
+      "num_tokens": 831010.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.8346680492162704,
+      "epoch": 0.192,
+      "grad_norm": 144.0,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 5.3793,
+      "mean_token_accuracy": 0.23444932950660585,
+      "num_tokens": 997538.0,
+      "step": 60
+    },
+    {
+      "entropy": 2.941519719362259,
+      "epoch": 0.224,
+      "grad_norm": 82.0,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 4.7373,
+      "mean_token_accuracy": 0.2581239205785096,
+      "num_tokens": 1166249.0,
+      "step": 70
+    },
+    {
+      "entropy": 3.3048562452197077,
+      "epoch": 0.256,
+      "grad_norm": 54.5,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 4.2811,
+      "mean_token_accuracy": 0.2765548786148429,
+      "num_tokens": 1332878.0,
+      "step": 80
+    },
+    {
+      "entropy": 3.4035431027412413,
+      "epoch": 0.288,
+      "grad_norm": 35.75,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 3.8316,
+      "mean_token_accuracy": 0.314096855930984,
+      "num_tokens": 1499929.0,
+      "step": 90
+    },
+    {
+      "entropy": 3.354951748251915,
+      "epoch": 0.32,
+      "grad_norm": 24.375,
+      "learning_rate": 1.98e-06,
+      "loss": 3.5139,
+      "mean_token_accuracy": 0.3438800803385675,
+      "num_tokens": 1660639.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.32,
+      "eval_coding_entropy": 2.007658284664154,
+      "eval_coding_loss": 2.2495148181915283,
+      "eval_coding_mean_token_accuracy": 0.4928018909692764,
+      "eval_coding_num_tokens": 1660639.0,
+      "eval_coding_runtime": 79.8304,
+      "eval_coding_samples_per_second": 6.263,
+      "eval_coding_steps_per_second": 3.132,
+      "step": 100
+    },
+    {
+      "epoch": 0.32,
+      "eval_physics_entropy": 3.313183270454407,
+      "eval_physics_loss": 3.434373617172241,
+      "eval_physics_mean_token_accuracy": 0.34905624216794967,
+      "eval_physics_num_tokens": 1660639.0,
+      "eval_physics_runtime": 50.2026,
+      "eval_physics_samples_per_second": 9.96,
+      "eval_physics_steps_per_second": 4.98,
+      "step": 100
+    },
+    {
+      "entropy": 3.2520448341965675,
+      "epoch": 0.352,
+      "grad_norm": 23.25,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 3.302,
+      "mean_token_accuracy": 0.3621321846731007,
+      "num_tokens": 1827104.0,
+      "step": 110
+    },
+    {
+      "entropy": 3.0592679560184477,
+      "epoch": 0.384,
+      "grad_norm": 17.875,
+      "learning_rate": 2.38e-06,
+      "loss": 3.0918,
+      "mean_token_accuracy": 0.3877219457179308,
+      "num_tokens": 1991838.0,
+      "step": 120
+    },
+    {
+      "entropy": 2.9749809861183167,
+      "epoch": 0.416,
+      "grad_norm": 17.25,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 2.9574,
+      "mean_token_accuracy": 0.4056189233437181,
+      "num_tokens": 2160667.0,
+      "step": 130
+    },
+    {
+      "entropy": 2.849206253886223,
+      "epoch": 0.448,
+      "grad_norm": 16.0,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 2.8639,
+      "mean_token_accuracy": 0.42187846302986143,
+      "num_tokens": 2318749.0,
+      "step": 140
+    },
+    {
+      "entropy": 2.8139488607645036,
+      "epoch": 0.48,
+      "grad_norm": 17.125,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 2.7754,
+      "mean_token_accuracy": 0.42931156381964686,
+      "num_tokens": 2485149.0,
+      "step": 150
+    },
+    {
+      "entropy": 2.741229709237814,
+      "epoch": 0.512,
+      "grad_norm": 13.9375,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 2.7193,
+      "mean_token_accuracy": 0.44515763986855744,
+      "num_tokens": 2649860.0,
+      "step": 160
+    },
+    {
+      "entropy": 2.665092994272709,
+      "epoch": 0.544,
+      "grad_norm": 16.875,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 2.6856,
+      "mean_token_accuracy": 0.45115046463906766,
+      "num_tokens": 2808370.0,
+      "step": 170
+    },
+    {
+      "entropy": 2.5758446902036667,
+      "epoch": 0.576,
+      "grad_norm": 16.25,
+      "learning_rate": 3.58e-06,
+      "loss": 2.5455,
+      "mean_token_accuracy": 0.4649216070771217,
+      "num_tokens": 2983767.0,
+      "step": 180
+    },
+    {
+      "entropy": 2.5472018755972385,
+      "epoch": 0.608,
+      "grad_norm": 16.25,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 2.5247,
+      "mean_token_accuracy": 0.4711376942694187,
+      "num_tokens": 3155652.0,
+      "step": 190
+    },
+    {
+      "entropy": 2.5505440190434454,
+      "epoch": 0.64,
+      "grad_norm": 14.125,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 2.5257,
+      "mean_token_accuracy": 0.47651755474507806,
+      "num_tokens": 3318780.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_coding_entropy": 1.9238625903129578,
+      "eval_coding_loss": 2.138624429702759,
+      "eval_coding_mean_token_accuracy": 0.5144366272687912,
+      "eval_coding_num_tokens": 3318780.0,
+      "eval_coding_runtime": 78.6984,
+      "eval_coding_samples_per_second": 6.353,
+      "eval_coding_steps_per_second": 3.177,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_physics_entropy": 2.538371024131775,
+      "eval_physics_loss": 2.53389835357666,
+      "eval_physics_mean_token_accuracy": 0.4783249371051788,
+      "eval_physics_num_tokens": 3318780.0,
+      "eval_physics_runtime": 50.0865,
+      "eval_physics_samples_per_second": 9.983,
+      "eval_physics_steps_per_second": 4.991,
+      "step": 200
+    },
+    {
+      "entropy": 2.4629563339054585,
+      "epoch": 0.672,
+      "grad_norm": 14.375,
+      "learning_rate": 4.18e-06,
+      "loss": 2.4378,
+      "mean_token_accuracy": 0.4855367152020335,
+      "num_tokens": 3487395.0,
+      "step": 210
+    },
+    {
+      "entropy": 2.491329661011696,
+      "epoch": 0.704,
+      "grad_norm": 13.75,
+      "learning_rate": 4.38e-06,
+      "loss": 2.4681,
+      "mean_token_accuracy": 0.4864032404497266,
+      "num_tokens": 3651864.0,
+      "step": 220
+    },
+    {
+      "entropy": 2.4365530490875242,
+      "epoch": 0.736,
+      "grad_norm": 15.0,
+      "learning_rate": 4.58e-06,
+      "loss": 2.3642,
+      "mean_token_accuracy": 0.5005801904946565,
+      "num_tokens": 3818374.0,
+      "step": 230
+    },
+    {
+      "entropy": 2.372233358770609,
+      "epoch": 0.768,
+      "grad_norm": 13.1875,
+      "learning_rate": 4.78e-06,
+      "loss": 2.3883,
+      "mean_token_accuracy": 0.4903482008725405,
+      "num_tokens": 3988751.0,
+      "step": 240
+    },
+    {
+      "entropy": 2.4364359505474567,
+      "epoch": 0.8,
+      "grad_norm": 14.8125,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 2.3493,
+      "mean_token_accuracy": 0.5006071737036109,
+      "num_tokens": 4155352.0,
+      "step": 250
+    },
+    {
+      "entropy": 2.3030896626412867,
+      "epoch": 0.832,
+      "grad_norm": 15.625,
+      "learning_rate": 5.18e-06,
+      "loss": 2.2813,
+      "mean_token_accuracy": 0.5109015060588717,
+      "num_tokens": 4322729.0,
+      "step": 260
+    },
+    {
+      "entropy": 2.265988755226135,
+      "epoch": 0.864,
+      "grad_norm": 14.6875,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 2.2583,
+      "mean_token_accuracy": 0.5163270516321063,
+      "num_tokens": 4488640.0,
+      "step": 270
+    },
+    {
+      "entropy": 2.261782082915306,
+      "epoch": 0.896,
+      "grad_norm": 17.125,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 2.2071,
+      "mean_token_accuracy": 0.5210555041208863,
+      "num_tokens": 4652901.0,
+      "step": 280
+    },
+    {
+      "entropy": 2.253998390585184,
+      "epoch": 0.928,
+      "grad_norm": 14.5,
+      "learning_rate": 5.78e-06,
+      "loss": 2.2247,
+      "mean_token_accuracy": 0.5201520511880517,
+      "num_tokens": 4816150.0,
+      "step": 290
+    },
+    {
+      "entropy": 2.2820666775107386,
+      "epoch": 0.96,
+      "grad_norm": 18.875,
+      "learning_rate": 5.98e-06,
+      "loss": 2.2194,
+      "mean_token_accuracy": 0.5178691545501352,
+      "num_tokens": 4979460.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "eval_coding_entropy": 1.8525783290863038,
+      "eval_coding_loss": 2.0979971885681152,
+      "eval_coding_mean_token_accuracy": 0.5218188935518264,
+      "eval_coding_num_tokens": 4979460.0,
+      "eval_coding_runtime": 78.6898,
+      "eval_coding_samples_per_second": 6.354,
+      "eval_coding_steps_per_second": 3.177,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "eval_physics_entropy": 2.2524451808929444,
+      "eval_physics_loss": 2.2459053993225098,
+      "eval_physics_mean_token_accuracy": 0.5246009675264358,
+      "eval_physics_num_tokens": 4979460.0,
+      "eval_physics_runtime": 50.0344,
+      "eval_physics_samples_per_second": 9.993,
+      "eval_physics_steps_per_second": 4.997,
+      "step": 300
+    },
+    {
+      "entropy": 2.2345134645700453,
+      "epoch": 0.992,
+      "grad_norm": 15.375,
+      "learning_rate": 6.18e-06,
+      "loss": 2.1692,
+      "mean_token_accuracy": 0.5310839308425784,
+      "num_tokens": 5141579.0,
+      "step": 310
+    },
+    {
+      "entropy": 2.175699161855798,
+      "epoch": 1.0224,
+      "grad_norm": 15.25,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 2.1693,
+      "mean_token_accuracy": 0.5342735516789713,
+      "num_tokens": 5293853.0,
+      "step": 320
+    },
+    {
+      "entropy": 2.1913078941404818,
+      "epoch": 1.0544,
+      "grad_norm": 15.3125,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 2.1142,
+      "mean_token_accuracy": 0.5346984969452023,
+      "num_tokens": 5468879.0,
+      "step": 330
+    },
+    {
+      "entropy": 2.1974997885525225,
+      "epoch": 1.0864,
+      "grad_norm": 16.375,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 2.1502,
+      "mean_token_accuracy": 0.5345484068617224,
+      "num_tokens": 5631568.0,
+      "step": 340
+    },
+    {
+      "entropy": 2.174886829406023,
+      "epoch": 1.1184,
+      "grad_norm": 12.875,
+      "learning_rate": 6.98e-06,
+      "loss": 2.1314,
+      "mean_token_accuracy": 0.5390687845647335,
+      "num_tokens": 5791547.0,
+      "step": 350
+    },
+    {
+      "entropy": 2.075213722884655,
+      "epoch": 1.1504,
+      "grad_norm": 16.5,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 2.0332,
+      "mean_token_accuracy": 0.5553111273795366,
+      "num_tokens": 5956551.0,
+      "step": 360
+    },
+    {
+      "entropy": 2.0863427110016346,
+      "epoch": 1.1824,
+      "grad_norm": 15.0625,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 2.056,
+      "mean_token_accuracy": 0.5494146255776287,
+      "num_tokens": 6115916.0,
+      "step": 370
+    },
+    {
+      "entropy": 2.104805537313223,
+      "epoch": 1.2144,
+      "grad_norm": 15.875,
+      "learning_rate": 7.58e-06,
+      "loss": 2.0575,
+      "mean_token_accuracy": 0.5510358564555645,
+      "num_tokens": 6276148.0,
+      "step": 380
+    },
+    {
+      "entropy": 2.0606579631567,
+      "epoch": 1.2464,
+      "grad_norm": 13.875,
+      "learning_rate": 7.78e-06,
+      "loss": 2.0071,
+      "mean_token_accuracy": 0.5626341335475444,
+      "num_tokens": 6439971.0,
+      "step": 390
+    },
+    {
+      "entropy": 2.0226216495037077,
+      "epoch": 1.2784,
+      "grad_norm": 17.125,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 1.9925,
+      "mean_token_accuracy": 0.5558732001110911,
+      "num_tokens": 6604463.0,
+      "step": 400
+    },
+    {
+      "epoch": 1.2784,
+      "eval_coding_entropy": 1.8207996063232421,
+      "eval_coding_loss": 2.0917093753814697,
+      "eval_coding_mean_token_accuracy": 0.5248159831762313,
+      "eval_coding_num_tokens": 6604463.0,
+      "eval_coding_runtime": 79.178,
+      "eval_coding_samples_per_second": 6.315,
+      "eval_coding_steps_per_second": 3.157,
+      "step": 400
+    },
+    {
+      "epoch": 1.2784,
+      "eval_physics_entropy": 2.0453660078048705,
+      "eval_physics_loss": 2.0581541061401367,
+      "eval_physics_mean_token_accuracy": 0.5561744419336319,
+      "eval_physics_num_tokens": 6604463.0,
+      "eval_physics_runtime": 50.5085,
+      "eval_physics_samples_per_second": 9.899,
+      "eval_physics_steps_per_second": 4.95,
+      "step": 400
+    },
+    {
+      "entropy": 2.0329784117639065,
+      "epoch": 1.3104,
+      "grad_norm": 15.0,
+      "learning_rate": 8.18e-06,
+      "loss": 1.9612,
+      "mean_token_accuracy": 0.5634729105979204,
+      "num_tokens": 6769215.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.9791719630360602,
+      "epoch": 1.3424,
+      "grad_norm": 26.25,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 1.943,
+      "mean_token_accuracy": 0.5674662893638015,
+      "num_tokens": 6937523.0,
+      "step": 420
+    },
+    {
+      "entropy": 2.0105444215238095,
+      "epoch": 1.3744,
+      "grad_norm": 18.375,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 1.9751,
+      "mean_token_accuracy": 0.5621760170906782,
+      "num_tokens": 7103833.0,
+      "step": 430
+    },
+    {
+      "entropy": 2.0342012770473956,
+      "epoch": 1.4064,
+      "grad_norm": 18.0,
+      "learning_rate": 8.78e-06,
+      "loss": 1.9921,
+      "mean_token_accuracy": 0.5628394693136215,
+      "num_tokens": 7266794.0,
+      "step": 440
+    },
+    {
+      "entropy": 2.0377430960536005,
+      "epoch": 1.4384000000000001,
+      "grad_norm": 15.25,
+      "learning_rate": 8.98e-06,
+      "loss": 1.9897,
+      "mean_token_accuracy": 0.5596502210944891,
+      "num_tokens": 7438987.0,
+      "step": 450
+    },
+    {
+      "entropy": 2.0356026656925676,
+      "epoch": 1.4704,
+      "grad_norm": 15.0625,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 1.95,
+      "mean_token_accuracy": 0.5652700768783688,
+      "num_tokens": 7604024.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.8638560153543948,
+      "epoch": 1.5024,
+      "grad_norm": 14.6875,
+      "learning_rate": 9.38e-06,
+      "loss": 1.8501,
+      "mean_token_accuracy": 0.5885478794574738,
+      "num_tokens": 7772263.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.910068366676569,
+      "epoch": 1.5344,
+      "grad_norm": 13.4375,
+      "learning_rate": 9.58e-06,
+      "loss": 1.862,
+      "mean_token_accuracy": 0.5832355048507452,
+      "num_tokens": 7944833.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.91564781665802,
+      "epoch": 1.5664,
+      "grad_norm": 15.75,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 1.8632,
+      "mean_token_accuracy": 0.5840621557086706,
+      "num_tokens": 8107703.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.8383589215576648,
+      "epoch": 1.5984,
+      "grad_norm": 18.0,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 1.8157,
+      "mean_token_accuracy": 0.5885938903316855,
+      "num_tokens": 8272928.0,
+      "step": 500
+    },
+    {
+      "epoch": 1.5984,
+      "eval_coding_entropy": 1.8546455001831055,
+      "eval_coding_loss": 2.0867998600006104,
+      "eval_coding_mean_token_accuracy": 0.5256605116128922,
+      "eval_coding_num_tokens": 8272928.0,
+      "eval_coding_runtime": 78.6681,
+      "eval_coding_samples_per_second": 6.356,
+      "eval_coding_steps_per_second": 3.178,
+      "step": 500
+    },
+    {
+      "epoch": 1.5984,
+      "eval_physics_entropy": 1.9303974876403809,
+      "eval_physics_loss": 1.9122997522354126,
+      "eval_physics_mean_token_accuracy": 0.5807216377258301,
+      "eval_physics_num_tokens": 8272928.0,
+      "eval_physics_runtime": 49.8483,
+      "eval_physics_samples_per_second": 10.03,
+      "eval_physics_steps_per_second": 5.015,
+      "step": 500
+    },
+    {
+      "entropy": 1.8996943302452565,
+      "epoch": 1.6303999999999998,
+      "grad_norm": 13.5625,
+      "learning_rate": 1.018e-05,
+      "loss": 1.8527,
+      "mean_token_accuracy": 0.5799180708825589,
+      "num_tokens": 8436701.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.923023121803999,
+      "epoch": 1.6623999999999999,
+      "grad_norm": 16.75,
+      "learning_rate": 1.038e-05,
+      "loss": 1.8692,
+      "mean_token_accuracy": 0.5779866192489862,
+      "num_tokens": 8601859.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.9147369302809238,
+      "epoch": 1.6944,
+      "grad_norm": 14.6875,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 1.8551,
+      "mean_token_accuracy": 0.5828457817435264,
+      "num_tokens": 8770215.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.7832183502614498,
+      "epoch": 1.7264,
+      "grad_norm": 15.1875,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 1.7598,
+      "mean_token_accuracy": 0.5977033786475658,
+      "num_tokens": 8943490.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.815915060043335,
+      "epoch": 1.7584,
+      "grad_norm": 13.3125,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 1.7421,
+      "mean_token_accuracy": 0.5974750472232699,
+      "num_tokens": 9105492.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.768533306568861,
+      "epoch": 1.7904,
+      "grad_norm": 13.0,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 1.7205,
+      "mean_token_accuracy": 0.6018104804679751,
+      "num_tokens": 9279120.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.805129376798868,
+      "epoch": 1.8224,
+      "grad_norm": 14.25,
+      "learning_rate": 1.138e-05,
+      "loss": 1.7643,
+      "mean_token_accuracy": 0.5969077125191689,
+      "num_tokens": 9449267.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.786954965442419,
+      "epoch": 1.8544,
+      "grad_norm": 24.25,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 1.75,
+      "mean_token_accuracy": 0.6012308387085795,
+      "num_tokens": 9617568.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.7876968085765839,
+      "epoch": 1.8864,
+      "grad_norm": 16.625,
+      "learning_rate": 1.178e-05,
+      "loss": 1.7478,
+      "mean_token_accuracy": 0.6033026060089469,
+      "num_tokens": 9784216.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.7894642509520053,
+      "epoch": 1.9184,
+      "grad_norm": 13.25,
+      "learning_rate": 1.198e-05,
+      "loss": 1.7256,
+      "mean_token_accuracy": 0.6044006902724505,
+      "num_tokens": 9943304.0,
+      "step": 600
+    },
+    {
+      "epoch": 1.9184,
+      "eval_coding_entropy": 1.809903567314148,
+      "eval_coding_loss": 2.097890853881836,
+      "eval_coding_mean_token_accuracy": 0.5257191739082336,
+      "eval_coding_num_tokens": 9943304.0,
+      "eval_coding_runtime": 78.9114,
+      "eval_coding_samples_per_second": 6.336,
+      "eval_coding_steps_per_second": 3.168,
+      "step": 600
+    },
+    {
+      "epoch": 1.9184,
+      "eval_physics_entropy": 1.7725872592926026,
+      "eval_physics_loss": 1.7945505380630493,
+      "eval_physics_mean_token_accuracy": 0.6027934015989304,
+      "eval_physics_num_tokens": 9943304.0,
+      "eval_physics_runtime": 50.0049,
+      "eval_physics_samples_per_second": 9.999,
+      "eval_physics_steps_per_second": 5.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.7505566235631704,
+      "epoch": 1.9504000000000001,
+      "grad_norm": 14.125,
+      "learning_rate": 1.218e-05,
+      "loss": 1.7226,
+      "mean_token_accuracy": 0.6070320427417755,
+      "num_tokens": 10112076.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.7650270752608777,
+      "epoch": 1.9824000000000002,
+      "grad_norm": 15.5625,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 1.7303,
+      "mean_token_accuracy": 0.6072711888700724,
+      "num_tokens": 10270035.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.753685184999516,
+      "epoch": 2.0128,
+      "grad_norm": 14.1875,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 1.6964,
+      "mean_token_accuracy": 0.6074773392787105,
+      "num_tokens": 10424317.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.7055613581091165,
+      "epoch": 2.0448,
+      "grad_norm": 14.125,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 1.6594,
+      "mean_token_accuracy": 0.616450079344213,
+      "num_tokens": 10596317.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.681853237003088,
+      "epoch": 2.0768,
+      "grad_norm": 12.5,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 1.6309,
+      "mean_token_accuracy": 0.622531720623374,
+      "num_tokens": 10761453.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.6445298943668605,
+      "epoch": 2.1088,
+      "grad_norm": 15.4375,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 1.6113,
+      "mean_token_accuracy": 0.6273578329011797,
+      "num_tokens": 10921701.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.7012917585670948,
+      "epoch": 2.1408,
+      "grad_norm": 15.75,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 1.6472,
+      "mean_token_accuracy": 0.616516831330955,
+      "num_tokens": 11087232.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.6926489636301993,
+      "epoch": 2.1728,
+      "grad_norm": 14.8125,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 1.6416,
+      "mean_token_accuracy": 0.6202321622520686,
+      "num_tokens": 11253164.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.6239103347063064,
+      "epoch": 2.2048,
+      "grad_norm": 14.3125,
+      "learning_rate": 1.378e-05,
+      "loss": 1.5771,
+      "mean_token_accuracy": 0.6309411523863673,
+      "num_tokens": 11421725.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.6312100168317556,
+      "epoch": 2.2368,
+      "grad_norm": 17.5,
+      "learning_rate": 1.398e-05,
+      "loss": 1.6106,
+      "mean_token_accuracy": 0.6250216152518988,
+      "num_tokens": 11585449.0,
+      "step": 700
+    },
+    {
+      "epoch": 2.2368,
+      "eval_coding_entropy": 1.7832098503112792,
+      "eval_coding_loss": 2.109489679336548,
+      "eval_coding_mean_token_accuracy": 0.525750536441803,
+      "eval_coding_num_tokens": 11585449.0,
+      "eval_coding_runtime": 78.6079,
+      "eval_coding_samples_per_second": 6.361,
+      "eval_coding_steps_per_second": 3.18,
+      "step": 700
+    },
+    {
+      "epoch": 2.2368,
+      "eval_physics_entropy": 1.6392567100524902,
+      "eval_physics_loss": 1.7045031785964966,
+      "eval_physics_mean_token_accuracy": 0.6181177039146424,
+      "eval_physics_num_tokens": 11585449.0,
+      "eval_physics_runtime": 49.9995,
+      "eval_physics_samples_per_second": 10.0,
+      "eval_physics_steps_per_second": 5.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.6536100145429373,
+      "epoch": 2.2688,
+      "grad_norm": 12.625,
+      "learning_rate": 1.418e-05,
+      "loss": 1.6265,
+      "mean_token_accuracy": 0.626517367362976,
+      "num_tokens": 11754276.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.6963056311011315,
+      "epoch": 2.3008,
+      "grad_norm": 19.625,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 1.6336,
+      "mean_token_accuracy": 0.6203271795064211,
+      "num_tokens": 11914373.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.6301159981638194,
+      "epoch": 2.3327999999999998,
+      "grad_norm": 16.0,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 1.5862,
+      "mean_token_accuracy": 0.6303246023133398,
+      "num_tokens": 12075851.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.5643031619489194,
+      "epoch": 2.3648,
+      "grad_norm": 17.125,
+      "learning_rate": 1.478e-05,
+      "loss": 1.5157,
+      "mean_token_accuracy": 0.6383260345086456,
+      "num_tokens": 12243196.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.6507456723600626,
+      "epoch": 2.3968,
+      "grad_norm": 15.375,
+      "learning_rate": 1.498e-05,
+      "loss": 1.6088,
+      "mean_token_accuracy": 0.6292662421241403,
+      "num_tokens": 12408334.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.5932432316243648,
+      "epoch": 2.4288,
+      "grad_norm": 17.25,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 1.569,
+      "mean_token_accuracy": 0.6341303730383515,
+      "num_tokens": 12577925.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.5955147668719292,
+      "epoch": 2.4608,
+      "grad_norm": 12.5,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 1.556,
+      "mean_token_accuracy": 0.633737426996231,
+      "num_tokens": 12747689.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.6611970141530037,
+      "epoch": 2.4928,
+      "grad_norm": 13.375,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 1.6333,
+      "mean_token_accuracy": 0.6246412046253681,
+      "num_tokens": 12911304.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.5851813267916441,
+      "epoch": 2.5248,
+      "grad_norm": 13.875,
+      "learning_rate": 1.578e-05,
+      "loss": 1.5437,
+      "mean_token_accuracy": 0.6365990586578846,
+      "num_tokens": 13077988.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.5275409139692784,
+      "epoch": 2.5568,
+      "grad_norm": 15.625,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 1.4801,
+      "mean_token_accuracy": 0.6462847730144858,
+      "num_tokens": 13251116.0,
+      "step": 800
+    },
+    {
+      "epoch": 2.5568,
+      "eval_coding_entropy": 1.8128439927101134,
+      "eval_coding_loss": 2.1043496131896973,
+      "eval_coding_mean_token_accuracy": 0.5263604500293732,
+      "eval_coding_num_tokens": 13251116.0,
+      "eval_coding_runtime": 78.6306,
+      "eval_coding_samples_per_second": 6.359,
+      "eval_coding_steps_per_second": 3.179,
+      "step": 800
+    },
+    {
+      "epoch": 2.5568,
+      "eval_physics_entropy": 1.6093163886070252,
+      "eval_physics_loss": 1.6318042278289795,
+      "eval_physics_mean_token_accuracy": 0.6309541233778,
+      "eval_physics_num_tokens": 13251116.0,
+      "eval_physics_runtime": 49.9907,
+      "eval_physics_samples_per_second": 10.002,
+      "eval_physics_steps_per_second": 5.001,
+      "step": 800
+    },
+    {
+      "entropy": 1.5732625368982553,
+      "epoch": 2.5888,
+      "grad_norm": 14.25,
+      "learning_rate": 1.618e-05,
+      "loss": 1.5149,
+      "mean_token_accuracy": 0.6410752791911364,
+      "num_tokens": 13415037.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.5751493390649558,
+      "epoch": 2.6208,
+      "grad_norm": 14.9375,
+      "learning_rate": 1.638e-05,
+      "loss": 1.5492,
+      "mean_token_accuracy": 0.6390313066542148,
+      "num_tokens": 13579275.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.580653928220272,
+      "epoch": 2.6528,
+      "grad_norm": 15.4375,
+      "learning_rate": 1.658e-05,
+      "loss": 1.5522,
+      "mean_token_accuracy": 0.6373755222186446,
+      "num_tokens": 13743818.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.555862707272172,
+      "epoch": 2.6848,
+      "grad_norm": 13.75,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 1.4989,
+      "mean_token_accuracy": 0.6430002739652991,
+      "num_tokens": 13906946.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.627287209033966,
+      "epoch": 2.7168,
+      "grad_norm": 14.5625,
+      "learning_rate": 1.698e-05,
+      "loss": 1.5775,
+      "mean_token_accuracy": 0.6343628978356719,
+      "num_tokens": 14075222.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.5674079015851021,
+      "epoch": 2.7488,
+      "grad_norm": 13.0625,
+      "learning_rate": 1.718e-05,
+      "loss": 1.5228,
+      "mean_token_accuracy": 0.6411064675077796,
+      "num_tokens": 14239601.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.5571764077991248,
+      "epoch": 2.7808,
+      "grad_norm": 13.3125,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 1.5286,
+      "mean_token_accuracy": 0.6401644449681043,
+      "num_tokens": 14408376.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.5362198933959008,
+      "epoch": 2.8128,
+      "grad_norm": 13.9375,
+      "learning_rate": 1.758e-05,
+      "loss": 1.4874,
+      "mean_token_accuracy": 0.6457877155393362,
+      "num_tokens": 14564818.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.528880115225911,
+      "epoch": 2.8448,
+      "grad_norm": 14.9375,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 1.4727,
+      "mean_token_accuracy": 0.6474113451316953,
+      "num_tokens": 14732642.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.5470367703586816,
+      "epoch": 2.8768000000000002,
+      "grad_norm": 15.0625,
+      "learning_rate": 1.798e-05,
+      "loss": 1.5178,
+      "mean_token_accuracy": 0.6419060477986932,
+      "num_tokens": 14906192.0,
+      "step": 900
+    },
+    {
+      "epoch": 2.8768000000000002,
+      "eval_coding_entropy": 1.8220215735435485,
+      "eval_coding_loss": 2.1269266605377197,
+      "eval_coding_mean_token_accuracy": 0.5220261276960373,
+      "eval_coding_num_tokens": 14906192.0,
+      "eval_coding_runtime": 79.0129,
+      "eval_coding_samples_per_second": 6.328,
+      "eval_coding_steps_per_second": 3.164,
+      "step": 900
+    },
+    {
+      "epoch": 2.8768000000000002,
+      "eval_physics_entropy": 1.5491982371807098,
+      "eval_physics_loss": 1.5605218410491943,
+      "eval_physics_mean_token_accuracy": 0.6426789830923081,
+      "eval_physics_num_tokens": 14906192.0,
+      "eval_physics_runtime": 50.7438,
+      "eval_physics_samples_per_second": 9.853,
+      "eval_physics_steps_per_second": 4.927,
+      "step": 900
+    },
+    {
+      "entropy": 1.5086696345359087,
+      "epoch": 2.9088000000000003,
+      "grad_norm": 13.4375,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 1.4592,
+      "mean_token_accuracy": 0.6512332953512668,
+      "num_tokens": 15070320.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.5167599759995938,
+      "epoch": 2.9408,
+      "grad_norm": 19.875,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 1.4681,
+      "mean_token_accuracy": 0.6510875273495913,
+      "num_tokens": 15231317.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.5221595961600543,
+      "epoch": 2.9728,
+      "grad_norm": 13.375,
+      "learning_rate": 1.858e-05,
+      "loss": 1.4812,
+      "mean_token_accuracy": 0.6497884158045053,
+      "num_tokens": 15396833.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.518610822919168,
+      "epoch": 3.0032,
+      "grad_norm": 13.4375,
+      "learning_rate": 1.878e-05,
+      "loss": 1.4657,
+      "mean_token_accuracy": 0.6544288014502901,
+      "num_tokens": 15557192.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.4148433946073056,
+      "epoch": 3.0352,
+      "grad_norm": 12.5,
+      "learning_rate": 1.898e-05,
+      "loss": 1.3688,
+      "mean_token_accuracy": 0.668708611652255,
+      "num_tokens": 15722813.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.4289791829884053,
+      "epoch": 3.0672,
+      "grad_norm": 13.5625,
+      "learning_rate": 1.918e-05,
+      "loss": 1.3747,
+      "mean_token_accuracy": 0.6649635829031467,
+      "num_tokens": 15886622.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.4406539868563413,
+      "epoch": 3.0992,
+      "grad_norm": 11.6875,
+      "learning_rate": 1.938e-05,
+      "loss": 1.3811,
+      "mean_token_accuracy": 0.6637488787993788,
+      "num_tokens": 16049773.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.3688473895192146,
+      "epoch": 3.1312,
+      "grad_norm": 12.3125,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 1.3291,
+      "mean_token_accuracy": 0.6713310034945608,
+      "num_tokens": 16216938.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.4295187518000603,
+      "epoch": 3.1632,
+      "grad_norm": 11.875,
+      "learning_rate": 1.978e-05,
+      "loss": 1.3984,
+      "mean_token_accuracy": 0.662257743254304,
+      "num_tokens": 16384680.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.4126375205814838,
+      "epoch": 3.1952,
+      "grad_norm": 12.1875,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 1.3643,
+      "mean_token_accuracy": 0.6724214576184749,
+      "num_tokens": 16560897.0,
+      "step": 1000
+    },
+    {
+      "epoch": 3.1952,
+      "eval_coding_entropy": 1.7359957580566405,
+      "eval_coding_loss": 2.143160820007324,
+      "eval_coding_mean_token_accuracy": 0.523926413655281,
+      "eval_coding_num_tokens": 16560897.0,
+      "eval_coding_runtime": 78.6123,
+      "eval_coding_samples_per_second": 6.36,
+      "eval_coding_steps_per_second": 3.18,
+      "step": 1000
+    },
+    {
+      "epoch": 3.1952,
+      "eval_physics_entropy": 1.422654648065567,
+      "eval_physics_loss": 1.5027931928634644,
+      "eval_physics_mean_token_accuracy": 0.6541903501749039,
+      "eval_physics_num_tokens": 16560897.0,
+      "eval_physics_runtime": 50.109,
+      "eval_physics_samples_per_second": 9.978,
+      "eval_physics_steps_per_second": 4.989,
+      "step": 1000
+    },
+    {
+      "entropy": 1.4092690028250217,
+      "epoch": 3.2272,
+      "grad_norm": 12.0,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 1.3786,
+      "mean_token_accuracy": 0.6692744750529528,
+      "num_tokens": 16723316.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.4263327240943908,
+      "epoch": 3.2592,
+      "grad_norm": 13.4375,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 1.3918,
+      "mean_token_accuracy": 0.6654507229104638,
+      "num_tokens": 16889599.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.3592466082423926,
+      "epoch": 3.2912,
+      "grad_norm": 15.4375,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 1.3077,
+      "mean_token_accuracy": 0.6776290779933334,
+      "num_tokens": 17062342.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.3674316611140966,
+      "epoch": 3.3232,
+      "grad_norm": 11.5625,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 1.3381,
+      "mean_token_accuracy": 0.6765869384631514,
+      "num_tokens": 17230975.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.4023310970515013,
+      "epoch": 3.3552,
+      "grad_norm": 9.875,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 1.368,
+      "mean_token_accuracy": 0.6705853873863816,
+      "num_tokens": 17402449.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.3861599292606115,
+      "epoch": 3.3872,
+      "grad_norm": 11.5,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 1.3472,
+      "mean_token_accuracy": 0.6767404664307832,
+      "num_tokens": 17574961.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.3845525227487088,
+      "epoch": 3.4192,
+      "grad_norm": 12.3125,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 1.3515,
+      "mean_token_accuracy": 0.6734926411882043,
+      "num_tokens": 17741134.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.3781002704054117,
+      "epoch": 3.4512,
+      "grad_norm": 10.875,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 1.3259,
+      "mean_token_accuracy": 0.677556062489748,
+      "num_tokens": 17902781.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.377650147676468,
+      "epoch": 3.4832,
+      "grad_norm": 13.8125,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 1.3571,
+      "mean_token_accuracy": 0.67311130464077,
+      "num_tokens": 18067791.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.359330151602626,
+      "epoch": 3.5152,
+      "grad_norm": 12.875,
+      "learning_rate": 1.978e-05,
+      "loss": 1.3247,
+      "mean_token_accuracy": 0.6772076480090619,
+      "num_tokens": 18235991.0,
+      "step": 1100
+    },
+    {
+      "epoch": 3.5152,
+      "eval_coding_entropy": 1.7204444952011109,
+      "eval_coding_loss": 2.1535234451293945,
+      "eval_coding_mean_token_accuracy": 0.5223781074285507,
+      "eval_coding_num_tokens": 18235991.0,
+      "eval_coding_runtime": 78.6984,
+      "eval_coding_samples_per_second": 6.353,
+      "eval_coding_steps_per_second": 3.177,
+      "step": 1100
+    },
+    {
+      "epoch": 3.5152,
+      "eval_physics_entropy": 1.3494070060253143,
+      "eval_physics_loss": 1.442825436592102,
+      "eval_physics_mean_token_accuracy": 0.6649874558448792,
+      "eval_physics_num_tokens": 18235991.0,
+      "eval_physics_runtime": 49.9996,
+      "eval_physics_samples_per_second": 10.0,
+      "eval_physics_steps_per_second": 5.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.3790622040629388,
+      "epoch": 3.5472,
+      "grad_norm": 12.375,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 1.3323,
+      "mean_token_accuracy": 0.673924190364778,
+      "num_tokens": 18401729.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.4172765392810107,
+      "epoch": 3.5792,
+      "grad_norm": 12.6875,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 1.3535,
+      "mean_token_accuracy": 0.6714193534106017,
+      "num_tokens": 18561781.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.3661954671144485,
+      "epoch": 3.6112,
+      "grad_norm": 11.125,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 1.3227,
+      "mean_token_accuracy": 0.676239874213934,
+      "num_tokens": 18729784.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.3301920879632234,
+      "epoch": 3.6432,
+      "grad_norm": 11.375,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 1.2935,
+      "mean_token_accuracy": 0.6833865344524384,
+      "num_tokens": 18889241.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.3433120604604483,
+      "epoch": 3.6752000000000002,
+      "grad_norm": 10.125,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 1.3032,
+      "mean_token_accuracy": 0.6798774655908346,
+      "num_tokens": 19048271.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.344341028854251,
+      "epoch": 3.7072000000000003,
+      "grad_norm": 11.9375,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 1.3114,
+      "mean_token_accuracy": 0.681334874778986,
+      "num_tokens": 19217080.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.3379347380250692,
+      "epoch": 3.7392,
+      "grad_norm": 11.8125,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 1.3028,
+      "mean_token_accuracy": 0.6815936049446464,
+      "num_tokens": 19374677.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.3498164001852273,
+      "epoch": 3.7712,
+      "grad_norm": 10.875,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 1.315,
+      "mean_token_accuracy": 0.6794227551668882,
+      "num_tokens": 19533723.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.3038409043103456,
+      "epoch": 3.8032,
+      "grad_norm": 11.625,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 1.2672,
+      "mean_token_accuracy": 0.6881153093650937,
+      "num_tokens": 19700672.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.3255383513867856,
+      "epoch": 3.8352,
+      "grad_norm": 12.5,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 1.2741,
+      "mean_token_accuracy": 0.6877249713987112,
+      "num_tokens": 19871261.0,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8352,
+      "eval_coding_entropy": 1.751271043777466,
+      "eval_coding_loss": 2.1689281463623047,
+      "eval_coding_mean_token_accuracy": 0.5190993611812591,
+      "eval_coding_num_tokens": 19871261.0,
+      "eval_coding_runtime": 78.6492,
+      "eval_coding_samples_per_second": 6.357,
+      "eval_coding_steps_per_second": 3.179,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8352,
+      "eval_physics_entropy": 1.3656754405498506,
+      "eval_physics_loss": 1.3909977674484253,
+      "eval_physics_mean_token_accuracy": 0.6745351340770721,
+      "eval_physics_num_tokens": 19871261.0,
+      "eval_physics_runtime": 50.0108,
+      "eval_physics_samples_per_second": 9.998,
+      "eval_physics_steps_per_second": 4.999,
+      "step": 1200
+    },
+    {
+      "entropy": 1.281702034920454,
+      "epoch": 3.8672,
+      "grad_norm": 9.625,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 1.2496,
+      "mean_token_accuracy": 0.6916576270014048,
+      "num_tokens": 20036482.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.3664431843906641,
+      "epoch": 3.8992,
+      "grad_norm": 10.1875,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 1.3061,
+      "mean_token_accuracy": 0.6760184271261096,
+      "num_tokens": 20200878.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.3086781930178404,
+      "epoch": 3.9312,
+      "grad_norm": 10.125,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 1.2591,
+      "mean_token_accuracy": 0.6873132940381765,
+      "num_tokens": 20369285.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.2660665262490511,
+      "epoch": 3.9632,
+      "grad_norm": 10.5625,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 1.2197,
+      "mean_token_accuracy": 0.6949825417250395,
+      "num_tokens": 20532097.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.3122623603791,
+      "epoch": 3.9952,
+      "grad_norm": 10.5,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 1.2684,
+      "mean_token_accuracy": 0.6880532449111343,
+      "num_tokens": 20698863.0,
+      "step": 1250
+    },
+    {
+      "entropy": 1.215138852204147,
+      "epoch": 4.0256,
+      "grad_norm": 9.25,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 1.1788,
+      "mean_token_accuracy": 0.7075269367349776,
+      "num_tokens": 20857766.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.162597357854247,
+      "epoch": 4.0576,
+      "grad_norm": 9.625,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 1.1318,
+      "mean_token_accuracy": 0.7138494092971086,
+      "num_tokens": 21026679.0,
+      "step": 1270
+    },
+    {
+      "entropy": 1.206483306735754,
+      "epoch": 4.0896,
+      "grad_norm": 9.8125,
+      "learning_rate": 1.938e-05,
+      "loss": 1.16,
+      "mean_token_accuracy": 0.7069427033886313,
+      "num_tokens": 21186241.0,
+      "step": 1280
+    },
+    {
+      "entropy": 1.205837368220091,
+      "epoch": 4.1216,
+      "grad_norm": 9.4375,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 1.1551,
+      "mean_token_accuracy": 0.7110590524971485,
+      "num_tokens": 21350399.0,
+      "step": 1290
+    },
+    {
+      "entropy": 1.199402868375182,
+      "epoch": 4.1536,
+      "grad_norm": 10.0625,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 1.1589,
+      "mean_token_accuracy": 0.7075492311269045,
+      "num_tokens": 21518243.0,
+      "step": 1300
+    },
+    {
+      "epoch": 4.1536,
+      "eval_coding_entropy": 1.5845919437408447,
+      "eval_coding_loss": 2.2161450386047363,
+      "eval_coding_mean_token_accuracy": 0.519056293129921,
+      "eval_coding_num_tokens": 21518243.0,
+      "eval_coding_runtime": 78.6572,
+      "eval_coding_samples_per_second": 6.357,
+      "eval_coding_steps_per_second": 3.178,
+      "step": 1300
+    },
+    {
+      "epoch": 4.1536,
+      "eval_physics_entropy": 1.161534303188324,
+      "eval_physics_loss": 1.362545371055603,
+      "eval_physics_mean_token_accuracy": 0.6832804265022278,
+      "eval_physics_num_tokens": 21518243.0,
+      "eval_physics_runtime": 50.3764,
+      "eval_physics_samples_per_second": 9.925,
+      "eval_physics_steps_per_second": 4.963,
+      "step": 1300
+    },
+    {
+      "entropy": 1.187236387655139,
+      "epoch": 4.1856,
+      "grad_norm": 9.8125,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 1.1492,
+      "mean_token_accuracy": 0.7097648229449988,
+      "num_tokens": 21677750.0,
+      "step": 1310
+    },
+    {
+      "entropy": 1.1768355205655099,
+      "epoch": 4.2176,
+      "grad_norm": 9.75,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 1.1161,
+      "mean_token_accuracy": 0.7118838958442211,
+      "num_tokens": 21845173.0,
+      "step": 1320
+    },
+    {
+      "entropy": 1.1649330925196408,
+      "epoch": 4.2496,
+      "grad_norm": 9.5625,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 1.1095,
+      "mean_token_accuracy": 0.7167212620377541,
+      "num_tokens": 22009137.0,
+      "step": 1330
+    },
+    {
+      "entropy": 1.2149413120001555,
+      "epoch": 4.2816,
+      "grad_norm": 11.5625,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 1.1817,
+      "mean_token_accuracy": 0.7032578254118562,
+      "num_tokens": 22175046.0,
+      "step": 1340
+    },
+    {
+      "entropy": 1.229884122684598,
+      "epoch": 4.3136,
+      "grad_norm": 9.0625,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 1.1663,
+      "mean_token_accuracy": 0.7047730697318911,
+      "num_tokens": 22341391.0,
+      "step": 1350
+    },
+    {
+      "entropy": 1.1495930388569833,
+      "epoch": 4.3456,
+      "grad_norm": 9.5,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 1.1066,
+      "mean_token_accuracy": 0.7165895994752646,
+      "num_tokens": 22507465.0,
+      "step": 1360
+    },
+    {
+      "entropy": 1.1687410920858383,
+      "epoch": 4.3776,
+      "grad_norm": 8.9375,
+      "learning_rate": 1.918e-05,
+      "loss": 1.1492,
+      "mean_token_accuracy": 0.7107833050191402,
+      "num_tokens": 22672707.0,
+      "step": 1370
+    },
+    {
+      "entropy": 1.1701302990317344,
+      "epoch": 4.4096,
+      "grad_norm": 9.6875,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 1.1427,
+      "mean_token_accuracy": 0.7115983698517084,
+      "num_tokens": 22838278.0,
+      "step": 1380
+    },
+    {
+      "entropy": 1.1510434754192829,
+      "epoch": 4.4416,
+      "grad_norm": 8.75,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 1.1088,
+      "mean_token_accuracy": 0.7155276730656623,
+      "num_tokens": 23006714.0,
+      "step": 1390
+    },
+    {
+      "entropy": 1.1388842329382896,
+      "epoch": 4.4736,
+      "grad_norm": 9.375,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 1.0994,
+      "mean_token_accuracy": 0.7179952016100287,
+      "num_tokens": 23171616.0,
+      "step": 1400
+    },
+    {
+      "epoch": 4.4736,
+      "eval_coding_entropy": 1.6269907784461974,
+      "eval_coding_loss": 2.209084987640381,
+      "eval_coding_mean_token_accuracy": 0.5191091188192367,
+      "eval_coding_num_tokens": 23171616.0,
+      "eval_coding_runtime": 78.6415,
+      "eval_coding_samples_per_second": 6.358,
+      "eval_coding_steps_per_second": 3.179,
+      "step": 1400
+    },
+    {
+      "epoch": 4.4736,
+      "eval_physics_entropy": 1.237709529876709,
+      "eval_physics_loss": 1.3176825046539307,
+      "eval_physics_mean_token_accuracy": 0.6890720264911652,
+      "eval_physics_num_tokens": 23171616.0,
+      "eval_physics_runtime": 50.0456,
+      "eval_physics_samples_per_second": 9.991,
+      "eval_physics_steps_per_second": 4.995,
+      "step": 1400
+    },
+    {
+      "entropy": 1.175365987792611,
+      "epoch": 4.5056,
+      "grad_norm": 9.875,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 1.1347,
+      "mean_token_accuracy": 0.7122149540111422,
+      "num_tokens": 23339342.0,
+      "step": 1410
+    },
+    {
+      "entropy": 1.1470170836895703,
+      "epoch": 4.5376,
+      "grad_norm": 10.0625,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 1.1395,
+      "mean_token_accuracy": 0.7165789626538753,
+      "num_tokens": 23502858.0,
+      "step": 1420
+    },
+    {
+      "entropy": 1.1967794239521026,
+      "epoch": 4.5696,
+      "grad_norm": 10.375,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 1.1572,
+      "mean_token_accuracy": 0.7101291220635175,
+      "num_tokens": 23671520.0,
+      "step": 1430
+    },
+    {
+      "entropy": 1.1716468155384063,
+      "epoch": 4.6016,
+      "grad_norm": 11.3125,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 1.1501,
+      "mean_token_accuracy": 0.7126103952527046,
+      "num_tokens": 23832756.0,
+      "step": 1440
+    },
+    {
+      "entropy": 1.16734869889915,
+      "epoch": 4.6336,
+      "grad_norm": 10.1875,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 1.11,
+      "mean_token_accuracy": 0.7169415984302759,
+      "num_tokens": 23994203.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.1217462074011564,
+      "epoch": 4.6655999999999995,
+      "grad_norm": 9.1875,
+      "learning_rate": 1.898e-05,
+      "loss": 1.0867,
+      "mean_token_accuracy": 0.7218449834734202,
+      "num_tokens": 24170398.0,
+      "step": 1460
+    },
+    {
+      "entropy": 1.1713214162737131,
+      "epoch": 4.6975999999999996,
+      "grad_norm": 10.625,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 1.1403,
+      "mean_token_accuracy": 0.7142901595681905,
+      "num_tokens": 24333281.0,
+      "step": 1470
+    },
+    {
+      "entropy": 1.1525508970022202,
+      "epoch": 4.7296,
+      "grad_norm": 9.0625,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 1.11,
+      "mean_token_accuracy": 0.7160584177821875,
+      "num_tokens": 24502037.0,
+      "step": 1480
+    },
+    {
+      "entropy": 1.216130430623889,
+      "epoch": 4.7616,
+      "grad_norm": 10.875,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 1.1641,
+      "mean_token_accuracy": 0.7066824365407228,
+      "num_tokens": 24667335.0,
+      "step": 1490
+    },
+    {
+      "entropy": 1.1448049381375314,
+      "epoch": 4.7936,
+      "grad_norm": 9.6875,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 1.1069,
+      "mean_token_accuracy": 0.7182889092713595,
+      "num_tokens": 24829965.0,
+      "step": 1500
+    },
+    {
+      "epoch": 4.7936,
+      "eval_coding_entropy": 1.6142102317810059,
+      "eval_coding_loss": 2.2128803730010986,
+      "eval_coding_mean_token_accuracy": 0.5190054897069931,
+      "eval_coding_num_tokens": 24829965.0,
+      "eval_coding_runtime": 78.6632,
+      "eval_coding_samples_per_second": 6.356,
+      "eval_coding_steps_per_second": 3.178,
+      "step": 1500
+    },
+    {
+      "epoch": 4.7936,
+      "eval_physics_entropy": 1.1880030524730683,
+      "eval_physics_loss": 1.2855262756347656,
+      "eval_physics_mean_token_accuracy": 0.696103319644928,
+      "eval_physics_num_tokens": 24829965.0,
+      "eval_physics_runtime": 49.8488,
+      "eval_physics_samples_per_second": 10.03,
+      "eval_physics_steps_per_second": 5.015,
+      "step": 1500
+    },
+    {
+      "entropy": 1.1403514467179776,
+      "epoch": 4.8256,
+      "grad_norm": 8.5,
+      "learning_rate": 1.886888888888889e-05,
+      "loss": 1.0965,
+      "mean_token_accuracy": 0.7215337453410029,
+      "num_tokens": 24993001.0,
+      "step": 1510
+    },
+    {
+      "entropy": 1.1571423266083003,
+      "epoch": 4.8576,
+      "grad_norm": 9.0,
+      "learning_rate": 1.884666666666667e-05,
+      "loss": 1.1022,
+      "mean_token_accuracy": 0.7180591471493244,
+      "num_tokens": 25161270.0,
+      "step": 1520
+    },
+    {
+      "entropy": 1.1574883088469505,
+      "epoch": 4.8896,
+      "grad_norm": 8.875,
+      "learning_rate": 1.8824444444444445e-05,
+      "loss": 1.1232,
+      "mean_token_accuracy": 0.7166322693228722,
+      "num_tokens": 25325907.0,
+      "step": 1530
+    },
+    {
+      "entropy": 1.100039078295231,
+      "epoch": 4.9216,
+      "grad_norm": 10.0,
+      "learning_rate": 1.8802222222222226e-05,
+      "loss": 1.0705,
+      "mean_token_accuracy": 0.7251184478402137,
+      "num_tokens": 25492085.0,
+      "step": 1540
+    },
+    {
+      "entropy": 1.2036859635263681,
+      "epoch": 4.9536,
+      "grad_norm": 9.3125,
+      "learning_rate": 1.878e-05,
+      "loss": 1.1486,
+      "mean_token_accuracy": 0.7091086957603693,
+      "num_tokens": 25653637.0,
+      "step": 1550
+    },
+    {
+      "entropy": 1.1511400450021028,
+      "epoch": 4.9856,
+      "grad_norm": 9.6875,
+      "learning_rate": 1.875777777777778e-05,
+      "loss": 1.0991,
+      "mean_token_accuracy": 0.7183625839650631,
+      "num_tokens": 25825573.0,
+      "step": 1560
+    },
+    {
+      "entropy": 1.0320005854101557,
+      "epoch": 5.016,
+      "grad_norm": 9.3125,
+      "learning_rate": 1.873555555555556e-05,
+      "loss": 0.9873,
+      "mean_token_accuracy": 0.7418808097902098,
+      "num_tokens": 25990067.0,
+      "step": 1570
+    },
+    {
+      "entropy": 1.0053182676434518,
+      "epoch": 5.048,
+      "grad_norm": 8.1875,
+      "learning_rate": 1.8713333333333336e-05,
+      "loss": 0.9567,
+      "mean_token_accuracy": 0.7489887418225407,
+      "num_tokens": 26157643.0,
+      "step": 1580
+    },
+    {
+      "entropy": 1.000998843461275,
+      "epoch": 5.08,
+      "grad_norm": 7.40625,
+      "learning_rate": 1.8691111111111114e-05,
+      "loss": 0.965,
+      "mean_token_accuracy": 0.7441015597432852,
+      "num_tokens": 26330383.0,
+      "step": 1590
+    },
+    {
+      "entropy": 1.0190535772591829,
+      "epoch": 5.112,
+      "grad_norm": 8.8125,
+      "learning_rate": 1.866888888888889e-05,
+      "loss": 0.9744,
+      "mean_token_accuracy": 0.7434236075729131,
+      "num_tokens": 26496271.0,
+      "step": 1600
+    },
+    {
+      "epoch": 5.112,
+      "eval_coding_entropy": 1.4611193208694457,
+      "eval_coding_loss": 2.2885076999664307,
+      "eval_coding_mean_token_accuracy": 0.515326168179512,
+      "eval_coding_num_tokens": 26496271.0,
+      "eval_coding_runtime": 78.6153,
+      "eval_coding_samples_per_second": 6.36,
+      "eval_coding_steps_per_second": 3.18,
+      "step": 1600
+    },
+    {
+      "epoch": 5.112,
+      "eval_physics_entropy": 1.0492182924747466,
+      "eval_physics_loss": 1.2861127853393555,
+      "eval_physics_mean_token_accuracy": 0.6994015083312989,
+      "eval_physics_num_tokens": 26496271.0,
+      "eval_physics_runtime": 50.0233,
+      "eval_physics_samples_per_second": 9.995,
+      "eval_physics_steps_per_second": 4.998,
+      "step": 1600
+    },
+    {
+      "entropy": 1.0435870230197906,
+      "epoch": 5.144,
+      "grad_norm": 8.8125,
+      "learning_rate": 1.864666666666667e-05,
+      "loss": 1.0014,
+      "mean_token_accuracy": 0.7376969318836928,
+      "num_tokens": 26657872.0,
+      "step": 1610
+    },
+    {
+      "entropy": 1.055980871617794,
+      "epoch": 5.176,
+      "grad_norm": 8.625,
+      "learning_rate": 1.8624444444444446e-05,
+      "loss": 1.0179,
+      "mean_token_accuracy": 0.7351004585623742,
+      "num_tokens": 26815558.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.9974001601338387,
+      "epoch": 5.208,
+      "grad_norm": 8.9375,
+      "learning_rate": 1.8602222222222224e-05,
+      "loss": 0.9557,
+      "mean_token_accuracy": 0.7475613258779049,
+      "num_tokens": 26988857.0,
+      "step": 1630
+    },
+    {
+      "entropy": 1.0223722621798514,
+      "epoch": 5.24,
+      "grad_norm": 7.40625,
+      "learning_rate": 1.858e-05,
+      "loss": 0.9779,
+      "mean_token_accuracy": 0.7432104963809252,
+      "num_tokens": 27149125.0,
+      "step": 1640
+    },
+    {
+      "entropy": 1.026456928253174,
+      "epoch": 5.272,
+      "grad_norm": 8.5,
+      "learning_rate": 1.855777777777778e-05,
+      "loss": 0.989,
+      "mean_token_accuracy": 0.7418703351169824,
+      "num_tokens": 27316852.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.9835772240534425,
+      "epoch": 5.304,
+      "grad_norm": 8.25,
+      "learning_rate": 1.8535555555555557e-05,
+      "loss": 0.9543,
+      "mean_token_accuracy": 0.7498055439442396,
+      "num_tokens": 27481303.0,
+      "step": 1660
+    },
+    {
+      "entropy": 1.0393506474792957,
+      "epoch": 5.336,
+      "grad_norm": 8.375,
+      "learning_rate": 1.8513333333333335e-05,
+      "loss": 1.003,
+      "mean_token_accuracy": 0.7385861840099096,
+      "num_tokens": 27658752.0,
+      "step": 1670
+    },
+    {
+      "entropy": 1.0121482180431485,
+      "epoch": 5.368,
+      "grad_norm": 7.9375,
+      "learning_rate": 1.8491111111111112e-05,
+      "loss": 0.975,
+      "mean_token_accuracy": 0.7442660683766007,
+      "num_tokens": 27825908.0,
+      "step": 1680
+    },
+    {
+      "entropy": 1.0493794478476048,
+      "epoch": 5.4,
+      "grad_norm": 8.375,
+      "learning_rate": 1.846888888888889e-05,
+      "loss": 0.9908,
+      "mean_token_accuracy": 0.738095085322857,
+      "num_tokens": 27994598.0,
+      "step": 1690
+    },
+    {
+      "entropy": 1.0492326274514199,
+      "epoch": 5.432,
+      "grad_norm": 8.625,
+      "learning_rate": 1.8446666666666667e-05,
+      "loss": 1.0058,
+      "mean_token_accuracy": 0.7364107050001621,
+      "num_tokens": 28163244.0,
+      "step": 1700
+    },
+    {
+      "epoch": 5.432,
+      "eval_coding_entropy": 1.4886770558357239,
+      "eval_coding_loss": 2.2841522693634033,
+      "eval_coding_mean_token_accuracy": 0.5166380307674407,
+      "eval_coding_num_tokens": 28163244.0,
+      "eval_coding_runtime": 78.7385,
+      "eval_coding_samples_per_second": 6.35,
+      "eval_coding_steps_per_second": 3.175,
+      "step": 1700
+    },
+    {
+      "epoch": 5.432,
+      "eval_physics_entropy": 1.0742596254348755,
+      "eval_physics_loss": 1.264482021331787,
+      "eval_physics_mean_token_accuracy": 0.7024200389385223,
+      "eval_physics_num_tokens": 28163244.0,
+      "eval_physics_runtime": 50.8953,
+      "eval_physics_samples_per_second": 9.824,
+      "eval_physics_steps_per_second": 4.912,
+      "step": 1700
+    },
+    {
+      "entropy": 1.0063464101403952,
+      "epoch": 5.464,
+      "grad_norm": 10.625,
+      "learning_rate": 1.842444444444445e-05,
+      "loss": 0.9783,
+      "mean_token_accuracy": 0.7455477464944125,
+      "num_tokens": 28327234.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.9889417450875044,
+      "epoch": 5.496,
+      "grad_norm": 9.9375,
+      "learning_rate": 1.8402222222222223e-05,
+      "loss": 0.9453,
+      "mean_token_accuracy": 0.7497266609221697,
+      "num_tokens": 28494226.0,
+      "step": 1720
+    },
+    {
+      "entropy": 1.0184832660481333,
+      "epoch": 5.5280000000000005,
+      "grad_norm": 7.125,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 0.9889,
+      "mean_token_accuracy": 0.7424718491733074,
+      "num_tokens": 28662926.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.9835422236472369,
+      "epoch": 5.5600000000000005,
+      "grad_norm": 8.1875,
+      "learning_rate": 1.8357777777777778e-05,
+      "loss": 0.9507,
+      "mean_token_accuracy": 0.749301765486598,
+      "num_tokens": 28827430.0,
+      "step": 1740
+    },
+    {
+      "entropy": 1.026626091822982,
+      "epoch": 5.592,
+      "grad_norm": 7.84375,
+      "learning_rate": 1.833555555555556e-05,
+      "loss": 0.9835,
+      "mean_token_accuracy": 0.7390441335737705,
+      "num_tokens": 28993938.0,
+      "step": 1750
+    },
+    {
+      "entropy": 1.0556916188448668,
+      "epoch": 5.624,
+      "grad_norm": 8.625,
+      "learning_rate": 1.8313333333333333e-05,
+      "loss": 1.0223,
+      "mean_token_accuracy": 0.7359289925545454,
+      "num_tokens": 29159365.0,
+      "step": 1760
+    },
+    {
+      "entropy": 1.03387993555516,
+      "epoch": 5.656,
+      "grad_norm": 8.625,
+      "learning_rate": 1.8291111111111114e-05,
+      "loss": 1.0093,
+      "mean_token_accuracy": 0.7393341969698668,
+      "num_tokens": 29320125.0,
+      "step": 1770
+    },
+    {
+      "entropy": 1.05478678047657,
+      "epoch": 5.688,
+      "grad_norm": 8.0625,
+      "learning_rate": 1.8268888888888888e-05,
+      "loss": 0.9925,
+      "mean_token_accuracy": 0.739771020039916,
+      "num_tokens": 29481876.0,
+      "step": 1780
+    },
+    {
+      "entropy": 1.0282816842198372,
+      "epoch": 5.72,
+      "grad_norm": 9.1875,
+      "learning_rate": 1.824666666666667e-05,
+      "loss": 0.9933,
+      "mean_token_accuracy": 0.7399836644530297,
+      "num_tokens": 29641581.0,
+      "step": 1790
+    },
+    {
+      "entropy": 1.0386770512908696,
+      "epoch": 5.752,
+      "grad_norm": 8.25,
+      "learning_rate": 1.8224444444444447e-05,
+      "loss": 0.9909,
+      "mean_token_accuracy": 0.7411898959428072,
+      "num_tokens": 29804639.0,
+      "step": 1800
+    },
+    {
+      "epoch": 5.752,
+      "eval_coding_entropy": 1.5030966238975525,
+      "eval_coding_loss": 2.3069663047790527,
+      "eval_coding_mean_token_accuracy": 0.5138696851730347,
+      "eval_coding_num_tokens": 29804639.0,
+      "eval_coding_runtime": 78.776,
+      "eval_coding_samples_per_second": 6.347,
+      "eval_coding_steps_per_second": 3.174,
+      "step": 1800
+    },
+    {
+      "epoch": 5.752,
+      "eval_physics_entropy": 1.0810832781791686,
+      "eval_physics_loss": 1.2493600845336914,
+      "eval_physics_mean_token_accuracy": 0.7056341125965119,
+      "eval_physics_num_tokens": 29804639.0,
+      "eval_physics_runtime": 50.0795,
+      "eval_physics_samples_per_second": 9.984,
+      "eval_physics_steps_per_second": 4.992,
+      "step": 1800
+    },
+    {
+      "entropy": 1.0106814747676254,
+      "epoch": 5.784,
+      "grad_norm": 7.3125,
+      "learning_rate": 1.8202222222222225e-05,
+      "loss": 0.9794,
+      "mean_token_accuracy": 0.742822153493762,
+      "num_tokens": 29978576.0,
+      "step": 1810
+    },
+    {
+      "entropy": 1.019618400745094,
+      "epoch": 5.816,
+      "grad_norm": 7.96875,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 0.9833,
+      "mean_token_accuracy": 0.7443012058734894,
+      "num_tokens": 30141916.0,
+      "step": 1820
+    },
+    {
+      "entropy": 1.0012058950960636,
+      "epoch": 5.848,
+      "grad_norm": 7.40625,
+      "learning_rate": 1.815777777777778e-05,
+      "loss": 0.976,
+      "mean_token_accuracy": 0.7464470114558935,
+      "num_tokens": 30307900.0,
+      "step": 1830
+    },
+    {
+      "entropy": 1.0612958241254091,
+      "epoch": 5.88,
+      "grad_norm": 7.90625,
+      "learning_rate": 1.8135555555555557e-05,
+      "loss": 1.0001,
+      "mean_token_accuracy": 0.7349641341716051,
+      "num_tokens": 30470302.0,
+      "step": 1840
+    },
+    {
+      "entropy": 1.0391322318464518,
+      "epoch": 5.912,
+      "grad_norm": 8.625,
+      "learning_rate": 1.8113333333333335e-05,
+      "loss": 1.0014,
+      "mean_token_accuracy": 0.7386055026203394,
+      "num_tokens": 30636029.0,
+      "step": 1850
+    },
+    {
+      "entropy": 1.056866429001093,
+      "epoch": 5.944,
+      "grad_norm": 7.71875,
+      "learning_rate": 1.8091111111111113e-05,
+      "loss": 1.0134,
+      "mean_token_accuracy": 0.7359001263976097,
+      "num_tokens": 30799363.0,
+      "step": 1860
+    },
+    {
+      "entropy": 1.055723300948739,
+      "epoch": 5.976,
+      "grad_norm": 9.0,
+      "learning_rate": 1.806888888888889e-05,
+      "loss": 1.0137,
+      "mean_token_accuracy": 0.7384760480374097,
+      "num_tokens": 30960267.0,
+      "step": 1870
+    },
+    {
+      "entropy": 1.0095378219297058,
+      "epoch": 6.0064,
+      "grad_norm": 7.28125,
+      "learning_rate": 1.8046666666666668e-05,
+      "loss": 0.963,
+      "mean_token_accuracy": 0.7492663370151269,
+      "num_tokens": 31115221.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.88621536642313,
+      "epoch": 6.0384,
+      "grad_norm": 8.625,
+      "learning_rate": 1.8024444444444445e-05,
+      "loss": 0.8479,
+      "mean_token_accuracy": 0.7714764319360257,
+      "num_tokens": 31282640.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.8804416542872786,
+      "epoch": 6.0704,
+      "grad_norm": 7.78125,
+      "learning_rate": 1.8002222222222223e-05,
+      "loss": 0.8476,
+      "mean_token_accuracy": 0.7695284474641084,
+      "num_tokens": 31452030.0,
+      "step": 1900
+    },
+    {
+      "epoch": 6.0704,
+      "eval_coding_entropy": 1.4332211565971376,
+      "eval_coding_loss": 2.3655660152435303,
+      "eval_coding_mean_token_accuracy": 0.5109253574609757,
+      "eval_coding_num_tokens": 31452030.0,
+      "eval_coding_runtime": 78.657,
+      "eval_coding_samples_per_second": 6.357,
+      "eval_coding_steps_per_second": 3.178,
+      "step": 1900
+    },
+    {
+      "epoch": 6.0704,
+      "eval_physics_entropy": 0.9726086575984955,
+      "eval_physics_loss": 1.2605265378952026,
+      "eval_physics_mean_token_accuracy": 0.7070128087997436,
+      "eval_physics_num_tokens": 31452030.0,
+      "eval_physics_runtime": 50.0718,
+      "eval_physics_samples_per_second": 9.986,
+      "eval_physics_steps_per_second": 4.993,
+      "step": 1900
+    },
+    {
+      "entropy": 0.8962142301723361,
+      "epoch": 6.1024,
+      "grad_norm": 8.5625,
+      "learning_rate": 1.798e-05,
+      "loss": 0.8464,
+      "mean_token_accuracy": 0.7724565159529447,
+      "num_tokens": 31617821.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.9005906537175179,
+      "epoch": 6.1344,
+      "grad_norm": 8.0,
+      "learning_rate": 1.7957777777777778e-05,
+      "loss": 0.8613,
+      "mean_token_accuracy": 0.7654172223061323,
+      "num_tokens": 31783257.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.9097023481503129,
+      "epoch": 6.1664,
+      "grad_norm": 8.125,
+      "learning_rate": 1.7935555555555556e-05,
+      "loss": 0.8747,
+      "mean_token_accuracy": 0.7655117467045784,
+      "num_tokens": 31945987.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.9006351741030812,
+      "epoch": 6.1984,
+      "grad_norm": 8.375,
+      "learning_rate": 1.7913333333333337e-05,
+      "loss": 0.8511,
+      "mean_token_accuracy": 0.7700866155326367,
+      "num_tokens": 32108923.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.868392788618803,
+      "epoch": 6.2304,
+      "grad_norm": 7.4375,
+      "learning_rate": 1.789111111111111e-05,
+      "loss": 0.8263,
+      "mean_token_accuracy": 0.7728913404047489,
+      "num_tokens": 32280527.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.9419200228527188,
+      "epoch": 6.2624,
+      "grad_norm": 8.0625,
+      "learning_rate": 1.7868888888888892e-05,
+      "loss": 0.8833,
+      "mean_token_accuracy": 0.7601143646985292,
+      "num_tokens": 32448689.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.8681814942508936,
+      "epoch": 6.2943999999999996,
+      "grad_norm": 6.84375,
+      "learning_rate": 1.7846666666666666e-05,
+      "loss": 0.8263,
+      "mean_token_accuracy": 0.7745695225894451,
+      "num_tokens": 32623655.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.9166517918929458,
+      "epoch": 6.3264,
+      "grad_norm": 8.75,
+      "learning_rate": 1.7824444444444447e-05,
+      "loss": 0.881,
+      "mean_token_accuracy": 0.7623546797782182,
+      "num_tokens": 32788431.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.9212754137814045,
+      "epoch": 6.3584,
+      "grad_norm": 7.25,
+      "learning_rate": 1.780222222222222e-05,
+      "loss": 0.8954,
+      "mean_token_accuracy": 0.7639275800436736,
+      "num_tokens": 32953331.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.9211506059393286,
+      "epoch": 6.3904,
+      "grad_norm": 7.15625,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 0.867,
+      "mean_token_accuracy": 0.7647997327148914,
+      "num_tokens": 33123102.0,
+      "step": 2000
+    },
+    {
+      "epoch": 6.3904,
+      "eval_coding_entropy": 1.4045324511528015,
+      "eval_coding_loss": 2.4114694595336914,
+      "eval_coding_mean_token_accuracy": 0.5078031500577926,
+      "eval_coding_num_tokens": 33123102.0,
+      "eval_coding_runtime": 78.6861,
+      "eval_coding_samples_per_second": 6.354,
+      "eval_coding_steps_per_second": 3.177,
+      "step": 2000
+    },
+    {
+      "epoch": 6.3904,
+      "eval_physics_entropy": 0.9753130996227264,
+      "eval_physics_loss": 1.2583574056625366,
+      "eval_physics_mean_token_accuracy": 0.7085516684055329,
+      "eval_physics_num_tokens": 33123102.0,
+      "eval_physics_runtime": 50.343,
+      "eval_physics_samples_per_second": 9.932,
+      "eval_physics_steps_per_second": 4.966,
+      "step": 2000
+    },
+    {
+      "entropy": 0.9211070956662297,
+      "epoch": 6.4224,
+      "grad_norm": 9.1875,
+      "learning_rate": 1.7757777777777777e-05,
+      "loss": 0.8724,
+      "mean_token_accuracy": 0.7644503612071276,
+      "num_tokens": 33286459.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.9069164827466011,
+      "epoch": 6.4544,
+      "grad_norm": 8.3125,
+      "learning_rate": 1.7735555555555558e-05,
+      "loss": 0.8667,
+      "mean_token_accuracy": 0.7666214227676391,
+      "num_tokens": 33452198.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.9228206187486648,
+      "epoch": 6.4864,
+      "grad_norm": 7.21875,
+      "learning_rate": 1.7713333333333335e-05,
+      "loss": 0.8878,
+      "mean_token_accuracy": 0.7641974788159132,
+      "num_tokens": 33617056.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.9319445975124836,
+      "epoch": 6.5184,
+      "grad_norm": 9.125,
+      "learning_rate": 1.7691111111111113e-05,
+      "loss": 0.8906,
+      "mean_token_accuracy": 0.7599198468029499,
+      "num_tokens": 33778782.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.8847375756129623,
+      "epoch": 6.5504,
+      "grad_norm": 7.34375,
+      "learning_rate": 1.766888888888889e-05,
+      "loss": 0.8573,
+      "mean_token_accuracy": 0.7707682568579912,
+      "num_tokens": 33946435.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.872058791294694,
+      "epoch": 6.5824,
+      "grad_norm": 7.28125,
+      "learning_rate": 1.7646666666666668e-05,
+      "loss": 0.8497,
+      "mean_token_accuracy": 0.7731397032737732,
+      "num_tokens": 34110878.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.9090044975280762,
+      "epoch": 6.6144,
+      "grad_norm": 7.78125,
+      "learning_rate": 1.7624444444444446e-05,
+      "loss": 0.8648,
+      "mean_token_accuracy": 0.7620598401874303,
+      "num_tokens": 34276122.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.9097535092383623,
+      "epoch": 6.6464,
+      "grad_norm": 7.46875,
+      "learning_rate": 1.7602222222222223e-05,
+      "loss": 0.8553,
+      "mean_token_accuracy": 0.7660120245069265,
+      "num_tokens": 34445468.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.9293602157384158,
+      "epoch": 6.6784,
+      "grad_norm": 7.9375,
+      "learning_rate": 1.758e-05,
+      "loss": 0.8933,
+      "mean_token_accuracy": 0.7610664397478104,
+      "num_tokens": 34607967.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.9214682994410396,
+      "epoch": 6.7104,
+      "grad_norm": 7.8125,
+      "learning_rate": 1.755777777777778e-05,
+      "loss": 0.8762,
+      "mean_token_accuracy": 0.7647664651274682,
+      "num_tokens": 34767297.0,
+      "step": 2100
+    },
+    {
+      "epoch": 6.7104,
+      "eval_coding_entropy": 1.394696186542511,
+      "eval_coding_loss": 2.4158456325531006,
+      "eval_coding_mean_token_accuracy": 0.5068432171344757,
+      "eval_coding_num_tokens": 34767297.0,
+      "eval_coding_runtime": 78.8249,
+      "eval_coding_samples_per_second": 6.343,
+      "eval_coding_steps_per_second": 3.172,
+      "step": 2100
+    },
+    {
+      "epoch": 6.7104,
+      "eval_physics_entropy": 0.957178023815155,
+      "eval_physics_loss": 1.2451750040054321,
+      "eval_physics_mean_token_accuracy": 0.7113168475627899,
+      "eval_physics_num_tokens": 34767297.0,
+      "eval_physics_runtime": 50.1029,
+      "eval_physics_samples_per_second": 9.979,
+      "eval_physics_steps_per_second": 4.99,
+      "step": 2100
+    },
+    {
+      "entropy": 0.9239593407139182,
+      "epoch": 6.7424,
+      "grad_norm": 8.1875,
+      "learning_rate": 1.7535555555555556e-05,
+      "loss": 0.8691,
+      "mean_token_accuracy": 0.7621566083282232,
+      "num_tokens": 34934673.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.9330404713749886,
+      "epoch": 6.7744,
+      "grad_norm": 7.46875,
+      "learning_rate": 1.7513333333333334e-05,
+      "loss": 0.8909,
+      "mean_token_accuracy": 0.7622119640931487,
+      "num_tokens": 35095715.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.9513458956032992,
+      "epoch": 6.8064,
+      "grad_norm": 8.6875,
+      "learning_rate": 1.749111111111111e-05,
+      "loss": 0.8904,
+      "mean_token_accuracy": 0.7597831141203641,
+      "num_tokens": 35252925.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.8813711836934089,
+      "epoch": 6.8384,
+      "grad_norm": 7.5625,
+      "learning_rate": 1.746888888888889e-05,
+      "loss": 0.858,
+      "mean_token_accuracy": 0.7700365465134382,
+      "num_tokens": 35421004.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.9245317034423352,
+      "epoch": 6.8704,
+      "grad_norm": 9.5,
+      "learning_rate": 1.7446666666666667e-05,
+      "loss": 0.8874,
+      "mean_token_accuracy": 0.760653006285429,
+      "num_tokens": 35588850.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.9203046636655927,
+      "epoch": 6.9024,
+      "grad_norm": 7.375,
+      "learning_rate": 1.7424444444444444e-05,
+      "loss": 0.8813,
+      "mean_token_accuracy": 0.7625456716865301,
+      "num_tokens": 35756753.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.9076596148312092,
+      "epoch": 6.9344,
+      "grad_norm": 7.71875,
+      "learning_rate": 1.7402222222222222e-05,
+      "loss": 0.8613,
+      "mean_token_accuracy": 0.7665100399404764,
+      "num_tokens": 35927158.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.9311045955866575,
+      "epoch": 6.9664,
+      "grad_norm": 9.5,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 0.8829,
+      "mean_token_accuracy": 0.7643363267183304,
+      "num_tokens": 36090024.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.884549498744309,
+      "epoch": 6.9984,
+      "grad_norm": 8.375,
+      "learning_rate": 1.735777777777778e-05,
+      "loss": 0.8518,
+      "mean_token_accuracy": 0.7702371954917908,
+      "num_tokens": 36258284.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.820740462133759,
+      "epoch": 7.0288,
+      "grad_norm": 7.84375,
+      "learning_rate": 1.7335555555555558e-05,
+      "loss": 0.7853,
+      "mean_token_accuracy": 0.7873634801883447,
+      "num_tokens": 36413412.0,
+      "step": 2200
+    },
+    {
+      "epoch": 7.0288,
+      "eval_coding_entropy": 1.367724521636963,
+      "eval_coding_loss": 2.446176290512085,
+      "eval_coding_mean_token_accuracy": 0.5063173699378968,
+      "eval_coding_num_tokens": 36413412.0,
+      "eval_coding_runtime": 78.7364,
+      "eval_coding_samples_per_second": 6.35,
+      "eval_coding_steps_per_second": 3.175,
+      "step": 2200
+    },
+    {
+      "epoch": 7.0288,
+      "eval_physics_entropy": 0.9128225688934326,
+      "eval_physics_loss": 1.2582210302352905,
+      "eval_physics_mean_token_accuracy": 0.7116236865520478,
+      "eval_physics_num_tokens": 36413412.0,
+      "eval_physics_runtime": 50.0817,
+      "eval_physics_samples_per_second": 9.984,
+      "eval_physics_steps_per_second": 4.992,
+      "step": 2200
+    },
+    {
+      "entropy": 0.8266337281093001,
+      "epoch": 7.0608,
+      "grad_norm": 6.78125,
+      "learning_rate": 1.7313333333333336e-05,
+      "loss": 0.778,
+      "mean_token_accuracy": 0.7868986543267965,
+      "num_tokens": 36577814.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.8053497424349189,
+      "epoch": 7.0928,
+      "grad_norm": 6.28125,
+      "learning_rate": 1.7291111111111113e-05,
+      "loss": 0.7591,
+      "mean_token_accuracy": 0.7914245255291462,
+      "num_tokens": 36743769.0,
+      "step": 2220
+    },
+    {
+      "entropy": 0.7795431900769472,
+      "epoch": 7.1248,
+      "grad_norm": 7.34375,
+      "learning_rate": 1.726888888888889e-05,
+      "loss": 0.7401,
+      "mean_token_accuracy": 0.7962938852608203,
+      "num_tokens": 36905500.0,
+      "step": 2230
+    },
+    {
+      "entropy": 0.771372621320188,
+      "epoch": 7.1568,
+      "grad_norm": 8.125,
+      "learning_rate": 1.724666666666667e-05,
+      "loss": 0.7274,
+      "mean_token_accuracy": 0.7963879514485598,
+      "num_tokens": 37072276.0,
+      "step": 2240
+    },
+    {
+      "entropy": 0.8004817988723516,
+      "epoch": 7.1888,
+      "grad_norm": 7.0625,
+      "learning_rate": 1.7224444444444446e-05,
+      "loss": 0.7504,
+      "mean_token_accuracy": 0.7901833631098271,
+      "num_tokens": 37243206.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.7968222606927157,
+      "epoch": 7.2208,
+      "grad_norm": 7.65625,
+      "learning_rate": 1.7202222222222224e-05,
+      "loss": 0.7544,
+      "mean_token_accuracy": 0.7902005612850189,
+      "num_tokens": 37403782.0,
+      "step": 2260
+    },
+    {
+      "entropy": 0.8174678221344948,
+      "epoch": 7.2528,
+      "grad_norm": 8.75,
+      "learning_rate": 1.718e-05,
+      "loss": 0.7639,
+      "mean_token_accuracy": 0.7886193033307791,
+      "num_tokens": 37579757.0,
+      "step": 2270
+    },
+    {
+      "entropy": 0.8302789879962802,
+      "epoch": 7.2848,
+      "grad_norm": 8.5625,
+      "learning_rate": 1.715777777777778e-05,
+      "loss": 0.7712,
+      "mean_token_accuracy": 0.7847586594521999,
+      "num_tokens": 37748293.0,
+      "step": 2280
+    },
+    {
+      "entropy": 0.8039891485124826,
+      "epoch": 7.3168,
+      "grad_norm": 7.625,
+      "learning_rate": 1.7135555555555557e-05,
+      "loss": 0.774,
+      "mean_token_accuracy": 0.7895373247563839,
+      "num_tokens": 37911700.0,
+      "step": 2290
+    },
+    {
+      "entropy": 0.8014172846451402,
+      "epoch": 7.3488,
+      "grad_norm": 6.90625,
+      "learning_rate": 1.7113333333333334e-05,
+      "loss": 0.7544,
+      "mean_token_accuracy": 0.7906766787171364,
+      "num_tokens": 38075680.0,
+      "step": 2300
+    },
+    {
+      "epoch": 7.3488,
+      "eval_coding_entropy": 1.2993780288696288,
+      "eval_coding_loss": 2.5096776485443115,
+      "eval_coding_mean_token_accuracy": 0.5038638023138047,
+      "eval_coding_num_tokens": 38075680.0,
+      "eval_coding_runtime": 78.7018,
+      "eval_coding_samples_per_second": 6.353,
+      "eval_coding_steps_per_second": 3.177,
+      "step": 2300
+    },
+    {
+      "epoch": 7.3488,
+      "eval_physics_entropy": 0.8744897862672806,
+      "eval_physics_loss": 1.2736599445343018,
+      "eval_physics_mean_token_accuracy": 0.7111985127925873,
+      "eval_physics_num_tokens": 38075680.0,
+      "eval_physics_runtime": 50.1077,
+      "eval_physics_samples_per_second": 9.979,
+      "eval_physics_steps_per_second": 4.989,
+      "step": 2300
+    },
+    {
+      "entropy": 0.7986550811678171,
+      "epoch": 7.3808,
+      "grad_norm": 7.0,
+      "learning_rate": 1.7091111111111112e-05,
+      "loss": 0.7524,
+      "mean_token_accuracy": 0.7910935759544373,
+      "num_tokens": 38244451.0,
+      "step": 2310
+    },
+    {
+      "entropy": 0.8117536768317223,
+      "epoch": 7.4128,
+      "grad_norm": 8.6875,
+      "learning_rate": 1.706888888888889e-05,
+      "loss": 0.7628,
+      "mean_token_accuracy": 0.7867957148700953,
+      "num_tokens": 38412760.0,
+      "step": 2320
+    },
+    {
+      "entropy": 0.7980742670595646,
+      "epoch": 7.4448,
+      "grad_norm": 7.59375,
+      "learning_rate": 1.704666666666667e-05,
+      "loss": 0.7656,
+      "mean_token_accuracy": 0.7893401090055704,
+      "num_tokens": 38573566.0,
+      "step": 2330
+    },
+    {
+      "entropy": 0.8302219083532691,
+      "epoch": 7.4768,
+      "grad_norm": 7.15625,
+      "learning_rate": 1.7024444444444445e-05,
+      "loss": 0.7821,
+      "mean_token_accuracy": 0.7837908625602722,
+      "num_tokens": 38740056.0,
+      "step": 2340
+    },
+    {
+      "entropy": 0.8381878055632115,
+      "epoch": 7.5088,
+      "grad_norm": 7.90625,
+      "learning_rate": 1.7002222222222226e-05,
+      "loss": 0.7875,
+      "mean_token_accuracy": 0.7828322991728782,
+      "num_tokens": 38905300.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.8154581474140287,
+      "epoch": 7.5408,
+      "grad_norm": 6.71875,
+      "learning_rate": 1.698e-05,
+      "loss": 0.7561,
+      "mean_token_accuracy": 0.7866865880787373,
+      "num_tokens": 39071340.0,
+      "step": 2360
+    },
+    {
+      "entropy": 0.7772837754338979,
+      "epoch": 7.5728,
+      "grad_norm": 7.5625,
+      "learning_rate": 1.695777777777778e-05,
+      "loss": 0.7423,
+      "mean_token_accuracy": 0.7955018553882838,
+      "num_tokens": 39241074.0,
+      "step": 2370
+    },
+    {
+      "entropy": 0.8149220038205385,
+      "epoch": 7.6048,
+      "grad_norm": 8.8125,
+      "learning_rate": 1.6935555555555555e-05,
+      "loss": 0.7747,
+      "mean_token_accuracy": 0.7869505804032088,
+      "num_tokens": 39406361.0,
+      "step": 2380
+    },
+    {
+      "entropy": 0.8315773596987128,
+      "epoch": 7.6368,
+      "grad_norm": 8.5,
+      "learning_rate": 1.6913333333333336e-05,
+      "loss": 0.797,
+      "mean_token_accuracy": 0.781806480884552,
+      "num_tokens": 39565971.0,
+      "step": 2390
+    },
+    {
+      "entropy": 0.8063804341480136,
+      "epoch": 7.6688,
+      "grad_norm": 6.875,
+      "learning_rate": 1.689111111111111e-05,
+      "loss": 0.7593,
+      "mean_token_accuracy": 0.7898448396474123,
+      "num_tokens": 39732063.0,
+      "step": 2400
+    },
+    {
+      "epoch": 7.6688,
+      "eval_coding_entropy": 1.2983161787986754,
+      "eval_coding_loss": 2.5096092224121094,
+      "eval_coding_mean_token_accuracy": 0.5032128055095673,
+      "eval_coding_num_tokens": 39732063.0,
+      "eval_coding_runtime": 78.6646,
+      "eval_coding_samples_per_second": 6.356,
+      "eval_coding_steps_per_second": 3.178,
+      "step": 2400
+    },
+    {
+      "epoch": 7.6688,
+      "eval_physics_entropy": 0.8967354652881623,
+      "eval_physics_loss": 1.2583036422729492,
+      "eval_physics_mean_token_accuracy": 0.7123808844089508,
+      "eval_physics_num_tokens": 39732063.0,
+      "eval_physics_runtime": 50.0526,
+      "eval_physics_samples_per_second": 9.989,
+      "eval_physics_steps_per_second": 4.995,
+      "step": 2400
+    },
+    {
+      "entropy": 0.8263503026217223,
+      "epoch": 7.7008,
+      "grad_norm": 8.0,
+      "learning_rate": 1.686888888888889e-05,
+      "loss": 0.7795,
+      "mean_token_accuracy": 0.7855708755552768,
+      "num_tokens": 39893958.0,
+      "step": 2410
+    },
+    {
+      "entropy": 0.8092098431661725,
+      "epoch": 7.7328,
+      "grad_norm": 9.25,
+      "learning_rate": 1.684666666666667e-05,
+      "loss": 0.7729,
+      "mean_token_accuracy": 0.7862338222563267,
+      "num_tokens": 40056286.0,
+      "step": 2420
+    },
+    {
+      "entropy": 0.7706609938293696,
+      "epoch": 7.7648,
+      "grad_norm": 7.4375,
+      "learning_rate": 1.6824444444444447e-05,
+      "loss": 0.7289,
+      "mean_token_accuracy": 0.7959092561155557,
+      "num_tokens": 40224885.0,
+      "step": 2430
+    },
+    {
+      "entropy": 0.8446670968085528,
+      "epoch": 7.7968,
+      "grad_norm": 7.875,
+      "learning_rate": 1.6802222222222224e-05,
+      "loss": 0.789,
+      "mean_token_accuracy": 0.7807364620268344,
+      "num_tokens": 40386001.0,
+      "step": 2440
+    },
+    {
+      "entropy": 0.82587516810745,
+      "epoch": 7.8288,
+      "grad_norm": 7.40625,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 0.7797,
+      "mean_token_accuracy": 0.7859326565638185,
+      "num_tokens": 40550394.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.8284306824207306,
+      "epoch": 7.8608,
+      "grad_norm": 7.40625,
+      "learning_rate": 1.675777777777778e-05,
+      "loss": 0.7847,
+      "mean_token_accuracy": 0.7851311691105366,
+      "num_tokens": 40714123.0,
+      "step": 2460
+    },
+    {
+      "entropy": 0.8275166479870677,
+      "epoch": 7.8928,
+      "grad_norm": 7.90625,
+      "learning_rate": 1.6735555555555557e-05,
+      "loss": 0.7895,
+      "mean_token_accuracy": 0.7840345434844493,
+      "num_tokens": 40878910.0,
+      "step": 2470
+    },
+    {
+      "entropy": 0.791316207125783,
+      "epoch": 7.9248,
+      "grad_norm": 7.15625,
+      "learning_rate": 1.6713333333333335e-05,
+      "loss": 0.7559,
+      "mean_token_accuracy": 0.7909971076995135,
+      "num_tokens": 41048568.0,
+      "step": 2480
+    },
+    {
+      "entropy": 0.8075045378878712,
+      "epoch": 7.9568,
+      "grad_norm": 7.375,
+      "learning_rate": 1.6691111111111112e-05,
+      "loss": 0.7689,
+      "mean_token_accuracy": 0.7869745776057243,
+      "num_tokens": 41215871.0,
+      "step": 2490
+    },
+    {
+      "entropy": 0.8214766839519143,
+      "epoch": 7.9888,
+      "grad_norm": 7.03125,
+      "learning_rate": 1.666888888888889e-05,
+      "loss": 0.7761,
+      "mean_token_accuracy": 0.7834805313497781,
+      "num_tokens": 41389758.0,
+      "step": 2500
+    },
+    {
+      "epoch": 7.9888,
+      "eval_coding_entropy": 1.3161049132347107,
+      "eval_coding_loss": 2.5040156841278076,
+      "eval_coding_mean_token_accuracy": 0.5050035991668701,
+      "eval_coding_num_tokens": 41389758.0,
+      "eval_coding_runtime": 78.6756,
+      "eval_coding_samples_per_second": 6.355,
+      "eval_coding_steps_per_second": 3.178,
+      "step": 2500
+    },
+    {
+      "epoch": 7.9888,
+      "eval_physics_entropy": 0.8914902930259705,
+      "eval_physics_loss": 1.2486798763275146,
+      "eval_physics_mean_token_accuracy": 0.7153664343357086,
+      "eval_physics_num_tokens": 41389758.0,
+      "eval_physics_runtime": 49.9673,
+      "eval_physics_samples_per_second": 10.007,
+      "eval_physics_steps_per_second": 5.003,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 32,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.599915186815615e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c554301251913eab71ec7e8c9fd2c408b4dce9ce071b040acdfce0799d0e2b8
+size 6353