Upload folder using huggingface_hub

.gitattributes

@@ -37,3 +37,4 @@ checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-2500/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-2500/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-2500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-2500/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-2500/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-2500/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8da490df74ae7549a88cd33d6e09f43550d16878560e7aa6e5ebd7f5dd7b107
+size 4962221464

checkpoint-2500/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f518401e1a640f01cecfed1ddf55a6071a1007e2a61180ee1fd348f9aae1449
+size 3670322200

checkpoint-2500/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-2500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:443a3b5bae14adc1a9311835fe553299f86a54131e1ab9a377712fbfc327d9e3
+size 7927918951

checkpoint-2500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a20c201038acb2b5be3712dad6f6fa43496cb4adb00a20e372deb568fe40db13
+size 14645

checkpoint-2500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd200143f139327b4d41bcb99ffeaa13b2acc11ded5b3dbab181a6ff2ce855d7
+size 1465

checkpoint-2500/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-2500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-2500/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-2500/trainer_state.json

@@ -0,0 +1,3084 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 100,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.5604241602122784,
+      "epoch": 0.008,
+      "grad_norm": 360.0,
+      "learning_rate": 1.8e-07,
+      "loss": 6.7873,
+      "mean_token_accuracy": 0.21116912770085036,
+      "num_tokens": 175126.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.595553621649742,
+      "epoch": 0.016,
+      "grad_norm": 372.0,
+      "learning_rate": 3.8e-07,
+      "loss": 6.7228,
+      "mean_token_accuracy": 0.20832345820963383,
+      "num_tokens": 340696.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.6845590390264986,
+      "epoch": 0.024,
+      "grad_norm": 270.0,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 6.3033,
+      "mean_token_accuracy": 0.21763876075856387,
+      "num_tokens": 509066.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.8930466674268245,
+      "epoch": 0.032,
+      "grad_norm": 302.0,
+      "learning_rate": 7.8e-07,
+      "loss": 5.6306,
+      "mean_token_accuracy": 0.22912178803235292,
+      "num_tokens": 675930.0,
+      "step": 40
+    },
+    {
+      "entropy": 3.081789229810238,
+      "epoch": 0.04,
+      "grad_norm": 123.0,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 4.9613,
+      "mean_token_accuracy": 0.24729052558541298,
+      "num_tokens": 841685.0,
+      "step": 50
+    },
+    {
+      "entropy": 3.404916675388813,
+      "epoch": 0.048,
+      "grad_norm": 66.0,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 4.2301,
+      "mean_token_accuracy": 0.27200257908552883,
+      "num_tokens": 1007956.0,
+      "step": 60
+    },
+    {
+      "entropy": 3.6782597199082376,
+      "epoch": 0.056,
+      "grad_norm": 38.0,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 3.7823,
+      "mean_token_accuracy": 0.3019493265077472,
+      "num_tokens": 1174735.0,
+      "step": 70
+    },
+    {
+      "entropy": 3.4533004209399225,
+      "epoch": 0.064,
+      "grad_norm": 33.75,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 3.4895,
+      "mean_token_accuracy": 0.33877146458253266,
+      "num_tokens": 1340560.0,
+      "step": 80
+    },
+    {
+      "entropy": 3.139299413561821,
+      "epoch": 0.072,
+      "grad_norm": 31.625,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 3.1861,
+      "mean_token_accuracy": 0.3694635771214962,
+      "num_tokens": 1510110.0,
+      "step": 90
+    },
+    {
+      "entropy": 2.9895101204514503,
+      "epoch": 0.08,
+      "grad_norm": 21.25,
+      "learning_rate": 1.98e-06,
+      "loss": 2.9954,
+      "mean_token_accuracy": 0.39561883062124253,
+      "num_tokens": 1679432.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_coding_entropy": 2.0758819575309753,
+      "eval_coding_loss": 2.1979613304138184,
+      "eval_coding_mean_token_accuracy": 0.502328829407692,
+      "eval_coding_num_tokens": 1679432.0,
+      "eval_coding_runtime": 79.4698,
+      "eval_coding_samples_per_second": 6.292,
+      "eval_coding_steps_per_second": 3.146,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_physics_entropy": 2.9290144443511963,
+      "eval_physics_loss": 2.974918842315674,
+      "eval_physics_mean_token_accuracy": 0.40643512630462647,
+      "eval_physics_num_tokens": 1679432.0,
+      "eval_physics_runtime": 52.0573,
+      "eval_physics_samples_per_second": 9.605,
+      "eval_physics_steps_per_second": 4.802,
+      "step": 100
+    },
+    {
+      "entropy": 2.8267079159617423,
+      "epoch": 0.088,
+      "grad_norm": 21.125,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 2.8166,
+      "mean_token_accuracy": 0.4188435094431043,
+      "num_tokens": 1848521.0,
+      "step": 110
+    },
+    {
+      "entropy": 2.731858339905739,
+      "epoch": 0.096,
+      "grad_norm": 19.625,
+      "learning_rate": 2.38e-06,
+      "loss": 2.7192,
+      "mean_token_accuracy": 0.43734086528420446,
+      "num_tokens": 2021570.0,
+      "step": 120
+    },
+    {
+      "entropy": 2.742488440871239,
+      "epoch": 0.104,
+      "grad_norm": 20.5,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 2.7106,
+      "mean_token_accuracy": 0.4448789706453681,
+      "num_tokens": 2186993.0,
+      "step": 130
+    },
+    {
+      "entropy": 2.546742835640907,
+      "epoch": 0.112,
+      "grad_norm": 19.875,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 2.5347,
+      "mean_token_accuracy": 0.46661383416503666,
+      "num_tokens": 2359297.0,
+      "step": 140
+    },
+    {
+      "entropy": 2.5924400255084037,
+      "epoch": 0.12,
+      "grad_norm": 21.25,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 2.5665,
+      "mean_token_accuracy": 0.4689426450058818,
+      "num_tokens": 2522375.0,
+      "step": 150
+    },
+    {
+      "entropy": 2.4927076123654843,
+      "epoch": 0.128,
+      "grad_norm": 21.125,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 2.4571,
+      "mean_token_accuracy": 0.4879589693620801,
+      "num_tokens": 2683168.0,
+      "step": 160
+    },
+    {
+      "entropy": 2.4049933359026907,
+      "epoch": 0.136,
+      "grad_norm": 18.5,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 2.3829,
+      "mean_token_accuracy": 0.4920692486688495,
+      "num_tokens": 2850132.0,
+      "step": 170
+    },
+    {
+      "entropy": 2.3835881277918816,
+      "epoch": 0.144,
+      "grad_norm": 20.5,
+      "learning_rate": 3.58e-06,
+      "loss": 2.3245,
+      "mean_token_accuracy": 0.5039734555408358,
+      "num_tokens": 3019426.0,
+      "step": 180
+    },
+    {
+      "entropy": 2.3372473388910295,
+      "epoch": 0.152,
+      "grad_norm": 18.25,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 2.3194,
+      "mean_token_accuracy": 0.5051064729690552,
+      "num_tokens": 3182640.0,
+      "step": 190
+    },
+    {
+      "entropy": 2.27973321005702,
+      "epoch": 0.16,
+      "grad_norm": 21.125,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 2.213,
+      "mean_token_accuracy": 0.5190408078953623,
+      "num_tokens": 3354390.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_coding_entropy": 1.9250439591407775,
+      "eval_coding_loss": 2.1208813190460205,
+      "eval_coding_mean_token_accuracy": 0.5191248273849487,
+      "eval_coding_num_tokens": 3354390.0,
+      "eval_coding_runtime": 79.1564,
+      "eval_coding_samples_per_second": 6.317,
+      "eval_coding_steps_per_second": 3.158,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_physics_entropy": 2.230250364303589,
+      "eval_physics_loss": 2.271095037460327,
+      "eval_physics_mean_token_accuracy": 0.5189732195138931,
+      "eval_physics_num_tokens": 3354390.0,
+      "eval_physics_runtime": 52.0613,
+      "eval_physics_samples_per_second": 9.604,
+      "eval_physics_steps_per_second": 4.802,
+      "step": 200
+    },
+    {
+      "entropy": 2.265408169478178,
+      "epoch": 0.168,
+      "grad_norm": 20.5,
+      "learning_rate": 4.18e-06,
+      "loss": 2.2294,
+      "mean_token_accuracy": 0.5198547914624214,
+      "num_tokens": 3521075.0,
+      "step": 210
+    },
+    {
+      "entropy": 2.2567644871771337,
+      "epoch": 0.176,
+      "grad_norm": 26.375,
+      "learning_rate": 4.38e-06,
+      "loss": 2.1962,
+      "mean_token_accuracy": 0.5276498578488826,
+      "num_tokens": 3684374.0,
+      "step": 220
+    },
+    {
+      "entropy": 2.2172055408358573,
+      "epoch": 0.184,
+      "grad_norm": 18.5,
+      "learning_rate": 4.58e-06,
+      "loss": 2.1651,
+      "mean_token_accuracy": 0.5290316773578525,
+      "num_tokens": 3848536.0,
+      "step": 230
+    },
+    {
+      "entropy": 2.152971290051937,
+      "epoch": 0.192,
+      "grad_norm": 18.75,
+      "learning_rate": 4.78e-06,
+      "loss": 2.1125,
+      "mean_token_accuracy": 0.5370275607332587,
+      "num_tokens": 4023915.0,
+      "step": 240
+    },
+    {
+      "entropy": 2.148153053969145,
+      "epoch": 0.2,
+      "grad_norm": 20.125,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 2.1104,
+      "mean_token_accuracy": 0.541122961603105,
+      "num_tokens": 4190485.0,
+      "step": 250
+    },
+    {
+      "entropy": 2.0243859082460403,
+      "epoch": 0.208,
+      "grad_norm": 18.125,
+      "learning_rate": 5.18e-06,
+      "loss": 1.975,
+      "mean_token_accuracy": 0.558369823731482,
+      "num_tokens": 4352045.0,
+      "step": 260
+    },
+    {
+      "entropy": 2.0731096126139166,
+      "epoch": 0.216,
+      "grad_norm": 19.625,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 2.0131,
+      "mean_token_accuracy": 0.5515947749838233,
+      "num_tokens": 4514042.0,
+      "step": 270
+    },
+    {
+      "entropy": 2.0090589359402657,
+      "epoch": 0.224,
+      "grad_norm": 17.375,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 1.9505,
+      "mean_token_accuracy": 0.5629682159051299,
+      "num_tokens": 4681371.0,
+      "step": 280
+    },
+    {
+      "entropy": 2.002196607738733,
+      "epoch": 0.232,
+      "grad_norm": 18.625,
+      "learning_rate": 5.78e-06,
+      "loss": 1.958,
+      "mean_token_accuracy": 0.5620508158579469,
+      "num_tokens": 4847650.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.9909410774707794,
+      "epoch": 0.24,
+      "grad_norm": 18.375,
+      "learning_rate": 5.98e-06,
+      "loss": 1.9375,
+      "mean_token_accuracy": 0.5657549701631069,
+      "num_tokens": 5021784.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_coding_entropy": 1.916715157032013,
+      "eval_coding_loss": 2.0919320583343506,
+      "eval_coding_mean_token_accuracy": 0.5256761227846145,
+      "eval_coding_num_tokens": 5021784.0,
+      "eval_coding_runtime": 79.1652,
+      "eval_coding_samples_per_second": 6.316,
+      "eval_coding_steps_per_second": 3.158,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_physics_entropy": 1.9569331226348876,
+      "eval_physics_loss": 1.9659218788146973,
+      "eval_physics_mean_token_accuracy": 0.5716171189546585,
+      "eval_physics_num_tokens": 5021784.0,
+      "eval_physics_runtime": 52.0616,
+      "eval_physics_samples_per_second": 9.604,
+      "eval_physics_steps_per_second": 4.802,
+      "step": 300
+    },
+    {
+      "entropy": 1.9608435668051243,
+      "epoch": 0.248,
+      "grad_norm": 17.125,
+      "learning_rate": 6.18e-06,
+      "loss": 1.9066,
+      "mean_token_accuracy": 0.5718545202165842,
+      "num_tokens": 5201507.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.929166065901518,
+      "epoch": 0.256,
+      "grad_norm": 16.875,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 1.8819,
+      "mean_token_accuracy": 0.5770876344293356,
+      "num_tokens": 5367308.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.9197401903569697,
+      "epoch": 0.264,
+      "grad_norm": 18.5,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 1.8598,
+      "mean_token_accuracy": 0.5781059630215168,
+      "num_tokens": 5536442.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.9471155002713203,
+      "epoch": 0.272,
+      "grad_norm": 21.5,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 1.8791,
+      "mean_token_accuracy": 0.5742578504607081,
+      "num_tokens": 5699101.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.8642883323132993,
+      "epoch": 0.28,
+      "grad_norm": 17.875,
+      "learning_rate": 6.98e-06,
+      "loss": 1.8298,
+      "mean_token_accuracy": 0.587579975090921,
+      "num_tokens": 5860398.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.883735204488039,
+      "epoch": 0.288,
+      "grad_norm": 19.75,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 1.8306,
+      "mean_token_accuracy": 0.5885121200233698,
+      "num_tokens": 6022837.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.832478718459606,
+      "epoch": 0.296,
+      "grad_norm": 17.0,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 1.7789,
+      "mean_token_accuracy": 0.5987123249098658,
+      "num_tokens": 6183758.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.8392824359238147,
+      "epoch": 0.304,
+      "grad_norm": 15.1875,
+      "learning_rate": 7.58e-06,
+      "loss": 1.7763,
+      "mean_token_accuracy": 0.595327608473599,
+      "num_tokens": 6343826.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.8003668662160635,
+      "epoch": 0.312,
+      "grad_norm": 19.5,
+      "learning_rate": 7.78e-06,
+      "loss": 1.7664,
+      "mean_token_accuracy": 0.5983955936506391,
+      "num_tokens": 6502713.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.7905294828116893,
+      "epoch": 0.32,
+      "grad_norm": 15.3125,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 1.7435,
+      "mean_token_accuracy": 0.6020280076190829,
+      "num_tokens": 6670164.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_coding_entropy": 1.8930716361999511,
+      "eval_coding_loss": 2.0930795669555664,
+      "eval_coding_mean_token_accuracy": 0.5246047481298447,
+      "eval_coding_num_tokens": 6670164.0,
+      "eval_coding_runtime": 79.1697,
+      "eval_coding_samples_per_second": 6.316,
+      "eval_coding_steps_per_second": 3.158,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_physics_entropy": 1.7485348978042603,
+      "eval_physics_loss": 1.7591620683670044,
+      "eval_physics_mean_token_accuracy": 0.6077653738260269,
+      "eval_physics_num_tokens": 6670164.0,
+      "eval_physics_runtime": 52.1144,
+      "eval_physics_samples_per_second": 9.594,
+      "eval_physics_steps_per_second": 4.797,
+      "step": 400
+    },
+    {
+      "entropy": 1.791950485110283,
+      "epoch": 0.328,
+      "grad_norm": 16.25,
+      "learning_rate": 8.18e-06,
+      "loss": 1.7309,
+      "mean_token_accuracy": 0.60343619287014,
+      "num_tokens": 6831503.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.6932228744029998,
+      "epoch": 0.336,
+      "grad_norm": 17.75,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 1.6384,
+      "mean_token_accuracy": 0.6203264124691487,
+      "num_tokens": 7003404.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.6966097623109817,
+      "epoch": 0.344,
+      "grad_norm": 19.75,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 1.6428,
+      "mean_token_accuracy": 0.6151626098901033,
+      "num_tokens": 7172909.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.688229451328516,
+      "epoch": 0.352,
+      "grad_norm": 18.625,
+      "learning_rate": 8.78e-06,
+      "loss": 1.6446,
+      "mean_token_accuracy": 0.619301825016737,
+      "num_tokens": 7342593.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.684376384317875,
+      "epoch": 0.36,
+      "grad_norm": 15.1875,
+      "learning_rate": 8.98e-06,
+      "loss": 1.6384,
+      "mean_token_accuracy": 0.6227533668279648,
+      "num_tokens": 7506313.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.6351792596280574,
+      "epoch": 0.368,
+      "grad_norm": 17.5,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 1.5883,
+      "mean_token_accuracy": 0.6302843233570457,
+      "num_tokens": 7675569.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.6966083988547325,
+      "epoch": 0.376,
+      "grad_norm": 18.625,
+      "learning_rate": 9.38e-06,
+      "loss": 1.6569,
+      "mean_token_accuracy": 0.6154216848313808,
+      "num_tokens": 7842654.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.6671229541301726,
+      "epoch": 0.384,
+      "grad_norm": 16.625,
+      "learning_rate": 9.58e-06,
+      "loss": 1.6255,
+      "mean_token_accuracy": 0.6238152993842959,
+      "num_tokens": 8012218.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.634845321625471,
+      "epoch": 0.392,
+      "grad_norm": 15.0,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 1.6026,
+      "mean_token_accuracy": 0.6318565379828215,
+      "num_tokens": 8181305.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.5764483150094748,
+      "epoch": 0.4,
+      "grad_norm": 16.125,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 1.5239,
+      "mean_token_accuracy": 0.6454750349745154,
+      "num_tokens": 8350120.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_coding_entropy": 1.9365736389160155,
+      "eval_coding_loss": 2.0959813594818115,
+      "eval_coding_mean_token_accuracy": 0.5250564186573029,
+      "eval_coding_num_tokens": 8350120.0,
+      "eval_coding_runtime": 79.1814,
+      "eval_coding_samples_per_second": 6.315,
+      "eval_coding_steps_per_second": 3.157,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_physics_entropy": 1.671481309890747,
+      "eval_physics_loss": 1.6045700311660767,
+      "eval_physics_mean_token_accuracy": 0.6350576342344284,
+      "eval_physics_num_tokens": 8350120.0,
+      "eval_physics_runtime": 51.8423,
+      "eval_physics_samples_per_second": 9.645,
+      "eval_physics_steps_per_second": 4.822,
+      "step": 500
+    },
+    {
+      "entropy": 1.5771338425576686,
+      "epoch": 0.408,
+      "grad_norm": 15.3125,
+      "learning_rate": 1.018e-05,
+      "loss": 1.5352,
+      "mean_token_accuracy": 0.6389181109145283,
+      "num_tokens": 8520118.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.5821339197456836,
+      "epoch": 0.416,
+      "grad_norm": 16.125,
+      "learning_rate": 1.038e-05,
+      "loss": 1.5332,
+      "mean_token_accuracy": 0.6355980787426233,
+      "num_tokens": 8688773.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.5948798354715108,
+      "epoch": 0.424,
+      "grad_norm": 15.625,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 1.5238,
+      "mean_token_accuracy": 0.6424462385475636,
+      "num_tokens": 8855184.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.529571657255292,
+      "epoch": 0.432,
+      "grad_norm": 14.0625,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 1.4919,
+      "mean_token_accuracy": 0.6440655821934342,
+      "num_tokens": 9022094.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.5117546495050191,
+      "epoch": 0.44,
+      "grad_norm": 16.375,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 1.4678,
+      "mean_token_accuracy": 0.6497685741633177,
+      "num_tokens": 9190509.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.513111385703087,
+      "epoch": 0.448,
+      "grad_norm": 16.25,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 1.4693,
+      "mean_token_accuracy": 0.6491031926125288,
+      "num_tokens": 9360608.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.470874259248376,
+      "epoch": 0.456,
+      "grad_norm": 14.3125,
+      "learning_rate": 1.138e-05,
+      "loss": 1.415,
+      "mean_token_accuracy": 0.6601609066128731,
+      "num_tokens": 9522789.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.5079048488289117,
+      "epoch": 0.464,
+      "grad_norm": 15.875,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 1.4715,
+      "mean_token_accuracy": 0.6530725235119462,
+      "num_tokens": 9688896.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.4853499613702297,
+      "epoch": 0.472,
+      "grad_norm": 14.125,
+      "learning_rate": 1.178e-05,
+      "loss": 1.4262,
+      "mean_token_accuracy": 0.6561389841139317,
+      "num_tokens": 9858464.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.5173018813133239,
+      "epoch": 0.48,
+      "grad_norm": 21.375,
+      "learning_rate": 1.198e-05,
+      "loss": 1.4895,
+      "mean_token_accuracy": 0.6496954914182425,
+      "num_tokens": 10025864.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_coding_entropy": 1.953908012866974,
+      "eval_coding_loss": 2.1033880710601807,
+      "eval_coding_mean_token_accuracy": 0.5249351325035095,
+      "eval_coding_num_tokens": 10025864.0,
+      "eval_coding_runtime": 79.131,
+      "eval_coding_samples_per_second": 6.319,
+      "eval_coding_steps_per_second": 3.159,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_physics_entropy": 1.541918823480606,
+      "eval_physics_loss": 1.4983493089675903,
+      "eval_physics_mean_token_accuracy": 0.6534626665115356,
+      "eval_physics_num_tokens": 10025864.0,
+      "eval_physics_runtime": 52.0434,
+      "eval_physics_samples_per_second": 9.607,
+      "eval_physics_steps_per_second": 4.804,
+      "step": 600
+    },
+    {
+      "entropy": 1.4748347599059344,
+      "epoch": 0.488,
+      "grad_norm": 16.625,
+      "learning_rate": 1.218e-05,
+      "loss": 1.4246,
+      "mean_token_accuracy": 0.6571080841124057,
+      "num_tokens": 10191232.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.5049679655581714,
+      "epoch": 0.496,
+      "grad_norm": 14.6875,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 1.4596,
+      "mean_token_accuracy": 0.6528839386999608,
+      "num_tokens": 10362573.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.4375012002885341,
+      "epoch": 0.504,
+      "grad_norm": 14.4375,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 1.3841,
+      "mean_token_accuracy": 0.6608929991722107,
+      "num_tokens": 10528450.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.5176888063549996,
+      "epoch": 0.512,
+      "grad_norm": 14.25,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 1.4569,
+      "mean_token_accuracy": 0.6528515750542283,
+      "num_tokens": 10694440.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.4692929420620202,
+      "epoch": 0.52,
+      "grad_norm": 10.6875,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 1.4449,
+      "mean_token_accuracy": 0.655676519870758,
+      "num_tokens": 10857597.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.4483005907386541,
+      "epoch": 0.528,
+      "grad_norm": 12.125,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 1.4046,
+      "mean_token_accuracy": 0.665894140675664,
+      "num_tokens": 11022357.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.3869127415120601,
+      "epoch": 0.536,
+      "grad_norm": 12.8125,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 1.3467,
+      "mean_token_accuracy": 0.672945449501276,
+      "num_tokens": 11185045.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.4482205394655465,
+      "epoch": 0.544,
+      "grad_norm": 12.8125,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 1.3907,
+      "mean_token_accuracy": 0.6631886938586831,
+      "num_tokens": 11345911.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.4304094772785902,
+      "epoch": 0.552,
+      "grad_norm": 13.125,
+      "learning_rate": 1.378e-05,
+      "loss": 1.3881,
+      "mean_token_accuracy": 0.6646447636187076,
+      "num_tokens": 11518974.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.3900580175220967,
+      "epoch": 0.56,
+      "grad_norm": 10.875,
+      "learning_rate": 1.398e-05,
+      "loss": 1.34,
+      "mean_token_accuracy": 0.6744402695447207,
+      "num_tokens": 11687225.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_coding_entropy": 1.8969693536758423,
+      "eval_coding_loss": 2.1377148628234863,
+      "eval_coding_mean_token_accuracy": 0.5214611231088638,
+      "eval_coding_num_tokens": 11687225.0,
+      "eval_coding_runtime": 79.1442,
+      "eval_coding_samples_per_second": 6.318,
+      "eval_coding_steps_per_second": 3.159,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_physics_entropy": 1.3678591132164002,
+      "eval_physics_loss": 1.3962782621383667,
+      "eval_physics_mean_token_accuracy": 0.6698634608983993,
+      "eval_physics_num_tokens": 11687225.0,
+      "eval_physics_runtime": 52.0406,
+      "eval_physics_samples_per_second": 9.608,
+      "eval_physics_steps_per_second": 4.804,
+      "step": 700
+    },
+    {
+      "entropy": 1.3716834906488657,
+      "epoch": 0.568,
+      "grad_norm": 18.125,
+      "learning_rate": 1.418e-05,
+      "loss": 1.342,
+      "mean_token_accuracy": 0.67361662350595,
+      "num_tokens": 11848324.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.3629539489746094,
+      "epoch": 0.576,
+      "grad_norm": 12.9375,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 1.3293,
+      "mean_token_accuracy": 0.6735996751114726,
+      "num_tokens": 12015720.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.3565699491649865,
+      "epoch": 0.584,
+      "grad_norm": 13.0625,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 1.3375,
+      "mean_token_accuracy": 0.6773057371377945,
+      "num_tokens": 12178798.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.3613810572773217,
+      "epoch": 0.592,
+      "grad_norm": 12.125,
+      "learning_rate": 1.478e-05,
+      "loss": 1.319,
+      "mean_token_accuracy": 0.6774894919246435,
+      "num_tokens": 12343393.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.3767595831304789,
+      "epoch": 0.6,
+      "grad_norm": 11.6875,
+      "learning_rate": 1.498e-05,
+      "loss": 1.3265,
+      "mean_token_accuracy": 0.6750151777639986,
+      "num_tokens": 12509336.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.3187911182641983,
+      "epoch": 0.608,
+      "grad_norm": 12.5,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 1.2905,
+      "mean_token_accuracy": 0.6845489364117384,
+      "num_tokens": 12671474.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.3427253626286983,
+      "epoch": 0.616,
+      "grad_norm": 10.75,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 1.2963,
+      "mean_token_accuracy": 0.6806909158825875,
+      "num_tokens": 12835956.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.3635981816798448,
+      "epoch": 0.624,
+      "grad_norm": 11.0,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 1.3102,
+      "mean_token_accuracy": 0.6765737216919661,
+      "num_tokens": 12996077.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.3231649551540614,
+      "epoch": 0.632,
+      "grad_norm": 12.375,
+      "learning_rate": 1.578e-05,
+      "loss": 1.2993,
+      "mean_token_accuracy": 0.6843808237463236,
+      "num_tokens": 13162966.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.3815045531839132,
+      "epoch": 0.64,
+      "grad_norm": 11.0,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 1.3403,
+      "mean_token_accuracy": 0.676003416068852,
+      "num_tokens": 13325291.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_coding_entropy": 1.8999772543907165,
+      "eval_coding_loss": 2.137219190597534,
+      "eval_coding_mean_token_accuracy": 0.5216508195400238,
+      "eval_coding_num_tokens": 13325291.0,
+      "eval_coding_runtime": 79.1227,
+      "eval_coding_samples_per_second": 6.319,
+      "eval_coding_steps_per_second": 3.16,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_physics_entropy": 1.3213207037448884,
+      "eval_physics_loss": 1.3272991180419922,
+      "eval_physics_mean_token_accuracy": 0.6834507223367691,
+      "eval_physics_num_tokens": 13325291.0,
+      "eval_physics_runtime": 52.0498,
+      "eval_physics_samples_per_second": 9.606,
+      "eval_physics_steps_per_second": 4.803,
+      "step": 800
+    },
+    {
+      "entropy": 1.3278860542923212,
+      "epoch": 0.648,
+      "grad_norm": 12.0,
+      "learning_rate": 1.618e-05,
+      "loss": 1.2916,
+      "mean_token_accuracy": 0.6818249279633164,
+      "num_tokens": 13495304.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.2705524377524853,
+      "epoch": 0.656,
+      "grad_norm": 10.875,
+      "learning_rate": 1.638e-05,
+      "loss": 1.2362,
+      "mean_token_accuracy": 0.6954903973266482,
+      "num_tokens": 13671434.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.3271659668534994,
+      "epoch": 0.664,
+      "grad_norm": 13.375,
+      "learning_rate": 1.658e-05,
+      "loss": 1.2753,
+      "mean_token_accuracy": 0.6867586117237806,
+      "num_tokens": 13829692.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.2718054752796888,
+      "epoch": 0.672,
+      "grad_norm": 11.125,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 1.2239,
+      "mean_token_accuracy": 0.6929404513910413,
+      "num_tokens": 13993668.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.2376167189329863,
+      "epoch": 0.68,
+      "grad_norm": 9.75,
+      "learning_rate": 1.698e-05,
+      "loss": 1.2134,
+      "mean_token_accuracy": 0.6992333011701704,
+      "num_tokens": 14173446.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.3201266448944806,
+      "epoch": 0.688,
+      "grad_norm": 9.0625,
+      "learning_rate": 1.718e-05,
+      "loss": 1.2773,
+      "mean_token_accuracy": 0.6858949340879917,
+      "num_tokens": 14334907.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.2640008773654698,
+      "epoch": 0.696,
+      "grad_norm": 8.875,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 1.237,
+      "mean_token_accuracy": 0.6919090248644352,
+      "num_tokens": 14512555.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.2943935077637434,
+      "epoch": 0.704,
+      "grad_norm": 11.9375,
+      "learning_rate": 1.758e-05,
+      "loss": 1.2627,
+      "mean_token_accuracy": 0.6883153811097145,
+      "num_tokens": 14670639.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.2209337592124938,
+      "epoch": 0.712,
+      "grad_norm": 10.5,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 1.2024,
+      "mean_token_accuracy": 0.7000889934599399,
+      "num_tokens": 14832916.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.2753230314701796,
+      "epoch": 0.72,
+      "grad_norm": 9.0,
+      "learning_rate": 1.798e-05,
+      "loss": 1.2268,
+      "mean_token_accuracy": 0.6935238633304834,
+      "num_tokens": 14987995.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_coding_entropy": 1.9145963129997254,
+      "eval_coding_loss": 2.1864311695098877,
+      "eval_coding_mean_token_accuracy": 0.5160261226892471,
+      "eval_coding_num_tokens": 14987995.0,
+      "eval_coding_runtime": 79.3569,
+      "eval_coding_samples_per_second": 6.301,
+      "eval_coding_steps_per_second": 3.15,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_physics_entropy": 1.288757406234741,
+      "eval_physics_loss": 1.2604119777679443,
+      "eval_physics_mean_token_accuracy": 0.6937535845041275,
+      "eval_physics_num_tokens": 14987995.0,
+      "eval_physics_runtime": 52.2126,
+      "eval_physics_samples_per_second": 9.576,
+      "eval_physics_steps_per_second": 4.788,
+      "step": 900
+    },
+    {
+      "entropy": 1.208376468345523,
+      "epoch": 0.728,
+      "grad_norm": 10.3125,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 1.1886,
+      "mean_token_accuracy": 0.7017698954790831,
+      "num_tokens": 15158982.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.2472068972885608,
+      "epoch": 0.736,
+      "grad_norm": 9.6875,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 1.2129,
+      "mean_token_accuracy": 0.6974750630557537,
+      "num_tokens": 15324659.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.225362879037857,
+      "epoch": 0.744,
+      "grad_norm": 10.4375,
+      "learning_rate": 1.858e-05,
+      "loss": 1.183,
+      "mean_token_accuracy": 0.7002156199887395,
+      "num_tokens": 15493621.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.2113514818251132,
+      "epoch": 0.752,
+      "grad_norm": 11.3125,
+      "learning_rate": 1.878e-05,
+      "loss": 1.1911,
+      "mean_token_accuracy": 0.7010745037347078,
+      "num_tokens": 15657342.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.251608308404684,
+      "epoch": 0.76,
+      "grad_norm": 9.0,
+      "learning_rate": 1.898e-05,
+      "loss": 1.2108,
+      "mean_token_accuracy": 0.6982708286494017,
+      "num_tokens": 15815189.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.2250138234347105,
+      "epoch": 0.768,
+      "grad_norm": 9.375,
+      "learning_rate": 1.918e-05,
+      "loss": 1.2033,
+      "mean_token_accuracy": 0.6997085321694613,
+      "num_tokens": 15988993.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2000590208917856,
+      "epoch": 0.776,
+      "grad_norm": 10.375,
+      "learning_rate": 1.938e-05,
+      "loss": 1.1522,
+      "mean_token_accuracy": 0.7049363922327757,
+      "num_tokens": 16163512.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.2060553971678019,
+      "epoch": 0.784,
+      "grad_norm": 7.875,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 1.1664,
+      "mean_token_accuracy": 0.7088450845330954,
+      "num_tokens": 16333857.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.1793909084051848,
+      "epoch": 0.792,
+      "grad_norm": 7.96875,
+      "learning_rate": 1.978e-05,
+      "loss": 1.1466,
+      "mean_token_accuracy": 0.709895571321249,
+      "num_tokens": 16494118.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.2104855172336102,
+      "epoch": 0.8,
+      "grad_norm": 10.0625,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 1.1781,
+      "mean_token_accuracy": 0.7040123168379069,
+      "num_tokens": 16664690.0,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_coding_entropy": 1.9454177751541137,
+      "eval_coding_loss": 2.2875986099243164,
+      "eval_coding_mean_token_accuracy": 0.49295639526844026,
+      "eval_coding_num_tokens": 16664690.0,
+      "eval_coding_runtime": 79.1544,
+      "eval_coding_samples_per_second": 6.317,
+      "eval_coding_steps_per_second": 3.158,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_physics_entropy": 1.249910979270935,
+      "eval_physics_loss": 1.215478539466858,
+      "eval_physics_mean_token_accuracy": 0.7021359713077545,
+      "eval_physics_num_tokens": 16664690.0,
+      "eval_physics_runtime": 51.8213,
+      "eval_physics_samples_per_second": 9.649,
+      "eval_physics_steps_per_second": 4.824,
+      "step": 1000
+    },
+    {
+      "entropy": 1.2079178389161824,
+      "epoch": 0.808,
+      "grad_norm": 10.6875,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 1.1621,
+      "mean_token_accuracy": 0.7027403622865677,
+      "num_tokens": 16829315.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.1786697298288344,
+      "epoch": 0.816,
+      "grad_norm": 8.0,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 1.1494,
+      "mean_token_accuracy": 0.7092785466462374,
+      "num_tokens": 16989307.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.1985579200088978,
+      "epoch": 0.824,
+      "grad_norm": 9.25,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 1.1467,
+      "mean_token_accuracy": 0.7067175708711148,
+      "num_tokens": 17155547.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.164666483551264,
+      "epoch": 0.832,
+      "grad_norm": 7.71875,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 1.1401,
+      "mean_token_accuracy": 0.7113697534427047,
+      "num_tokens": 17317524.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.1878154788166284,
+      "epoch": 0.84,
+      "grad_norm": 6.78125,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 1.1408,
+      "mean_token_accuracy": 0.7077680010348558,
+      "num_tokens": 17479138.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.1530823826789856,
+      "epoch": 0.848,
+      "grad_norm": 6.90625,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 1.1155,
+      "mean_token_accuracy": 0.71307920999825,
+      "num_tokens": 17650424.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.1825125798583032,
+      "epoch": 0.856,
+      "grad_norm": 7.8125,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 1.1398,
+      "mean_token_accuracy": 0.7109603315591813,
+      "num_tokens": 17814512.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.1390453603118658,
+      "epoch": 0.864,
+      "grad_norm": 6.53125,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 1.1154,
+      "mean_token_accuracy": 0.7171653687953949,
+      "num_tokens": 17985547.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.1706372637301683,
+      "epoch": 0.872,
+      "grad_norm": 8.0625,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 1.1272,
+      "mean_token_accuracy": 0.7104873843491077,
+      "num_tokens": 18147038.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.2016708802431821,
+      "epoch": 0.88,
+      "grad_norm": 7.9375,
+      "learning_rate": 1.978e-05,
+      "loss": 1.1414,
+      "mean_token_accuracy": 0.7089165017008782,
+      "num_tokens": 18315392.0,
+      "step": 1100
+    },
+    {
+      "epoch": 0.88,
+      "eval_coding_entropy": 1.9011136808395386,
+      "eval_coding_loss": 2.2340805530548096,
+      "eval_coding_mean_token_accuracy": 0.5084248929023742,
+      "eval_coding_num_tokens": 18315392.0,
+      "eval_coding_runtime": 79.1322,
+      "eval_coding_samples_per_second": 6.319,
+      "eval_coding_steps_per_second": 3.159,
+      "step": 1100
+    },
+    {
+      "epoch": 0.88,
+      "eval_physics_entropy": 1.1309143052101136,
+      "eval_physics_loss": 1.1532424688339233,
+      "eval_physics_mean_token_accuracy": 0.7144835109710693,
+      "eval_physics_num_tokens": 18315392.0,
+      "eval_physics_runtime": 52.1221,
+      "eval_physics_samples_per_second": 9.593,
+      "eval_physics_steps_per_second": 4.796,
+      "step": 1100
+    },
+    {
+      "entropy": 1.0508298195898533,
+      "epoch": 0.888,
+      "grad_norm": 8.25,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 1.0332,
+      "mean_token_accuracy": 0.7302042797207833,
+      "num_tokens": 18495457.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.1058499403297901,
+      "epoch": 0.896,
+      "grad_norm": 5.90625,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 1.0849,
+      "mean_token_accuracy": 0.7238155752420425,
+      "num_tokens": 18667124.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.1259642377495767,
+      "epoch": 0.904,
+      "grad_norm": 6.875,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 1.0818,
+      "mean_token_accuracy": 0.721034574508667,
+      "num_tokens": 18833366.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.145649480447173,
+      "epoch": 0.912,
+      "grad_norm": 7.65625,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 1.1107,
+      "mean_token_accuracy": 0.7126699529588223,
+      "num_tokens": 19002186.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.1212836857885122,
+      "epoch": 0.92,
+      "grad_norm": 6.5625,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 1.0845,
+      "mean_token_accuracy": 0.7232643336057663,
+      "num_tokens": 19167898.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.1022184763103724,
+      "epoch": 0.928,
+      "grad_norm": 7.78125,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 1.0742,
+      "mean_token_accuracy": 0.7239976346492767,
+      "num_tokens": 19335013.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.081941632181406,
+      "epoch": 0.936,
+      "grad_norm": 7.9375,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 1.0521,
+      "mean_token_accuracy": 0.7266536355018616,
+      "num_tokens": 19497952.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.1005669727921485,
+      "epoch": 0.944,
+      "grad_norm": 6.09375,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 1.0613,
+      "mean_token_accuracy": 0.7255159001797438,
+      "num_tokens": 19662733.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.0804655332118274,
+      "epoch": 0.952,
+      "grad_norm": 6.65625,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 1.0475,
+      "mean_token_accuracy": 0.728959995508194,
+      "num_tokens": 19829333.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.069432857632637,
+      "epoch": 0.96,
+      "grad_norm": 6.78125,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 1.0476,
+      "mean_token_accuracy": 0.7279221322387457,
+      "num_tokens": 19996789.0,
+      "step": 1200
+    },
+    {
+      "epoch": 0.96,
+      "eval_coding_entropy": 1.8921072945594788,
+      "eval_coding_loss": 2.2613472938537598,
+      "eval_coding_mean_token_accuracy": 0.5047942440509796,
+      "eval_coding_num_tokens": 19996789.0,
+      "eval_coding_runtime": 79.1902,
+      "eval_coding_samples_per_second": 6.314,
+      "eval_coding_steps_per_second": 3.157,
+      "step": 1200
+    },
+    {
+      "epoch": 0.96,
+      "eval_physics_entropy": 1.099768555045128,
+      "eval_physics_loss": 1.101595401763916,
+      "eval_physics_mean_token_accuracy": 0.7247749812602997,
+      "eval_physics_num_tokens": 19996789.0,
+      "eval_physics_runtime": 52.0796,
+      "eval_physics_samples_per_second": 9.601,
+      "eval_physics_steps_per_second": 4.8,
+      "step": 1200
+    },
+    {
+      "entropy": 1.0720167379826306,
+      "epoch": 0.968,
+      "grad_norm": 7.03125,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 1.0638,
+      "mean_token_accuracy": 0.7312380861490965,
+      "num_tokens": 20161977.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.088957086019218,
+      "epoch": 0.976,
+      "grad_norm": 7.34375,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 1.0538,
+      "mean_token_accuracy": 0.7287674982100725,
+      "num_tokens": 20332636.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.08813561424613,
+      "epoch": 0.984,
+      "grad_norm": 6.34375,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 1.0477,
+      "mean_token_accuracy": 0.729033438116312,
+      "num_tokens": 20495388.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.0714545506983995,
+      "epoch": 0.992,
+      "grad_norm": 6.53125,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 1.0405,
+      "mean_token_accuracy": 0.7327663522213698,
+      "num_tokens": 20662209.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.0615619242191314,
+      "epoch": 1.0,
+      "grad_norm": 6.78125,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 1.0355,
+      "mean_token_accuracy": 0.7302098982036114,
+      "num_tokens": 20832148.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.9550450762733818,
+      "epoch": 1.008,
+      "grad_norm": 5.03125,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 0.9201,
+      "mean_token_accuracy": 0.7552305307239294,
+      "num_tokens": 21005731.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.007374680787325,
+      "epoch": 1.016,
+      "grad_norm": 5.5625,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 0.972,
+      "mean_token_accuracy": 0.7435515383258462,
+      "num_tokens": 21167221.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.9439798355102539,
+      "epoch": 1.024,
+      "grad_norm": 6.0625,
+      "learning_rate": 1.938e-05,
+      "loss": 0.9237,
+      "mean_token_accuracy": 0.7548253701999783,
+      "num_tokens": 21335407.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.9643905458971858,
+      "epoch": 1.032,
+      "grad_norm": 5.8125,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 0.9309,
+      "mean_token_accuracy": 0.752015108242631,
+      "num_tokens": 21501759.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.9763634527102113,
+      "epoch": 1.04,
+      "grad_norm": 6.4375,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 0.9625,
+      "mean_token_accuracy": 0.7475563909858465,
+      "num_tokens": 21668490.0,
+      "step": 1300
+    },
+    {
+      "epoch": 1.04,
+      "eval_coding_entropy": 1.7818276858329773,
+      "eval_coding_loss": 2.242872953414917,
+      "eval_coding_mean_token_accuracy": 0.5101647647619247,
+      "eval_coding_num_tokens": 21668490.0,
+      "eval_coding_runtime": 79.2066,
+      "eval_coding_samples_per_second": 6.313,
+      "eval_coding_steps_per_second": 3.156,
+      "step": 1300
+    },
+    {
+      "epoch": 1.04,
+      "eval_physics_entropy": 1.046526582121849,
+      "eval_physics_loss": 1.0569711923599243,
+      "eval_physics_mean_token_accuracy": 0.7336113708019256,
+      "eval_physics_num_tokens": 21668490.0,
+      "eval_physics_runtime": 52.1165,
+      "eval_physics_samples_per_second": 9.594,
+      "eval_physics_steps_per_second": 4.797,
+      "step": 1300
+    },
+    {
+      "entropy": 0.9808619644492864,
+      "epoch": 1.048,
+      "grad_norm": 5.9375,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 0.95,
+      "mean_token_accuracy": 0.7478477224707604,
+      "num_tokens": 21835179.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.939110497944057,
+      "epoch": 1.056,
+      "grad_norm": 5.53125,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 0.9088,
+      "mean_token_accuracy": 0.755623085796833,
+      "num_tokens": 22006039.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.9877023957669735,
+      "epoch": 1.064,
+      "grad_norm": 5.25,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 0.9472,
+      "mean_token_accuracy": 0.7455626420676709,
+      "num_tokens": 22176185.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.9839252684265375,
+      "epoch": 1.072,
+      "grad_norm": 5.53125,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 0.9612,
+      "mean_token_accuracy": 0.7474370829761028,
+      "num_tokens": 22348206.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.9325820317491889,
+      "epoch": 1.08,
+      "grad_norm": 6.375,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 0.896,
+      "mean_token_accuracy": 0.7573953222483396,
+      "num_tokens": 22513925.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.9836602587252855,
+      "epoch": 1.088,
+      "grad_norm": 6.84375,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 0.9531,
+      "mean_token_accuracy": 0.7478942912071943,
+      "num_tokens": 22676163.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.9665768353268505,
+      "epoch": 1.096,
+      "grad_norm": 5.96875,
+      "learning_rate": 1.918e-05,
+      "loss": 0.9489,
+      "mean_token_accuracy": 0.7473849574103951,
+      "num_tokens": 22841009.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.9201742075383663,
+      "epoch": 1.104,
+      "grad_norm": 6.0,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 0.8934,
+      "mean_token_accuracy": 0.7597939491271972,
+      "num_tokens": 23009279.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.9489134527742863,
+      "epoch": 1.112,
+      "grad_norm": 5.71875,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 0.9142,
+      "mean_token_accuracy": 0.7531155597418546,
+      "num_tokens": 23177689.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.9486297775059939,
+      "epoch": 1.12,
+      "grad_norm": 6.1875,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 0.9228,
+      "mean_token_accuracy": 0.7521506313234567,
+      "num_tokens": 23346828.0,
+      "step": 1400
+    },
+    {
+      "epoch": 1.12,
+      "eval_coding_entropy": 1.7531000714302063,
+      "eval_coding_loss": 2.281940460205078,
+      "eval_coding_mean_token_accuracy": 0.5089057025909424,
+      "eval_coding_num_tokens": 23346828.0,
+      "eval_coding_runtime": 79.1787,
+      "eval_coding_samples_per_second": 6.315,
+      "eval_coding_steps_per_second": 3.157,
+      "step": 1400
+    },
+    {
+      "epoch": 1.12,
+      "eval_physics_entropy": 0.9558427100181579,
+      "eval_physics_loss": 1.0335971117019653,
+      "eval_physics_mean_token_accuracy": 0.7392593598365784,
+      "eval_physics_num_tokens": 23346828.0,
+      "eval_physics_runtime": 52.0901,
+      "eval_physics_samples_per_second": 9.599,
+      "eval_physics_steps_per_second": 4.799,
+      "step": 1400
+    },
+    {
+      "entropy": 0.9056999972090125,
+      "epoch": 1.1280000000000001,
+      "grad_norm": 6.5,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 0.8848,
+      "mean_token_accuracy": 0.7649097748100757,
+      "num_tokens": 23518156.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.9454022387042642,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 5.3125,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 0.9168,
+      "mean_token_accuracy": 0.755837232619524,
+      "num_tokens": 23689161.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.9329152461141348,
+      "epoch": 1.144,
+      "grad_norm": 5.59375,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 0.9142,
+      "mean_token_accuracy": 0.758990440890193,
+      "num_tokens": 23851046.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.961996317282319,
+      "epoch": 1.152,
+      "grad_norm": 4.875,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 0.9288,
+      "mean_token_accuracy": 0.7505928639322519,
+      "num_tokens": 24016439.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.9266120318323374,
+      "epoch": 1.16,
+      "grad_norm": 6.09375,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 0.9021,
+      "mean_token_accuracy": 0.7583789460361003,
+      "num_tokens": 24180461.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.9504798075184226,
+      "epoch": 1.168,
+      "grad_norm": 6.1875,
+      "learning_rate": 1.898e-05,
+      "loss": 0.9328,
+      "mean_token_accuracy": 0.7518207874149084,
+      "num_tokens": 24344431.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.9474708326160908,
+      "epoch": 1.176,
+      "grad_norm": 5.8125,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 0.9112,
+      "mean_token_accuracy": 0.7553022742271424,
+      "num_tokens": 24507213.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.9036590620875359,
+      "epoch": 1.184,
+      "grad_norm": 6.21875,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 0.8795,
+      "mean_token_accuracy": 0.762787076830864,
+      "num_tokens": 24668087.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.9184596356004476,
+      "epoch": 1.192,
+      "grad_norm": 5.53125,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 0.8956,
+      "mean_token_accuracy": 0.7611413709819317,
+      "num_tokens": 24832180.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.921324947103858,
+      "epoch": 1.2,
+      "grad_norm": 5.65625,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 0.8789,
+      "mean_token_accuracy": 0.7592108882963657,
+      "num_tokens": 25000629.0,
+      "step": 1500
+    },
+    {
+      "epoch": 1.2,
+      "eval_coding_entropy": 1.7903581528663635,
+      "eval_coding_loss": 2.2501142024993896,
+      "eval_coding_mean_token_accuracy": 0.513159777879715,
+      "eval_coding_num_tokens": 25000629.0,
+      "eval_coding_runtime": 79.2232,
+      "eval_coding_samples_per_second": 6.311,
+      "eval_coding_steps_per_second": 3.156,
+      "step": 1500
+    },
+    {
+      "epoch": 1.2,
+      "eval_physics_entropy": 0.990718915104866,
+      "eval_physics_loss": 0.9999967813491821,
+      "eval_physics_mean_token_accuracy": 0.7454895977973938,
+      "eval_physics_num_tokens": 25000629.0,
+      "eval_physics_runtime": 51.8001,
+      "eval_physics_samples_per_second": 9.652,
+      "eval_physics_steps_per_second": 4.826,
+      "step": 1500
+    },
+    {
+      "entropy": 0.9167924275621772,
+      "epoch": 1.208,
+      "grad_norm": 5.9375,
+      "learning_rate": 1.886888888888889e-05,
+      "loss": 0.892,
+      "mean_token_accuracy": 0.7588412739336491,
+      "num_tokens": 25164342.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.9410123957321048,
+      "epoch": 1.216,
+      "grad_norm": 5.40625,
+      "learning_rate": 1.884666666666667e-05,
+      "loss": 0.9122,
+      "mean_token_accuracy": 0.7562039643526077,
+      "num_tokens": 25324324.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.9521240815520287,
+      "epoch": 1.224,
+      "grad_norm": 5.0625,
+      "learning_rate": 1.8824444444444445e-05,
+      "loss": 0.8969,
+      "mean_token_accuracy": 0.7552298158407211,
+      "num_tokens": 25490292.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.8985677545890212,
+      "epoch": 1.232,
+      "grad_norm": 5.5625,
+      "learning_rate": 1.8802222222222226e-05,
+      "loss": 0.8661,
+      "mean_token_accuracy": 0.76500195376575,
+      "num_tokens": 25658795.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.8914181362837553,
+      "epoch": 1.24,
+      "grad_norm": 5.90625,
+      "learning_rate": 1.878e-05,
+      "loss": 0.8667,
+      "mean_token_accuracy": 0.7672365583479405,
+      "num_tokens": 25823796.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.9169950911775231,
+      "epoch": 1.248,
+      "grad_norm": 7.0625,
+      "learning_rate": 1.875777777777778e-05,
+      "loss": 0.8949,
+      "mean_token_accuracy": 0.7595178321003914,
+      "num_tokens": 25993177.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.9169469196349382,
+      "epoch": 1.256,
+      "grad_norm": 5.4375,
+      "learning_rate": 1.873555555555556e-05,
+      "loss": 0.8825,
+      "mean_token_accuracy": 0.7615648206323385,
+      "num_tokens": 26162587.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.896455617621541,
+      "epoch": 1.264,
+      "grad_norm": 4.625,
+      "learning_rate": 1.8713333333333336e-05,
+      "loss": 0.8742,
+      "mean_token_accuracy": 0.7666802130639553,
+      "num_tokens": 26329726.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.9278371959924698,
+      "epoch": 1.272,
+      "grad_norm": 5.0625,
+      "learning_rate": 1.8691111111111114e-05,
+      "loss": 0.8965,
+      "mean_token_accuracy": 0.7571958255022764,
+      "num_tokens": 26500094.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.9765459138900041,
+      "epoch": 1.28,
+      "grad_norm": 6.15625,
+      "learning_rate": 1.866888888888889e-05,
+      "loss": 0.9344,
+      "mean_token_accuracy": 0.7511742908507586,
+      "num_tokens": 26664093.0,
+      "step": 1600
+    },
+    {
+      "epoch": 1.28,
+      "eval_coding_entropy": 1.6453372964859008,
+      "eval_coding_loss": 2.296245574951172,
+      "eval_coding_mean_token_accuracy": 0.510434584736824,
+      "eval_coding_num_tokens": 26664093.0,
+      "eval_coding_runtime": 79.1166,
+      "eval_coding_samples_per_second": 6.32,
+      "eval_coding_steps_per_second": 3.16,
+      "step": 1600
+    },
+    {
+      "epoch": 1.28,
+      "eval_physics_entropy": 0.9116508244276047,
+      "eval_physics_loss": 0.981630265712738,
+      "eval_physics_mean_token_accuracy": 0.7496122629642487,
+      "eval_physics_num_tokens": 26664093.0,
+      "eval_physics_runtime": 51.9369,
+      "eval_physics_samples_per_second": 9.627,
+      "eval_physics_steps_per_second": 4.814,
+      "step": 1600
+    },
+    {
+      "entropy": 0.862612402252853,
+      "epoch": 1.288,
+      "grad_norm": 4.9375,
+      "learning_rate": 1.864666666666667e-05,
+      "loss": 0.8454,
+      "mean_token_accuracy": 0.7705432698130608,
+      "num_tokens": 26834758.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.9043381884694099,
+      "epoch": 1.296,
+      "grad_norm": 5.15625,
+      "learning_rate": 1.8624444444444446e-05,
+      "loss": 0.8718,
+      "mean_token_accuracy": 0.7643989242613316,
+      "num_tokens": 26994877.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.9238925974816083,
+      "epoch": 1.304,
+      "grad_norm": 4.375,
+      "learning_rate": 1.8602222222222224e-05,
+      "loss": 0.8873,
+      "mean_token_accuracy": 0.7609186865389347,
+      "num_tokens": 27162719.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.9316502708941699,
+      "epoch": 1.312,
+      "grad_norm": 5.03125,
+      "learning_rate": 1.858e-05,
+      "loss": 0.907,
+      "mean_token_accuracy": 0.7582122638821602,
+      "num_tokens": 27328198.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.9211711158975959,
+      "epoch": 1.32,
+      "grad_norm": 5.34375,
+      "learning_rate": 1.855777777777778e-05,
+      "loss": 0.8892,
+      "mean_token_accuracy": 0.7606807623058558,
+      "num_tokens": 27492744.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.892648309096694,
+      "epoch": 1.328,
+      "grad_norm": 5.28125,
+      "learning_rate": 1.8535555555555557e-05,
+      "loss": 0.8718,
+      "mean_token_accuracy": 0.7630397897213698,
+      "num_tokens": 27658818.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.8834815826267004,
+      "epoch": 1.336,
+      "grad_norm": 5.96875,
+      "learning_rate": 1.8513333333333335e-05,
+      "loss": 0.8589,
+      "mean_token_accuracy": 0.7691333398222924,
+      "num_tokens": 27821424.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.9194148594513536,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 5.25,
+      "learning_rate": 1.8491111111111112e-05,
+      "loss": 0.8972,
+      "mean_token_accuracy": 0.7602852337062359,
+      "num_tokens": 27991108.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.9116265259683132,
+      "epoch": 1.3519999999999999,
+      "grad_norm": 5.53125,
+      "learning_rate": 1.846888888888889e-05,
+      "loss": 0.8777,
+      "mean_token_accuracy": 0.7609406780451536,
+      "num_tokens": 28154773.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.8890536414459348,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 4.8125,
+      "learning_rate": 1.8446666666666667e-05,
+      "loss": 0.8562,
+      "mean_token_accuracy": 0.7662287544459104,
+      "num_tokens": 28327775.0,
+      "step": 1700
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "eval_coding_entropy": 1.716024712085724,
+      "eval_coding_loss": 2.259434461593628,
+      "eval_coding_mean_token_accuracy": 0.5130739635229111,
+      "eval_coding_num_tokens": 28327775.0,
+      "eval_coding_runtime": 78.6033,
+      "eval_coding_samples_per_second": 6.361,
+      "eval_coding_steps_per_second": 3.181,
+      "step": 1700
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "eval_physics_entropy": 0.914379727602005,
+      "eval_physics_loss": 0.9594675302505493,
+      "eval_physics_mean_token_accuracy": 0.7536831333637237,
+      "eval_physics_num_tokens": 28327775.0,
+      "eval_physics_runtime": 51.7228,
+      "eval_physics_samples_per_second": 9.667,
+      "eval_physics_steps_per_second": 4.833,
+      "step": 1700
+    },
+    {
+      "entropy": 0.8726189162582159,
+      "epoch": 1.3679999999999999,
+      "grad_norm": 5.0,
+      "learning_rate": 1.842444444444445e-05,
+      "loss": 0.8495,
+      "mean_token_accuracy": 0.7711174737662077,
+      "num_tokens": 28498470.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.9009999068453908,
+      "epoch": 1.376,
+      "grad_norm": 5.03125,
+      "learning_rate": 1.8402222222222223e-05,
+      "loss": 0.8623,
+      "mean_token_accuracy": 0.7660522252321244,
+      "num_tokens": 28666116.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.8770303575322032,
+      "epoch": 1.384,
+      "grad_norm": 5.1875,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 0.8659,
+      "mean_token_accuracy": 0.767541291192174,
+      "num_tokens": 28840893.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.8704624468460679,
+      "epoch": 1.392,
+      "grad_norm": 6.0,
+      "learning_rate": 1.8357777777777778e-05,
+      "loss": 0.8293,
+      "mean_token_accuracy": 0.7710605084896087,
+      "num_tokens": 29004293.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.866730142198503,
+      "epoch": 1.4,
+      "grad_norm": 5.125,
+      "learning_rate": 1.833555555555556e-05,
+      "loss": 0.8459,
+      "mean_token_accuracy": 0.7679262559860944,
+      "num_tokens": 29167707.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.8981963096186518,
+      "epoch": 1.408,
+      "grad_norm": 4.78125,
+      "learning_rate": 1.8313333333333333e-05,
+      "loss": 0.8666,
+      "mean_token_accuracy": 0.7673941906541586,
+      "num_tokens": 29330131.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.9249771688133478,
+      "epoch": 1.416,
+      "grad_norm": 4.9375,
+      "learning_rate": 1.8291111111111114e-05,
+      "loss": 0.9063,
+      "mean_token_accuracy": 0.7583306394517422,
+      "num_tokens": 29491469.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.8935416838154197,
+      "epoch": 1.424,
+      "grad_norm": 4.375,
+      "learning_rate": 1.8268888888888888e-05,
+      "loss": 0.8645,
+      "mean_token_accuracy": 0.7656872101128102,
+      "num_tokens": 29661850.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.8505860712379217,
+      "epoch": 1.432,
+      "grad_norm": 5.4375,
+      "learning_rate": 1.824666666666667e-05,
+      "loss": 0.8276,
+      "mean_token_accuracy": 0.7741888701915741,
+      "num_tokens": 29825641.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.862689713574946,
+      "epoch": 1.44,
+      "grad_norm": 5.21875,
+      "learning_rate": 1.8224444444444447e-05,
+      "loss": 0.8296,
+      "mean_token_accuracy": 0.7719305366277694,
+      "num_tokens": 29999392.0,
+      "step": 1800
+    },
+    {
+      "epoch": 1.44,
+      "eval_coding_entropy": 1.8027336740493773,
+      "eval_coding_loss": 2.246039628982544,
+      "eval_coding_mean_token_accuracy": 0.5113436381816864,
+      "eval_coding_num_tokens": 29999392.0,
+      "eval_coding_runtime": 79.2093,
+      "eval_coding_samples_per_second": 6.312,
+      "eval_coding_steps_per_second": 3.156,
+      "step": 1800
+    },
+    {
+      "epoch": 1.44,
+      "eval_physics_entropy": 0.913623704791069,
+      "eval_physics_loss": 0.9421977996826172,
+      "eval_physics_mean_token_accuracy": 0.7563698604106903,
+      "eval_physics_num_tokens": 29999392.0,
+      "eval_physics_runtime": 51.9126,
+      "eval_physics_samples_per_second": 9.632,
+      "eval_physics_steps_per_second": 4.816,
+      "step": 1800
+    },
+    {
+      "entropy": 0.876362593844533,
+      "epoch": 1.448,
+      "grad_norm": 5.71875,
+      "learning_rate": 1.8202222222222225e-05,
+      "loss": 0.8509,
+      "mean_token_accuracy": 0.7696236498653889,
+      "num_tokens": 30164371.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.847023755684495,
+      "epoch": 1.456,
+      "grad_norm": 4.625,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 0.8231,
+      "mean_token_accuracy": 0.7763396494090558,
+      "num_tokens": 30341004.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.871413410641253,
+      "epoch": 1.464,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.815777777777778e-05,
+      "loss": 0.8386,
+      "mean_token_accuracy": 0.7727533802390099,
+      "num_tokens": 30502703.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.939422488771379,
+      "epoch": 1.472,
+      "grad_norm": 5.46875,
+      "learning_rate": 1.8135555555555557e-05,
+      "loss": 0.9126,
+      "mean_token_accuracy": 0.7551350846886635,
+      "num_tokens": 30667689.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.8579436099156738,
+      "epoch": 1.48,
+      "grad_norm": 4.71875,
+      "learning_rate": 1.8113333333333335e-05,
+      "loss": 0.851,
+      "mean_token_accuracy": 0.772059577703476,
+      "num_tokens": 30836874.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.8878307269886136,
+      "epoch": 1.488,
+      "grad_norm": 5.0625,
+      "learning_rate": 1.8091111111111113e-05,
+      "loss": 0.8514,
+      "mean_token_accuracy": 0.7688593063503504,
+      "num_tokens": 30999697.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.8808000456541777,
+      "epoch": 1.496,
+      "grad_norm": 5.09375,
+      "learning_rate": 1.806888888888889e-05,
+      "loss": 0.8501,
+      "mean_token_accuracy": 0.770066749304533,
+      "num_tokens": 31166146.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.8454932847991585,
+      "epoch": 1.504,
+      "grad_norm": 4.875,
+      "learning_rate": 1.8046666666666668e-05,
+      "loss": 0.8113,
+      "mean_token_accuracy": 0.7748839784413576,
+      "num_tokens": 31332775.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.8540815737098455,
+      "epoch": 1.512,
+      "grad_norm": 4.6875,
+      "learning_rate": 1.8024444444444445e-05,
+      "loss": 0.8343,
+      "mean_token_accuracy": 0.7708807934075594,
+      "num_tokens": 31504675.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.9056571874767542,
+      "epoch": 1.52,
+      "grad_norm": 5.03125,
+      "learning_rate": 1.8002222222222223e-05,
+      "loss": 0.8672,
+      "mean_token_accuracy": 0.7641353111714124,
+      "num_tokens": 31666521.0,
+      "step": 1900
+    },
+    {
+      "epoch": 1.52,
+      "eval_coding_entropy": 1.6874914655685425,
+      "eval_coding_loss": 2.2765512466430664,
+      "eval_coding_mean_token_accuracy": 0.5112714154720306,
+      "eval_coding_num_tokens": 31666521.0,
+      "eval_coding_runtime": 79.0285,
+      "eval_coding_samples_per_second": 6.327,
+      "eval_coding_steps_per_second": 3.163,
+      "step": 1900
+    },
+    {
+      "epoch": 1.52,
+      "eval_physics_entropy": 0.8699047968387604,
+      "eval_physics_loss": 0.9348219037055969,
+      "eval_physics_mean_token_accuracy": 0.7589149172306061,
+      "eval_physics_num_tokens": 31666521.0,
+      "eval_physics_runtime": 51.8968,
+      "eval_physics_samples_per_second": 9.635,
+      "eval_physics_steps_per_second": 4.817,
+      "step": 1900
+    },
+    {
+      "entropy": 0.8628190591931343,
+      "epoch": 1.528,
+      "grad_norm": 6.5,
+      "learning_rate": 1.798e-05,
+      "loss": 0.842,
+      "mean_token_accuracy": 0.7697089036926628,
+      "num_tokens": 31831967.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.9051342781633138,
+      "epoch": 1.536,
+      "grad_norm": 4.9375,
+      "learning_rate": 1.7957777777777778e-05,
+      "loss": 0.8713,
+      "mean_token_accuracy": 0.7649960003793239,
+      "num_tokens": 31998232.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.8381768845021724,
+      "epoch": 1.544,
+      "grad_norm": 4.6875,
+      "learning_rate": 1.7935555555555556e-05,
+      "loss": 0.8225,
+      "mean_token_accuracy": 0.7794440522789955,
+      "num_tokens": 32163122.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.8429474547505379,
+      "epoch": 1.552,
+      "grad_norm": 4.59375,
+      "learning_rate": 1.7913333333333337e-05,
+      "loss": 0.8141,
+      "mean_token_accuracy": 0.77677460052073,
+      "num_tokens": 32328085.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.8625445317476987,
+      "epoch": 1.56,
+      "grad_norm": 4.65625,
+      "learning_rate": 1.789111111111111e-05,
+      "loss": 0.8221,
+      "mean_token_accuracy": 0.7729156997054816,
+      "num_tokens": 32498672.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.8660145524889231,
+      "epoch": 1.568,
+      "grad_norm": 4.59375,
+      "learning_rate": 1.7868888888888892e-05,
+      "loss": 0.8336,
+      "mean_token_accuracy": 0.7706921048462391,
+      "num_tokens": 32656603.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.8720831394195556,
+      "epoch": 1.576,
+      "grad_norm": 4.8125,
+      "learning_rate": 1.7846666666666666e-05,
+      "loss": 0.8522,
+      "mean_token_accuracy": 0.7667239267379046,
+      "num_tokens": 32815222.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.8575057543814182,
+      "epoch": 1.584,
+      "grad_norm": 4.875,
+      "learning_rate": 1.7824444444444447e-05,
+      "loss": 0.8218,
+      "mean_token_accuracy": 0.7737087935209275,
+      "num_tokens": 32983769.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.8643774883821607,
+      "epoch": 1.592,
+      "grad_norm": 4.59375,
+      "learning_rate": 1.780222222222222e-05,
+      "loss": 0.8371,
+      "mean_token_accuracy": 0.7728578582406044,
+      "num_tokens": 33144736.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.8772571442648769,
+      "epoch": 1.6,
+      "grad_norm": 3.828125,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 0.8394,
+      "mean_token_accuracy": 0.7701976072043181,
+      "num_tokens": 33309282.0,
+      "step": 2000
+    },
+    {
+      "epoch": 1.6,
+      "eval_coding_entropy": 1.7155875730514527,
+      "eval_coding_loss": 2.268036127090454,
+      "eval_coding_mean_token_accuracy": 0.5065519729852677,
+      "eval_coding_num_tokens": 33309282.0,
+      "eval_coding_runtime": 79.042,
+      "eval_coding_samples_per_second": 6.326,
+      "eval_coding_steps_per_second": 3.163,
+      "step": 2000
+    },
+    {
+      "epoch": 1.6,
+      "eval_physics_entropy": 0.8701518812179565,
+      "eval_physics_loss": 0.9167826771736145,
+      "eval_physics_mean_token_accuracy": 0.7627079751491547,
+      "eval_physics_num_tokens": 33309282.0,
+      "eval_physics_runtime": 51.8141,
+      "eval_physics_samples_per_second": 9.65,
+      "eval_physics_steps_per_second": 4.825,
+      "step": 2000
+    },
+    {
+      "entropy": 0.8041736917570234,
+      "epoch": 1.608,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.7757777777777777e-05,
+      "loss": 0.7911,
+      "mean_token_accuracy": 0.7850766744464636,
+      "num_tokens": 33485091.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.8584910430014133,
+      "epoch": 1.616,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.7735555555555558e-05,
+      "loss": 0.822,
+      "mean_token_accuracy": 0.7741672981530427,
+      "num_tokens": 33647616.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.8141503371298313,
+      "epoch": 1.624,
+      "grad_norm": 4.03125,
+      "learning_rate": 1.7713333333333335e-05,
+      "loss": 0.7946,
+      "mean_token_accuracy": 0.7806556668132544,
+      "num_tokens": 33814904.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.8944115854799748,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.7691111111111113e-05,
+      "loss": 0.8494,
+      "mean_token_accuracy": 0.7674026219174266,
+      "num_tokens": 33977586.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.8180335260927677,
+      "epoch": 1.6400000000000001,
+      "grad_norm": 4.65625,
+      "learning_rate": 1.766888888888889e-05,
+      "loss": 0.7943,
+      "mean_token_accuracy": 0.7807611677795648,
+      "num_tokens": 34143742.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.8157250108197331,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 5.5,
+      "learning_rate": 1.7646666666666668e-05,
+      "loss": 0.786,
+      "mean_token_accuracy": 0.7834763184189797,
+      "num_tokens": 34312436.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.8433098591864109,
+      "epoch": 1.6560000000000001,
+      "grad_norm": 5.25,
+      "learning_rate": 1.7624444444444446e-05,
+      "loss": 0.8172,
+      "mean_token_accuracy": 0.777406994253397,
+      "num_tokens": 34478504.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.8418531702831388,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 6.6875,
+      "learning_rate": 1.7602222222222223e-05,
+      "loss": 0.8182,
+      "mean_token_accuracy": 0.7756667707115412,
+      "num_tokens": 34641549.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.825640257447958,
+      "epoch": 1.6720000000000002,
+      "grad_norm": 4.71875,
+      "learning_rate": 1.758e-05,
+      "loss": 0.7972,
+      "mean_token_accuracy": 0.7794421594589949,
+      "num_tokens": 34808044.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.8450808398425579,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 5.1875,
+      "learning_rate": 1.755777777777778e-05,
+      "loss": 0.8058,
+      "mean_token_accuracy": 0.7755126718431711,
+      "num_tokens": 34977031.0,
+      "step": 2100
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "eval_coding_entropy": 1.6850582294464111,
+      "eval_coding_loss": 2.2593472003936768,
+      "eval_coding_mean_token_accuracy": 0.5131129772663117,
+      "eval_coding_num_tokens": 34977031.0,
+      "eval_coding_runtime": 79.2037,
+      "eval_coding_samples_per_second": 6.313,
+      "eval_coding_steps_per_second": 3.156,
+      "step": 2100
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "eval_physics_entropy": 0.8579872615337372,
+      "eval_physics_loss": 0.9012077450752258,
+      "eval_physics_mean_token_accuracy": 0.7655161898136139,
+      "eval_physics_num_tokens": 34977031.0,
+      "eval_physics_runtime": 52.1742,
+      "eval_physics_samples_per_second": 9.583,
+      "eval_physics_steps_per_second": 4.792,
+      "step": 2100
+    },
+    {
+      "entropy": 0.8239427028223872,
+      "epoch": 1.688,
+      "grad_norm": 4.90625,
+      "learning_rate": 1.7535555555555556e-05,
+      "loss": 0.8111,
+      "mean_token_accuracy": 0.7777137096971274,
+      "num_tokens": 35141418.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.8669725690037012,
+      "epoch": 1.696,
+      "grad_norm": 4.53125,
+      "learning_rate": 1.7513333333333334e-05,
+      "loss": 0.8166,
+      "mean_token_accuracy": 0.774109473079443,
+      "num_tokens": 35300962.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.8681604472920299,
+      "epoch": 1.704,
+      "grad_norm": 5.125,
+      "learning_rate": 1.749111111111111e-05,
+      "loss": 0.8338,
+      "mean_token_accuracy": 0.7683083459734916,
+      "num_tokens": 35465304.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.8354681877419352,
+      "epoch": 1.712,
+      "grad_norm": 4.6875,
+      "learning_rate": 1.746888888888889e-05,
+      "loss": 0.7984,
+      "mean_token_accuracy": 0.7774088971316815,
+      "num_tokens": 35642662.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.7997790666297078,
+      "epoch": 1.72,
+      "grad_norm": 4.375,
+      "learning_rate": 1.7446666666666667e-05,
+      "loss": 0.7903,
+      "mean_token_accuracy": 0.783223495259881,
+      "num_tokens": 35810260.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.8489536764100194,
+      "epoch": 1.728,
+      "grad_norm": 4.84375,
+      "learning_rate": 1.7424444444444444e-05,
+      "loss": 0.8092,
+      "mean_token_accuracy": 0.7768806152045726,
+      "num_tokens": 35980993.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.8214310843497514,
+      "epoch": 1.736,
+      "grad_norm": 5.4375,
+      "learning_rate": 1.7402222222222222e-05,
+      "loss": 0.7963,
+      "mean_token_accuracy": 0.7785018008202315,
+      "num_tokens": 36158360.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.8393697302788496,
+      "epoch": 1.744,
+      "grad_norm": 4.8125,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 0.8124,
+      "mean_token_accuracy": 0.7777101572602987,
+      "num_tokens": 36326797.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.8256299896165729,
+      "epoch": 1.752,
+      "grad_norm": 4.8125,
+      "learning_rate": 1.735777777777778e-05,
+      "loss": 0.8052,
+      "mean_token_accuracy": 0.7793278936296701,
+      "num_tokens": 36495982.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.8003837876021862,
+      "epoch": 1.76,
+      "grad_norm": 4.75,
+      "learning_rate": 1.7335555555555558e-05,
+      "loss": 0.7815,
+      "mean_token_accuracy": 0.7837960578501224,
+      "num_tokens": 36654797.0,
+      "step": 2200
+    },
+    {
+      "epoch": 1.76,
+      "eval_coding_entropy": 1.7353993301391601,
+      "eval_coding_loss": 2.24050235748291,
+      "eval_coding_mean_token_accuracy": 0.5156019266843795,
+      "eval_coding_num_tokens": 36654797.0,
+      "eval_coding_runtime": 79.2517,
+      "eval_coding_samples_per_second": 6.309,
+      "eval_coding_steps_per_second": 3.155,
+      "step": 2200
+    },
+    {
+      "epoch": 1.76,
+      "eval_physics_entropy": 0.8497708789110183,
+      "eval_physics_loss": 0.8915674090385437,
+      "eval_physics_mean_token_accuracy": 0.7682892513275147,
+      "eval_physics_num_tokens": 36654797.0,
+      "eval_physics_runtime": 52.1968,
+      "eval_physics_samples_per_second": 9.579,
+      "eval_physics_steps_per_second": 4.79,
+      "step": 2200
+    },
+    {
+      "entropy": 0.8612198643386364,
+      "epoch": 1.768,
+      "grad_norm": 5.84375,
+      "learning_rate": 1.7313333333333336e-05,
+      "loss": 0.8251,
+      "mean_token_accuracy": 0.7738977476954461,
+      "num_tokens": 36817463.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.8459191296249629,
+      "epoch": 1.776,
+      "grad_norm": 4.71875,
+      "learning_rate": 1.7291111111111113e-05,
+      "loss": 0.8264,
+      "mean_token_accuracy": 0.7742797108367085,
+      "num_tokens": 36989310.0,
+      "step": 2220
+    },
+    {
+      "entropy": 0.8024844925850629,
+      "epoch": 1.784,
+      "grad_norm": 4.78125,
+      "learning_rate": 1.726888888888889e-05,
+      "loss": 0.7582,
+      "mean_token_accuracy": 0.7858391430228948,
+      "num_tokens": 37157452.0,
+      "step": 2230
+    },
+    {
+      "entropy": 0.8630851672962307,
+      "epoch": 1.792,
+      "grad_norm": 5.59375,
+      "learning_rate": 1.724666666666667e-05,
+      "loss": 0.8317,
+      "mean_token_accuracy": 0.7712806731462478,
+      "num_tokens": 37318535.0,
+      "step": 2240
+    },
+    {
+      "entropy": 0.8015124432742595,
+      "epoch": 1.8,
+      "grad_norm": 3.734375,
+      "learning_rate": 1.7224444444444446e-05,
+      "loss": 0.7698,
+      "mean_token_accuracy": 0.7848561979830265,
+      "num_tokens": 37487831.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.8111644266173244,
+      "epoch": 1.808,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.7202222222222224e-05,
+      "loss": 0.7993,
+      "mean_token_accuracy": 0.7826206222176552,
+      "num_tokens": 37658889.0,
+      "step": 2260
+    },
+    {
+      "entropy": 0.8139818359166384,
+      "epoch": 1.8159999999999998,
+      "grad_norm": 4.71875,
+      "learning_rate": 1.718e-05,
+      "loss": 0.7889,
+      "mean_token_accuracy": 0.7834100909531116,
+      "num_tokens": 37829943.0,
+      "step": 2270
+    },
+    {
+      "entropy": 0.8303235996514559,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 4.8125,
+      "learning_rate": 1.715777777777778e-05,
+      "loss": 0.795,
+      "mean_token_accuracy": 0.7808313373476267,
+      "num_tokens": 37995198.0,
+      "step": 2280
+    },
+    {
+      "entropy": 0.8542025130242109,
+      "epoch": 1.8319999999999999,
+      "grad_norm": 5.0625,
+      "learning_rate": 1.7135555555555557e-05,
+      "loss": 0.8313,
+      "mean_token_accuracy": 0.7719877079129219,
+      "num_tokens": 38155297.0,
+      "step": 2290
+    },
+    {
+      "entropy": 0.8089347897097469,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 4.0,
+      "learning_rate": 1.7113333333333334e-05,
+      "loss": 0.7779,
+      "mean_token_accuracy": 0.7852071691304445,
+      "num_tokens": 38325542.0,
+      "step": 2300
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "eval_coding_entropy": 1.7236487927436828,
+      "eval_coding_loss": 2.2375903129577637,
+      "eval_coding_mean_token_accuracy": 0.5160671151876449,
+      "eval_coding_num_tokens": 38325542.0,
+      "eval_coding_runtime": 79.4113,
+      "eval_coding_samples_per_second": 6.296,
+      "eval_coding_steps_per_second": 3.148,
+      "step": 2300
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "eval_physics_entropy": 0.8456154173612594,
+      "eval_physics_loss": 0.8841537833213806,
+      "eval_physics_mean_token_accuracy": 0.7688510994911194,
+      "eval_physics_num_tokens": 38325542.0,
+      "eval_physics_runtime": 52.4055,
+      "eval_physics_samples_per_second": 9.541,
+      "eval_physics_steps_per_second": 4.77,
+      "step": 2300
+    },
+    {
+      "entropy": 0.805744357407093,
+      "epoch": 1.8479999999999999,
+      "grad_norm": 3.84375,
+      "learning_rate": 1.7091111111111112e-05,
+      "loss": 0.7829,
+      "mean_token_accuracy": 0.784181559085846,
+      "num_tokens": 38497246.0,
+      "step": 2310
+    },
+    {
+      "entropy": 0.8343328120186925,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.706888888888889e-05,
+      "loss": 0.7888,
+      "mean_token_accuracy": 0.7806125942617654,
+      "num_tokens": 38668148.0,
+      "step": 2320
+    },
+    {
+      "entropy": 0.8110112180933357,
+      "epoch": 1.8639999999999999,
+      "grad_norm": 4.5,
+      "learning_rate": 1.704666666666667e-05,
+      "loss": 0.7752,
+      "mean_token_accuracy": 0.7813364051282405,
+      "num_tokens": 38840232.0,
+      "step": 2330
+    },
+    {
+      "entropy": 0.8125607699155808,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 3.984375,
+      "learning_rate": 1.7024444444444445e-05,
+      "loss": 0.7895,
+      "mean_token_accuracy": 0.7821429204195738,
+      "num_tokens": 39001652.0,
+      "step": 2340
+    },
+    {
+      "entropy": 0.8465429963544011,
+      "epoch": 1.88,
+      "grad_norm": 5.78125,
+      "learning_rate": 1.7002222222222226e-05,
+      "loss": 0.8126,
+      "mean_token_accuracy": 0.7767612311989069,
+      "num_tokens": 39169312.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.8441216722130775,
+      "epoch": 1.888,
+      "grad_norm": 5.65625,
+      "learning_rate": 1.698e-05,
+      "loss": 0.8063,
+      "mean_token_accuracy": 0.7776072409003973,
+      "num_tokens": 39333474.0,
+      "step": 2360
+    },
+    {
+      "entropy": 0.8453333882614971,
+      "epoch": 1.896,
+      "grad_norm": 3.71875,
+      "learning_rate": 1.695777777777778e-05,
+      "loss": 0.813,
+      "mean_token_accuracy": 0.7757103960961104,
+      "num_tokens": 39499971.0,
+      "step": 2370
+    },
+    {
+      "entropy": 0.8293590556830168,
+      "epoch": 1.904,
+      "grad_norm": 5.40625,
+      "learning_rate": 1.6935555555555555e-05,
+      "loss": 0.8131,
+      "mean_token_accuracy": 0.7780664421617984,
+      "num_tokens": 39663055.0,
+      "step": 2380
+    },
+    {
+      "entropy": 0.8597976418212057,
+      "epoch": 1.912,
+      "grad_norm": 4.71875,
+      "learning_rate": 1.6913333333333336e-05,
+      "loss": 0.8246,
+      "mean_token_accuracy": 0.7739259105175733,
+      "num_tokens": 39823268.0,
+      "step": 2390
+    },
+    {
+      "entropy": 0.8054354287683964,
+      "epoch": 1.92,
+      "grad_norm": 5.5,
+      "learning_rate": 1.689111111111111e-05,
+      "loss": 0.7896,
+      "mean_token_accuracy": 0.7838379312306643,
+      "num_tokens": 39989824.0,
+      "step": 2400
+    },
+    {
+      "epoch": 1.92,
+      "eval_coding_entropy": 1.7244014859199523,
+      "eval_coding_loss": 2.2286953926086426,
+      "eval_coding_mean_token_accuracy": 0.5165949345827102,
+      "eval_coding_num_tokens": 39989824.0,
+      "eval_coding_runtime": 79.4303,
+      "eval_coding_samples_per_second": 6.295,
+      "eval_coding_steps_per_second": 3.147,
+      "step": 2400
+    },
+    {
+      "epoch": 1.92,
+      "eval_physics_entropy": 0.8283440818786622,
+      "eval_physics_loss": 0.8724973201751709,
+      "eval_physics_mean_token_accuracy": 0.7722779376506805,
+      "eval_physics_num_tokens": 39989824.0,
+      "eval_physics_runtime": 52.2349,
+      "eval_physics_samples_per_second": 9.572,
+      "eval_physics_steps_per_second": 4.786,
+      "step": 2400
+    },
+    {
+      "entropy": 0.8490746941417455,
+      "epoch": 1.928,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.686888888888889e-05,
+      "loss": 0.8138,
+      "mean_token_accuracy": 0.7763742931187153,
+      "num_tokens": 40159086.0,
+      "step": 2410
+    },
+    {
+      "entropy": 0.8009230157360434,
+      "epoch": 1.936,
+      "grad_norm": 4.46875,
+      "learning_rate": 1.684666666666667e-05,
+      "loss": 0.782,
+      "mean_token_accuracy": 0.7832826662808656,
+      "num_tokens": 40325070.0,
+      "step": 2420
+    },
+    {
+      "entropy": 0.8089035371318459,
+      "epoch": 1.944,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.6824444444444447e-05,
+      "loss": 0.7736,
+      "mean_token_accuracy": 0.7844303779304027,
+      "num_tokens": 40489400.0,
+      "step": 2430
+    },
+    {
+      "entropy": 0.7484890200197697,
+      "epoch": 1.952,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.6802222222222224e-05,
+      "loss": 0.736,
+      "mean_token_accuracy": 0.7953914526849986,
+      "num_tokens": 40659614.0,
+      "step": 2440
+    },
+    {
+      "entropy": 0.8137759083881975,
+      "epoch": 1.96,
+      "grad_norm": 4.875,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 0.7801,
+      "mean_token_accuracy": 0.7849993612617254,
+      "num_tokens": 40824844.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.8361523715779186,
+      "epoch": 1.968,
+      "grad_norm": 5.09375,
+      "learning_rate": 1.675777777777778e-05,
+      "loss": 0.799,
+      "mean_token_accuracy": 0.7781310323625803,
+      "num_tokens": 40991508.0,
+      "step": 2460
+    },
+    {
+      "entropy": 0.8114897888153791,
+      "epoch": 1.976,
+      "grad_norm": 4.0,
+      "learning_rate": 1.6735555555555557e-05,
+      "loss": 0.7787,
+      "mean_token_accuracy": 0.7807551980018616,
+      "num_tokens": 41160674.0,
+      "step": 2470
+    },
+    {
+      "entropy": 0.8438869906589389,
+      "epoch": 1.984,
+      "grad_norm": 4.6875,
+      "learning_rate": 1.6713333333333335e-05,
+      "loss": 0.8216,
+      "mean_token_accuracy": 0.7774552527815104,
+      "num_tokens": 41328779.0,
+      "step": 2480
+    },
+    {
+      "entropy": 0.8189408538863063,
+      "epoch": 1.992,
+      "grad_norm": 5.65625,
+      "learning_rate": 1.6691111111111112e-05,
+      "loss": 0.7889,
+      "mean_token_accuracy": 0.782219960168004,
+      "num_tokens": 41495501.0,
+      "step": 2490
+    },
+    {
+      "entropy": 0.8064166329801082,
+      "epoch": 2.0,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.666888888888889e-05,
+      "loss": 0.7833,
+      "mean_token_accuracy": 0.7833917330950498,
+      "num_tokens": 41664296.0,
+      "step": 2500
+    },
+    {
+      "epoch": 2.0,
+      "eval_coding_entropy": 1.717286424636841,
+      "eval_coding_loss": 2.2317516803741455,
+      "eval_coding_mean_token_accuracy": 0.5175079522132874,
+      "eval_coding_num_tokens": 41664296.0,
+      "eval_coding_runtime": 79.4337,
+      "eval_coding_samples_per_second": 6.295,
+      "eval_coding_steps_per_second": 3.147,
+      "step": 2500
+    },
+    {
+      "epoch": 2.0,
+      "eval_physics_entropy": 0.8259405951499939,
+      "eval_physics_loss": 0.8628295660018921,
+      "eval_physics_mean_token_accuracy": 0.7733515868186951,
+      "eval_physics_num_tokens": 41664296.0,
+      "eval_physics_runtime": 51.845,
+      "eval_physics_samples_per_second": 9.644,
+      "eval_physics_steps_per_second": 4.822,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.612734324683743e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d4a53d0a1f980794ff009d9265f9a77226e0f35f375611e5bac896105fe9f54
+size 6353