Upload folder using huggingface_hub

.gitattributes

@@ -37,3 +37,4 @@ checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-2500/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-2500/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-2500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-2500/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-2500/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-2500/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74d2e8fa441384a768c1c9abb09d8adc60167e4f6505d016636a6dbbb09e6e90
+size 4962221464

checkpoint-2500/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9bde39c5f2f2eb4df138bca3bf5857ca24ebab226fd2f6b82e589e25623fa57
+size 3670322200

checkpoint-2500/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-2500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:108b20514e4936234a5a2d15b24e2721c9829cc719da9127c8fdca532c4d5b69
+size 7927918951

checkpoint-2500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bbd07b04584d51483a537ebe86fdb5e065d175fef715f69db1afc368ee4945d
+size 14645

checkpoint-2500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd200143f139327b4d41bcb99ffeaa13b2acc11ded5b3dbab181a6ff2ce855d7
+size 1465

checkpoint-2500/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-2500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-2500/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-2500/trainer_state.json

@@ -0,0 +1,3084 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 100,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.3688963137567043,
+      "epoch": 0.016,
+      "grad_norm": 26.75,
+      "learning_rate": 1.8e-07,
+      "loss": 2.1543,
+      "mean_token_accuracy": 0.6007782766595483,
+      "num_tokens": 195510.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.3619943611323833,
+      "epoch": 0.032,
+      "grad_norm": 22.625,
+      "learning_rate": 3.8e-07,
+      "loss": 2.1586,
+      "mean_token_accuracy": 0.6013051267713309,
+      "num_tokens": 391008.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.4113613955676556,
+      "epoch": 0.048,
+      "grad_norm": 20.875,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 2.132,
+      "mean_token_accuracy": 0.6053086685016751,
+      "num_tokens": 590136.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.4471838753670454,
+      "epoch": 0.064,
+      "grad_norm": 17.125,
+      "learning_rate": 7.8e-07,
+      "loss": 2.0923,
+      "mean_token_accuracy": 0.6051462920382619,
+      "num_tokens": 791432.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.5650674737989902,
+      "epoch": 0.08,
+      "grad_norm": 13.625,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 2.0495,
+      "mean_token_accuracy": 0.6057845208793878,
+      "num_tokens": 990342.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.7182099640369415,
+      "epoch": 0.096,
+      "grad_norm": 8.8125,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 2.0641,
+      "mean_token_accuracy": 0.604715196043253,
+      "num_tokens": 1182339.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.8127950172871352,
+      "epoch": 0.112,
+      "grad_norm": 5.875,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 1.959,
+      "mean_token_accuracy": 0.6124658133834601,
+      "num_tokens": 1386193.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.8337691940367222,
+      "epoch": 0.128,
+      "grad_norm": 5.8125,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 1.9629,
+      "mean_token_accuracy": 0.610929672792554,
+      "num_tokens": 1583261.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.79930883012712,
+      "epoch": 0.144,
+      "grad_norm": 5.3125,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 1.9463,
+      "mean_token_accuracy": 0.6148752361536026,
+      "num_tokens": 1774826.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.8165954321622848,
+      "epoch": 0.16,
+      "grad_norm": 5.90625,
+      "learning_rate": 1.98e-06,
+      "loss": 1.9617,
+      "mean_token_accuracy": 0.6118428658694028,
+      "num_tokens": 1971310.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 1.7495100326538087,
+      "eval_biology_loss": 1.9257053136825562,
+      "eval_biology_mean_token_accuracy": 0.6249015061855316,
+      "eval_biology_num_tokens": 1971310.0,
+      "eval_biology_runtime": 39.1194,
+      "eval_biology_samples_per_second": 12.781,
+      "eval_biology_steps_per_second": 3.195,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_chemistry_entropy": 0.7336231064796448,
+      "eval_chemistry_loss": 0.8070052266120911,
+      "eval_chemistry_mean_token_accuracy": 0.8109221415519714,
+      "eval_chemistry_num_tokens": 1971310.0,
+      "eval_chemistry_runtime": 48.7021,
+      "eval_chemistry_samples_per_second": 10.266,
+      "eval_chemistry_steps_per_second": 2.567,
+      "step": 100
+    },
+    {
+      "entropy": 1.7839918725192547,
+      "epoch": 0.176,
+      "grad_norm": 4.75,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 1.911,
+      "mean_token_accuracy": 0.6167426453903317,
+      "num_tokens": 2169453.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.798803521692753,
+      "epoch": 0.192,
+      "grad_norm": 5.59375,
+      "learning_rate": 2.38e-06,
+      "loss": 1.9179,
+      "mean_token_accuracy": 0.6152951141819358,
+      "num_tokens": 2366844.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.7993743151426316,
+      "epoch": 0.208,
+      "grad_norm": 5.21875,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 1.9124,
+      "mean_token_accuracy": 0.6123070493340492,
+      "num_tokens": 2559708.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.7579403154551982,
+      "epoch": 0.224,
+      "grad_norm": 5.5,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 1.8823,
+      "mean_token_accuracy": 0.6201574027538299,
+      "num_tokens": 2756533.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.751310784369707,
+      "epoch": 0.24,
+      "grad_norm": 5.0,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 1.8497,
+      "mean_token_accuracy": 0.6232937585562468,
+      "num_tokens": 2948600.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.7673518907278776,
+      "epoch": 0.256,
+      "grad_norm": 5.21875,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 1.8636,
+      "mean_token_accuracy": 0.6187300987541675,
+      "num_tokens": 3141473.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.7522739164531231,
+      "epoch": 0.272,
+      "grad_norm": 5.0,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 1.8404,
+      "mean_token_accuracy": 0.6195556577295065,
+      "num_tokens": 3337285.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.735862059891224,
+      "epoch": 0.288,
+      "grad_norm": 4.9375,
+      "learning_rate": 3.58e-06,
+      "loss": 1.8084,
+      "mean_token_accuracy": 0.6230981133878231,
+      "num_tokens": 3541221.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.7118580989539622,
+      "epoch": 0.304,
+      "grad_norm": 4.65625,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 1.7784,
+      "mean_token_accuracy": 0.626662303134799,
+      "num_tokens": 3734952.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.7322832621634006,
+      "epoch": 0.32,
+      "grad_norm": 4.78125,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 1.7879,
+      "mean_token_accuracy": 0.623782691732049,
+      "num_tokens": 3922209.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 1.703305299282074,
+      "eval_biology_loss": 1.7626646757125854,
+      "eval_biology_mean_token_accuracy": 0.635826956987381,
+      "eval_biology_num_tokens": 3922209.0,
+      "eval_biology_runtime": 39.0903,
+      "eval_biology_samples_per_second": 12.791,
+      "eval_biology_steps_per_second": 3.198,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_chemistry_entropy": 0.7484376595020295,
+      "eval_chemistry_loss": 0.7622011303901672,
+      "eval_chemistry_mean_token_accuracy": 0.8104680519104004,
+      "eval_chemistry_num_tokens": 3922209.0,
+      "eval_chemistry_runtime": 48.6797,
+      "eval_chemistry_samples_per_second": 10.271,
+      "eval_chemistry_steps_per_second": 2.568,
+      "step": 200
+    },
+    {
+      "entropy": 1.6908923149108888,
+      "epoch": 0.336,
+      "grad_norm": 4.6875,
+      "learning_rate": 4.18e-06,
+      "loss": 1.7343,
+      "mean_token_accuracy": 0.6304708372801542,
+      "num_tokens": 4115941.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.7183935269713402,
+      "epoch": 0.352,
+      "grad_norm": 5.0,
+      "learning_rate": 4.38e-06,
+      "loss": 1.7323,
+      "mean_token_accuracy": 0.6271383590996266,
+      "num_tokens": 4308755.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.7019562765955925,
+      "epoch": 0.368,
+      "grad_norm": 4.15625,
+      "learning_rate": 4.58e-06,
+      "loss": 1.7129,
+      "mean_token_accuracy": 0.6312347020953893,
+      "num_tokens": 4505910.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.699073526263237,
+      "epoch": 0.384,
+      "grad_norm": 4.78125,
+      "learning_rate": 4.78e-06,
+      "loss": 1.6857,
+      "mean_token_accuracy": 0.6311217818409205,
+      "num_tokens": 4696101.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.7013882726430893,
+      "epoch": 0.4,
+      "grad_norm": 4.375,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 1.677,
+      "mean_token_accuracy": 0.6308177687227726,
+      "num_tokens": 4889445.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.7109287835657596,
+      "epoch": 0.416,
+      "grad_norm": 4.28125,
+      "learning_rate": 5.18e-06,
+      "loss": 1.6829,
+      "mean_token_accuracy": 0.6278879173099995,
+      "num_tokens": 5087624.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.7145581994205714,
+      "epoch": 0.432,
+      "grad_norm": 5.0625,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 1.681,
+      "mean_token_accuracy": 0.6289313135668635,
+      "num_tokens": 5273794.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.7117976807057858,
+      "epoch": 0.448,
+      "grad_norm": 5.15625,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 1.6672,
+      "mean_token_accuracy": 0.6299384724348783,
+      "num_tokens": 5463288.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.642435162886977,
+      "epoch": 0.464,
+      "grad_norm": 4.4375,
+      "learning_rate": 5.78e-06,
+      "loss": 1.6048,
+      "mean_token_accuracy": 0.641257194057107,
+      "num_tokens": 5656827.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.6684199325740336,
+      "epoch": 0.48,
+      "grad_norm": 4.3125,
+      "learning_rate": 5.98e-06,
+      "loss": 1.6272,
+      "mean_token_accuracy": 0.6384812213480473,
+      "num_tokens": 5853311.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 1.6503952503204347,
+      "eval_biology_loss": 1.6127734184265137,
+      "eval_biology_mean_token_accuracy": 0.6489202432632446,
+      "eval_biology_num_tokens": 5853311.0,
+      "eval_biology_runtime": 39.0858,
+      "eval_biology_samples_per_second": 12.792,
+      "eval_biology_steps_per_second": 3.198,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_chemistry_entropy": 0.7682121543884277,
+      "eval_chemistry_loss": 0.7135673761367798,
+      "eval_chemistry_mean_token_accuracy": 0.8120616941452027,
+      "eval_chemistry_num_tokens": 5853311.0,
+      "eval_chemistry_runtime": 48.6951,
+      "eval_chemistry_samples_per_second": 10.268,
+      "eval_chemistry_steps_per_second": 2.567,
+      "step": 300
+    },
+    {
+      "entropy": 1.5807546474039555,
+      "epoch": 0.496,
+      "grad_norm": 4.6875,
+      "learning_rate": 6.18e-06,
+      "loss": 1.5368,
+      "mean_token_accuracy": 0.6513836592435837,
+      "num_tokens": 6049788.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.678506825864315,
+      "epoch": 0.512,
+      "grad_norm": 4.65625,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 1.6246,
+      "mean_token_accuracy": 0.6385043509304523,
+      "num_tokens": 6243825.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.652913999557495,
+      "epoch": 0.528,
+      "grad_norm": 4.4375,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 1.6009,
+      "mean_token_accuracy": 0.64362001568079,
+      "num_tokens": 6434188.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.6507954083383083,
+      "epoch": 0.544,
+      "grad_norm": 4.25,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 1.6102,
+      "mean_token_accuracy": 0.6433101873844862,
+      "num_tokens": 6629832.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.6201883230358363,
+      "epoch": 0.56,
+      "grad_norm": 4.28125,
+      "learning_rate": 6.98e-06,
+      "loss": 1.5677,
+      "mean_token_accuracy": 0.64635454043746,
+      "num_tokens": 6824668.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.589719269424677,
+      "epoch": 0.576,
+      "grad_norm": 4.375,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 1.5451,
+      "mean_token_accuracy": 0.6491251610219478,
+      "num_tokens": 7025923.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.6323727253824472,
+      "epoch": 0.592,
+      "grad_norm": 4.0625,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 1.5618,
+      "mean_token_accuracy": 0.645591801777482,
+      "num_tokens": 7218224.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.6019253939390183,
+      "epoch": 0.608,
+      "grad_norm": 4.65625,
+      "learning_rate": 7.58e-06,
+      "loss": 1.562,
+      "mean_token_accuracy": 0.6492579754441976,
+      "num_tokens": 7421216.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.6110554873943328,
+      "epoch": 0.624,
+      "grad_norm": 4.375,
+      "learning_rate": 7.78e-06,
+      "loss": 1.5511,
+      "mean_token_accuracy": 0.6453482262790203,
+      "num_tokens": 7617258.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.6183445014059543,
+      "epoch": 0.64,
+      "grad_norm": 4.40625,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 1.5462,
+      "mean_token_accuracy": 0.6479769114404916,
+      "num_tokens": 7806173.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 1.5254363222122191,
+      "eval_biology_loss": 1.5303529500961304,
+      "eval_biology_mean_token_accuracy": 0.6589018092155456,
+      "eval_biology_num_tokens": 7806173.0,
+      "eval_biology_runtime": 39.136,
+      "eval_biology_samples_per_second": 12.776,
+      "eval_biology_steps_per_second": 3.194,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_chemistry_entropy": 0.7431424338817596,
+      "eval_chemistry_loss": 0.7057921290397644,
+      "eval_chemistry_mean_token_accuracy": 0.8118511362075805,
+      "eval_chemistry_num_tokens": 7806173.0,
+      "eval_chemistry_runtime": 48.743,
+      "eval_chemistry_samples_per_second": 10.258,
+      "eval_chemistry_steps_per_second": 2.564,
+      "step": 400
+    },
+    {
+      "entropy": 1.5993809100240468,
+      "epoch": 0.656,
+      "grad_norm": 4.6875,
+      "learning_rate": 8.18e-06,
+      "loss": 1.5514,
+      "mean_token_accuracy": 0.6487635668367148,
+      "num_tokens": 8000514.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.6190266445279122,
+      "epoch": 0.672,
+      "grad_norm": 4.40625,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 1.5587,
+      "mean_token_accuracy": 0.6449675388634205,
+      "num_tokens": 8187884.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.5749193426221608,
+      "epoch": 0.688,
+      "grad_norm": 4.125,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 1.523,
+      "mean_token_accuracy": 0.651202504709363,
+      "num_tokens": 8390643.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.5851164262741804,
+      "epoch": 0.704,
+      "grad_norm": 4.375,
+      "learning_rate": 8.78e-06,
+      "loss": 1.5246,
+      "mean_token_accuracy": 0.6505646403878927,
+      "num_tokens": 8583266.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.500511408224702,
+      "epoch": 0.72,
+      "grad_norm": 3.875,
+      "learning_rate": 8.98e-06,
+      "loss": 1.4375,
+      "mean_token_accuracy": 0.664596076309681,
+      "num_tokens": 8786233.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.5476290859282016,
+      "epoch": 0.736,
+      "grad_norm": 4.5625,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 1.476,
+      "mean_token_accuracy": 0.6562888745218516,
+      "num_tokens": 8982898.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.5444358900189399,
+      "epoch": 0.752,
+      "grad_norm": 4.625,
+      "learning_rate": 9.38e-06,
+      "loss": 1.4792,
+      "mean_token_accuracy": 0.6567551180720329,
+      "num_tokens": 9174361.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.494469254463911,
+      "epoch": 0.768,
+      "grad_norm": 4.125,
+      "learning_rate": 9.58e-06,
+      "loss": 1.4385,
+      "mean_token_accuracy": 0.6629225756973028,
+      "num_tokens": 9373430.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.5377752494066954,
+      "epoch": 0.784,
+      "grad_norm": 4.21875,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 1.4748,
+      "mean_token_accuracy": 0.6564478818327188,
+      "num_tokens": 9570409.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.546391127258539,
+      "epoch": 0.8,
+      "grad_norm": 4.46875,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 1.4842,
+      "mean_token_accuracy": 0.6546981681138277,
+      "num_tokens": 9766360.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 1.4740597500801087,
+      "eval_biology_loss": 1.4621686935424805,
+      "eval_biology_mean_token_accuracy": 0.6668880858421325,
+      "eval_biology_num_tokens": 9766360.0,
+      "eval_biology_runtime": 39.1212,
+      "eval_biology_samples_per_second": 12.781,
+      "eval_biology_steps_per_second": 3.195,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_chemistry_entropy": 0.739832299232483,
+      "eval_chemistry_loss": 0.7017239928245544,
+      "eval_chemistry_mean_token_accuracy": 0.8113184819221496,
+      "eval_chemistry_num_tokens": 9766360.0,
+      "eval_chemistry_runtime": 48.5413,
+      "eval_chemistry_samples_per_second": 10.301,
+      "eval_chemistry_steps_per_second": 2.575,
+      "step": 500
+    },
+    {
+      "entropy": 1.4985189002007246,
+      "epoch": 0.816,
+      "grad_norm": 4.625,
+      "learning_rate": 1.018e-05,
+      "loss": 1.4466,
+      "mean_token_accuracy": 0.6619349274784326,
+      "num_tokens": 9963834.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.48775667399168,
+      "epoch": 0.832,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.038e-05,
+      "loss": 1.4375,
+      "mean_token_accuracy": 0.6627471141517163,
+      "num_tokens": 10160950.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.479408584907651,
+      "epoch": 0.848,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 1.4177,
+      "mean_token_accuracy": 0.6654143054038286,
+      "num_tokens": 10363131.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.5001295048743486,
+      "epoch": 0.864,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 1.4434,
+      "mean_token_accuracy": 0.6616969238966703,
+      "num_tokens": 10558003.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.4739857867360115,
+      "epoch": 0.88,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 1.4201,
+      "mean_token_accuracy": 0.665744898840785,
+      "num_tokens": 10754470.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.5231038693338632,
+      "epoch": 0.896,
+      "grad_norm": 4.5,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 1.4565,
+      "mean_token_accuracy": 0.6590576630085707,
+      "num_tokens": 10948943.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.4710557978600263,
+      "epoch": 0.912,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.138e-05,
+      "loss": 1.4183,
+      "mean_token_accuracy": 0.6641684602946043,
+      "num_tokens": 11142613.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.4895741816610097,
+      "epoch": 0.928,
+      "grad_norm": 3.984375,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 1.4255,
+      "mean_token_accuracy": 0.6618844125419855,
+      "num_tokens": 11336779.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.4350463204085826,
+      "epoch": 0.944,
+      "grad_norm": 3.96875,
+      "learning_rate": 1.178e-05,
+      "loss": 1.3876,
+      "mean_token_accuracy": 0.6704638510942459,
+      "num_tokens": 11536381.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.4527774393558501,
+      "epoch": 0.96,
+      "grad_norm": 4.65625,
+      "learning_rate": 1.198e-05,
+      "loss": 1.401,
+      "mean_token_accuracy": 0.6658082317560912,
+      "num_tokens": 11735527.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_biology_entropy": 1.4157320256233215,
+      "eval_biology_loss": 1.4121752977371216,
+      "eval_biology_mean_token_accuracy": 0.6741212258338928,
+      "eval_biology_num_tokens": 11735527.0,
+      "eval_biology_runtime": 39.1158,
+      "eval_biology_samples_per_second": 12.783,
+      "eval_biology_steps_per_second": 3.196,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_chemistry_entropy": 0.7435694184303284,
+      "eval_chemistry_loss": 0.7019251585006714,
+      "eval_chemistry_mean_token_accuracy": 0.8104675717353821,
+      "eval_chemistry_num_tokens": 11735527.0,
+      "eval_chemistry_runtime": 48.7451,
+      "eval_chemistry_samples_per_second": 10.257,
+      "eval_chemistry_steps_per_second": 2.564,
+      "step": 600
+    },
+    {
+      "entropy": 1.4274786025285722,
+      "epoch": 0.976,
+      "grad_norm": 4.25,
+      "learning_rate": 1.218e-05,
+      "loss": 1.3729,
+      "mean_token_accuracy": 0.6719065200537443,
+      "num_tokens": 11930935.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.4573175739496946,
+      "epoch": 0.992,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 1.4041,
+      "mean_token_accuracy": 0.6670554719865323,
+      "num_tokens": 12129585.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.4293555557727813,
+      "epoch": 1.008,
+      "grad_norm": 3.796875,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 1.3792,
+      "mean_token_accuracy": 0.6710654832422733,
+      "num_tokens": 12325847.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.3706564407795667,
+      "epoch": 1.024,
+      "grad_norm": 3.96875,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 1.3138,
+      "mean_token_accuracy": 0.6818066164851189,
+      "num_tokens": 12530820.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.4076907094568014,
+      "epoch": 1.04,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 1.3577,
+      "mean_token_accuracy": 0.6723817244172097,
+      "num_tokens": 12725384.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.3665749713778497,
+      "epoch": 1.056,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 1.3016,
+      "mean_token_accuracy": 0.6819081131368876,
+      "num_tokens": 12924592.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.3977813702076674,
+      "epoch": 1.072,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 1.3368,
+      "mean_token_accuracy": 0.6750961408019066,
+      "num_tokens": 13112848.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.3841964561492204,
+      "epoch": 1.088,
+      "grad_norm": 3.859375,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 1.3301,
+      "mean_token_accuracy": 0.6785592496395111,
+      "num_tokens": 13305725.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.3711885944008828,
+      "epoch": 1.104,
+      "grad_norm": 4.0,
+      "learning_rate": 1.378e-05,
+      "loss": 1.3226,
+      "mean_token_accuracy": 0.6788452990353108,
+      "num_tokens": 13498685.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.3965423095971345,
+      "epoch": 1.12,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.398e-05,
+      "loss": 1.3255,
+      "mean_token_accuracy": 0.6777807965874671,
+      "num_tokens": 13682146.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_biology_entropy": 1.3078518261909484,
+      "eval_biology_loss": 1.3719801902770996,
+      "eval_biology_mean_token_accuracy": 0.6799422569274902,
+      "eval_biology_num_tokens": 13682146.0,
+      "eval_biology_runtime": 39.0883,
+      "eval_biology_samples_per_second": 12.792,
+      "eval_biology_steps_per_second": 3.198,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_chemistry_entropy": 0.6988301925659179,
+      "eval_chemistry_loss": 0.7011456489562988,
+      "eval_chemistry_mean_token_accuracy": 0.8096043920516968,
+      "eval_chemistry_num_tokens": 13682146.0,
+      "eval_chemistry_runtime": 48.7053,
+      "eval_chemistry_samples_per_second": 10.266,
+      "eval_chemistry_steps_per_second": 2.566,
+      "step": 700
+    },
+    {
+      "entropy": 1.3658908106386662,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 4.78125,
+      "learning_rate": 1.418e-05,
+      "loss": 1.322,
+      "mean_token_accuracy": 0.6789216380566359,
+      "num_tokens": 13876656.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.3465955082327128,
+      "epoch": 1.152,
+      "grad_norm": 3.890625,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 1.2917,
+      "mean_token_accuracy": 0.682017782703042,
+      "num_tokens": 14086104.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.3684371966868638,
+      "epoch": 1.168,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 1.314,
+      "mean_token_accuracy": 0.6794850911945105,
+      "num_tokens": 14274520.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.3601514294743537,
+      "epoch": 1.184,
+      "grad_norm": 3.65625,
+      "learning_rate": 1.478e-05,
+      "loss": 1.3094,
+      "mean_token_accuracy": 0.6805991049855947,
+      "num_tokens": 14473623.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.3518083952367306,
+      "epoch": 1.2,
+      "grad_norm": 3.828125,
+      "learning_rate": 1.498e-05,
+      "loss": 1.2965,
+      "mean_token_accuracy": 0.6813483159989119,
+      "num_tokens": 14661238.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.3173013672232627,
+      "epoch": 1.216,
+      "grad_norm": 3.796875,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 1.2626,
+      "mean_token_accuracy": 0.6857349291443825,
+      "num_tokens": 14865915.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.3409026943147182,
+      "epoch": 1.232,
+      "grad_norm": 3.625,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 1.2865,
+      "mean_token_accuracy": 0.6858917985111475,
+      "num_tokens": 15055682.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.344399669766426,
+      "epoch": 1.248,
+      "grad_norm": 3.875,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 1.2956,
+      "mean_token_accuracy": 0.6822925679385662,
+      "num_tokens": 15249576.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.3598602082580329,
+      "epoch": 1.264,
+      "grad_norm": 3.359375,
+      "learning_rate": 1.578e-05,
+      "loss": 1.2951,
+      "mean_token_accuracy": 0.6818954084068537,
+      "num_tokens": 15446149.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.36542206145823,
+      "epoch": 1.28,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 1.32,
+      "mean_token_accuracy": 0.6771101258695126,
+      "num_tokens": 15639317.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_biology_entropy": 1.3049154906272888,
+      "eval_biology_loss": 1.33707857131958,
+      "eval_biology_mean_token_accuracy": 0.683680823802948,
+      "eval_biology_num_tokens": 15639317.0,
+      "eval_biology_runtime": 39.1846,
+      "eval_biology_samples_per_second": 12.76,
+      "eval_biology_steps_per_second": 3.19,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_chemistry_entropy": 0.715079505443573,
+      "eval_chemistry_loss": 0.7021569609642029,
+      "eval_chemistry_mean_token_accuracy": 0.8091985578536988,
+      "eval_chemistry_num_tokens": 15639317.0,
+      "eval_chemistry_runtime": 48.799,
+      "eval_chemistry_samples_per_second": 10.246,
+      "eval_chemistry_steps_per_second": 2.562,
+      "step": 800
+    },
+    {
+      "entropy": 1.3627730403095484,
+      "epoch": 1.296,
+      "grad_norm": 3.375,
+      "learning_rate": 1.618e-05,
+      "loss": 1.3074,
+      "mean_token_accuracy": 0.6805873740464449,
+      "num_tokens": 15835805.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.3559819746762514,
+      "epoch": 1.312,
+      "grad_norm": 3.546875,
+      "learning_rate": 1.638e-05,
+      "loss": 1.2937,
+      "mean_token_accuracy": 0.6807271435856819,
+      "num_tokens": 16028375.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.356961441040039,
+      "epoch": 1.328,
+      "grad_norm": 3.546875,
+      "learning_rate": 1.658e-05,
+      "loss": 1.2983,
+      "mean_token_accuracy": 0.6812620472162962,
+      "num_tokens": 16230555.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.3171780712902545,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 3.390625,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 1.2739,
+      "mean_token_accuracy": 0.6877064649015665,
+      "num_tokens": 16436297.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.3393035437911749,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 3.90625,
+      "learning_rate": 1.698e-05,
+      "loss": 1.2744,
+      "mean_token_accuracy": 0.6847097586840392,
+      "num_tokens": 16630563.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.3660046834498645,
+      "epoch": 1.376,
+      "grad_norm": 3.859375,
+      "learning_rate": 1.718e-05,
+      "loss": 1.3048,
+      "mean_token_accuracy": 0.679730985313654,
+      "num_tokens": 16822602.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.3303084772080183,
+      "epoch": 1.392,
+      "grad_norm": 3.390625,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 1.2773,
+      "mean_token_accuracy": 0.6869184520095587,
+      "num_tokens": 17015750.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.3214296508580445,
+      "epoch": 1.408,
+      "grad_norm": 3.015625,
+      "learning_rate": 1.758e-05,
+      "loss": 1.2688,
+      "mean_token_accuracy": 0.6854220055043697,
+      "num_tokens": 17207248.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.3336139082908631,
+      "epoch": 1.424,
+      "grad_norm": 3.546875,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 1.29,
+      "mean_token_accuracy": 0.6841222684830427,
+      "num_tokens": 17403869.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.328351939469576,
+      "epoch": 1.44,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.798e-05,
+      "loss": 1.2696,
+      "mean_token_accuracy": 0.683881213143468,
+      "num_tokens": 17597428.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_biology_entropy": 1.293550329685211,
+      "eval_biology_loss": 1.3036084175109863,
+      "eval_biology_mean_token_accuracy": 0.6888437666893006,
+      "eval_biology_num_tokens": 17597428.0,
+      "eval_biology_runtime": 39.2009,
+      "eval_biology_samples_per_second": 12.755,
+      "eval_biology_steps_per_second": 3.189,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_chemistry_entropy": 0.711592486858368,
+      "eval_chemistry_loss": 0.7021830081939697,
+      "eval_chemistry_mean_token_accuracy": 0.8084347972869873,
+      "eval_chemistry_num_tokens": 17597428.0,
+      "eval_chemistry_runtime": 48.8241,
+      "eval_chemistry_samples_per_second": 10.241,
+      "eval_chemistry_steps_per_second": 2.56,
+      "step": 900
+    },
+    {
+      "entropy": 1.3218930535018445,
+      "epoch": 1.456,
+      "grad_norm": 3.515625,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 1.2763,
+      "mean_token_accuracy": 0.6863383412361145,
+      "num_tokens": 17798083.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.3118344832211732,
+      "epoch": 1.472,
+      "grad_norm": 3.265625,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 1.254,
+      "mean_token_accuracy": 0.6886441022157669,
+      "num_tokens": 17993802.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.309800398349762,
+      "epoch": 1.488,
+      "grad_norm": 3.453125,
+      "learning_rate": 1.858e-05,
+      "loss": 1.2575,
+      "mean_token_accuracy": 0.689780355989933,
+      "num_tokens": 18185570.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.3265162590891122,
+      "epoch": 1.504,
+      "grad_norm": 3.140625,
+      "learning_rate": 1.878e-05,
+      "loss": 1.2782,
+      "mean_token_accuracy": 0.6839083366096019,
+      "num_tokens": 18377903.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.323006783425808,
+      "epoch": 1.52,
+      "grad_norm": 3.296875,
+      "learning_rate": 1.898e-05,
+      "loss": 1.2749,
+      "mean_token_accuracy": 0.6838061761111021,
+      "num_tokens": 18579284.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.3307026766240597,
+      "epoch": 1.536,
+      "grad_norm": 3.515625,
+      "learning_rate": 1.918e-05,
+      "loss": 1.2627,
+      "mean_token_accuracy": 0.684997670724988,
+      "num_tokens": 18765324.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2984559267759324,
+      "epoch": 1.552,
+      "grad_norm": 3.515625,
+      "learning_rate": 1.938e-05,
+      "loss": 1.2484,
+      "mean_token_accuracy": 0.6866503942757844,
+      "num_tokens": 18966382.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.2994249012321233,
+      "epoch": 1.568,
+      "grad_norm": 3.4375,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 1.253,
+      "mean_token_accuracy": 0.6894579119980335,
+      "num_tokens": 19160509.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.2866214491426944,
+      "epoch": 1.584,
+      "grad_norm": 3.171875,
+      "learning_rate": 1.978e-05,
+      "loss": 1.2375,
+      "mean_token_accuracy": 0.6898514706641435,
+      "num_tokens": 19354662.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.3383112341165542,
+      "epoch": 1.6,
+      "grad_norm": 3.28125,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 1.2822,
+      "mean_token_accuracy": 0.6823111370205879,
+      "num_tokens": 19543088.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_biology_entropy": 1.2766407289505004,
+      "eval_biology_loss": 1.278047800064087,
+      "eval_biology_mean_token_accuracy": 0.6927900876998901,
+      "eval_biology_num_tokens": 19543088.0,
+      "eval_biology_runtime": 39.2,
+      "eval_biology_samples_per_second": 12.755,
+      "eval_biology_steps_per_second": 3.189,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_chemistry_entropy": 0.726632200717926,
+      "eval_chemistry_loss": 0.7049177289009094,
+      "eval_chemistry_mean_token_accuracy": 0.8069638891220092,
+      "eval_chemistry_num_tokens": 19543088.0,
+      "eval_chemistry_runtime": 48.5266,
+      "eval_chemistry_samples_per_second": 10.304,
+      "eval_chemistry_steps_per_second": 2.576,
+      "step": 1000
+    },
+    {
+      "entropy": 1.2842272642999888,
+      "epoch": 1.616,
+      "grad_norm": 3.078125,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 1.2365,
+      "mean_token_accuracy": 0.691414737701416,
+      "num_tokens": 19743506.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.3330749351531268,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 3.59375,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 1.2744,
+      "mean_token_accuracy": 0.6847961049526929,
+      "num_tokens": 19937567.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.3074569128453732,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 2.796875,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 1.248,
+      "mean_token_accuracy": 0.6873298797756433,
+      "num_tokens": 20129743.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.2786120809614658,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 1.227,
+      "mean_token_accuracy": 0.6933693554252386,
+      "num_tokens": 20331548.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.2929955948144198,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 3.359375,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 1.2438,
+      "mean_token_accuracy": 0.6894369896501302,
+      "num_tokens": 20524247.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.2887881770730019,
+      "epoch": 1.696,
+      "grad_norm": 3.0,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 1.2341,
+      "mean_token_accuracy": 0.6910015381872654,
+      "num_tokens": 20718121.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.3080885756760836,
+      "epoch": 1.712,
+      "grad_norm": 3.328125,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 1.2473,
+      "mean_token_accuracy": 0.6897755783051253,
+      "num_tokens": 20921419.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.2847151305526494,
+      "epoch": 1.728,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 1.2274,
+      "mean_token_accuracy": 0.6899862293154001,
+      "num_tokens": 21118581.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.2512852154672145,
+      "epoch": 1.744,
+      "grad_norm": 2.90625,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 1.2008,
+      "mean_token_accuracy": 0.6946600012481212,
+      "num_tokens": 21315128.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.2720022924244403,
+      "epoch": 1.76,
+      "grad_norm": 3.171875,
+      "learning_rate": 1.978e-05,
+      "loss": 1.2257,
+      "mean_token_accuracy": 0.694522064179182,
+      "num_tokens": 21511066.0,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_biology_entropy": 1.2505228676795959,
+      "eval_biology_loss": 1.2501847743988037,
+      "eval_biology_mean_token_accuracy": 0.6972134056091308,
+      "eval_biology_num_tokens": 21511066.0,
+      "eval_biology_runtime": 39.1908,
+      "eval_biology_samples_per_second": 12.758,
+      "eval_biology_steps_per_second": 3.19,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_chemistry_entropy": 0.7102864451408386,
+      "eval_chemistry_loss": 0.7046002149581909,
+      "eval_chemistry_mean_token_accuracy": 0.8067933526039124,
+      "eval_chemistry_num_tokens": 21511066.0,
+      "eval_chemistry_runtime": 48.815,
+      "eval_chemistry_samples_per_second": 10.243,
+      "eval_chemistry_steps_per_second": 2.561,
+      "step": 1100
+    },
+    {
+      "entropy": 1.281822531670332,
+      "epoch": 1.776,
+      "grad_norm": 3.375,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 1.227,
+      "mean_token_accuracy": 0.6919105790555478,
+      "num_tokens": 21704368.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.2503610096871853,
+      "epoch": 1.792,
+      "grad_norm": 3.03125,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 1.1958,
+      "mean_token_accuracy": 0.6962957978248596,
+      "num_tokens": 21905784.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.2656195681542157,
+      "epoch": 1.808,
+      "grad_norm": 3.1875,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 1.2172,
+      "mean_token_accuracy": 0.6964640185236931,
+      "num_tokens": 22094159.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.288933950662613,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 2.96875,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 1.2226,
+      "mean_token_accuracy": 0.6889266464859247,
+      "num_tokens": 22290654.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.3025077167898416,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 1.2518,
+      "mean_token_accuracy": 0.6867761600762605,
+      "num_tokens": 22485675.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.2597012262791396,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 3.125,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 1.2159,
+      "mean_token_accuracy": 0.6929025657474994,
+      "num_tokens": 22689900.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.2436271466314792,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 1.1869,
+      "mean_token_accuracy": 0.700148145109415,
+      "num_tokens": 22887275.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.2920378752052784,
+      "epoch": 1.888,
+      "grad_norm": 3.265625,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 1.2411,
+      "mean_token_accuracy": 0.6888992667198182,
+      "num_tokens": 23081241.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.2647251799702643,
+      "epoch": 1.904,
+      "grad_norm": 3.0625,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 1.2181,
+      "mean_token_accuracy": 0.693353895843029,
+      "num_tokens": 23276417.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.2675465818494558,
+      "epoch": 1.92,
+      "grad_norm": 3.1875,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 1.218,
+      "mean_token_accuracy": 0.6914148110896349,
+      "num_tokens": 23476340.0,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_biology_entropy": 1.2177199654579163,
+      "eval_biology_loss": 1.2274307012557983,
+      "eval_biology_mean_token_accuracy": 0.7003530249595642,
+      "eval_biology_num_tokens": 23476340.0,
+      "eval_biology_runtime": 39.1872,
+      "eval_biology_samples_per_second": 12.759,
+      "eval_biology_steps_per_second": 3.19,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_chemistry_entropy": 0.7087596535682679,
+      "eval_chemistry_loss": 0.7039869427680969,
+      "eval_chemistry_mean_token_accuracy": 0.8071865315437317,
+      "eval_chemistry_num_tokens": 23476340.0,
+      "eval_chemistry_runtime": 48.8046,
+      "eval_chemistry_samples_per_second": 10.245,
+      "eval_chemistry_steps_per_second": 2.561,
+      "step": 1200
+    },
+    {
+      "entropy": 1.2344753056764604,
+      "epoch": 1.936,
+      "grad_norm": 3.09375,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 1.1806,
+      "mean_token_accuracy": 0.6991059686988592,
+      "num_tokens": 23673208.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.2617500979453324,
+      "epoch": 1.952,
+      "grad_norm": 2.953125,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 1.2016,
+      "mean_token_accuracy": 0.6956066381186247,
+      "num_tokens": 23870993.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.2796590264886618,
+      "epoch": 1.968,
+      "grad_norm": 2.96875,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 1.2352,
+      "mean_token_accuracy": 0.6916444700211286,
+      "num_tokens": 24066234.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.2576303217560052,
+      "epoch": 1.984,
+      "grad_norm": 3.0,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 1.2102,
+      "mean_token_accuracy": 0.6947529528290033,
+      "num_tokens": 24262389.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.2539904467761516,
+      "epoch": 2.0,
+      "grad_norm": 3.078125,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 1.2133,
+      "mean_token_accuracy": 0.6965764258056879,
+      "num_tokens": 24455544.0,
+      "step": 1250
+    },
+    {
+      "entropy": 1.0643845289945602,
+      "epoch": 2.016,
+      "grad_norm": 3.953125,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 1.0024,
+      "mean_token_accuracy": 0.7283362425863743,
+      "num_tokens": 24645425.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.0463637366890908,
+      "epoch": 2.032,
+      "grad_norm": 3.421875,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 1.0027,
+      "mean_token_accuracy": 0.7285306710749865,
+      "num_tokens": 24835956.0,
+      "step": 1270
+    },
+    {
+      "entropy": 1.028846498951316,
+      "epoch": 2.048,
+      "grad_norm": 3.484375,
+      "learning_rate": 1.938e-05,
+      "loss": 0.9666,
+      "mean_token_accuracy": 0.732728997990489,
+      "num_tokens": 25036470.0,
+      "step": 1280
+    },
+    {
+      "entropy": 1.0426747165620327,
+      "epoch": 2.064,
+      "grad_norm": 3.3125,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 0.9818,
+      "mean_token_accuracy": 0.7311011333018541,
+      "num_tokens": 25230959.0,
+      "step": 1290
+    },
+    {
+      "entropy": 1.028098127245903,
+      "epoch": 2.08,
+      "grad_norm": 3.140625,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 0.9769,
+      "mean_token_accuracy": 0.7325459264218808,
+      "num_tokens": 25422625.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_biology_entropy": 1.0245879197120666,
+      "eval_biology_loss": 1.2628997564315796,
+      "eval_biology_mean_token_accuracy": 0.6987690315246582,
+      "eval_biology_num_tokens": 25422625.0,
+      "eval_biology_runtime": 39.1891,
+      "eval_biology_samples_per_second": 12.759,
+      "eval_biology_steps_per_second": 3.19,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_chemistry_entropy": 0.5973518455028534,
+      "eval_chemistry_loss": 0.7306837439537048,
+      "eval_chemistry_mean_token_accuracy": 0.8039612555503846,
+      "eval_chemistry_num_tokens": 25422625.0,
+      "eval_chemistry_runtime": 48.8067,
+      "eval_chemistry_samples_per_second": 10.244,
+      "eval_chemistry_steps_per_second": 2.561,
+      "step": 1300
+    },
+    {
+      "entropy": 1.0385996278375387,
+      "epoch": 2.096,
+      "grad_norm": 3.40625,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 0.9866,
+      "mean_token_accuracy": 0.7318900249898433,
+      "num_tokens": 25614286.0,
+      "step": 1310
+    },
+    {
+      "entropy": 1.0053702626377343,
+      "epoch": 2.112,
+      "grad_norm": 2.96875,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 0.9581,
+      "mean_token_accuracy": 0.7355095498263836,
+      "num_tokens": 25810384.0,
+      "step": 1320
+    },
+    {
+      "entropy": 1.018695557117462,
+      "epoch": 2.128,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 0.9669,
+      "mean_token_accuracy": 0.735399067774415,
+      "num_tokens": 26005047.0,
+      "step": 1330
+    },
+    {
+      "entropy": 1.0224322900176048,
+      "epoch": 2.144,
+      "grad_norm": 3.734375,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 0.9788,
+      "mean_token_accuracy": 0.7329027950763702,
+      "num_tokens": 26207169.0,
+      "step": 1340
+    },
+    {
+      "entropy": 1.0366303831338883,
+      "epoch": 2.16,
+      "grad_norm": 3.390625,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 0.9863,
+      "mean_token_accuracy": 0.7301317453384399,
+      "num_tokens": 26402150.0,
+      "step": 1350
+    },
+    {
+      "entropy": 1.0458955060690642,
+      "epoch": 2.176,
+      "grad_norm": 3.375,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 0.999,
+      "mean_token_accuracy": 0.7267820101231337,
+      "num_tokens": 26599464.0,
+      "step": 1360
+    },
+    {
+      "entropy": 1.0146347057074308,
+      "epoch": 2.192,
+      "grad_norm": 3.0,
+      "learning_rate": 1.918e-05,
+      "loss": 0.9545,
+      "mean_token_accuracy": 0.7366683144122362,
+      "num_tokens": 26790601.0,
+      "step": 1370
+    },
+    {
+      "entropy": 1.0320534974336624,
+      "epoch": 2.208,
+      "grad_norm": 3.140625,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 0.9896,
+      "mean_token_accuracy": 0.7306183617562055,
+      "num_tokens": 26988598.0,
+      "step": 1380
+    },
+    {
+      "entropy": 1.061497811973095,
+      "epoch": 2.224,
+      "grad_norm": 3.171875,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 1.0068,
+      "mean_token_accuracy": 0.7265766266733408,
+      "num_tokens": 27181933.0,
+      "step": 1390
+    },
+    {
+      "entropy": 1.0051178384572268,
+      "epoch": 2.24,
+      "grad_norm": 3.1875,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 0.9516,
+      "mean_token_accuracy": 0.7369365498423577,
+      "num_tokens": 27378686.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_biology_entropy": 1.017266115665436,
+      "eval_biology_loss": 1.2557278871536255,
+      "eval_biology_mean_token_accuracy": 0.700424069404602,
+      "eval_biology_num_tokens": 27378686.0,
+      "eval_biology_runtime": 39.0925,
+      "eval_biology_samples_per_second": 12.79,
+      "eval_biology_steps_per_second": 3.198,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_chemistry_entropy": 0.5852259058952332,
+      "eval_chemistry_loss": 0.7377539277076721,
+      "eval_chemistry_mean_token_accuracy": 0.8031037306785583,
+      "eval_chemistry_num_tokens": 27378686.0,
+      "eval_chemistry_runtime": 48.69,
+      "eval_chemistry_samples_per_second": 10.269,
+      "eval_chemistry_steps_per_second": 2.567,
+      "step": 1400
+    },
+    {
+      "entropy": 1.0071753267198802,
+      "epoch": 2.2560000000000002,
+      "grad_norm": 3.1875,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 0.9674,
+      "mean_token_accuracy": 0.7368769370019436,
+      "num_tokens": 27571908.0,
+      "step": 1410
+    },
+    {
+      "entropy": 1.0086324002593756,
+      "epoch": 2.2720000000000002,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 0.9656,
+      "mean_token_accuracy": 0.7324758760631085,
+      "num_tokens": 27770478.0,
+      "step": 1420
+    },
+    {
+      "entropy": 1.0463698647916317,
+      "epoch": 2.288,
+      "grad_norm": 3.421875,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 0.9934,
+      "mean_token_accuracy": 0.7270413629710675,
+      "num_tokens": 27965558.0,
+      "step": 1430
+    },
+    {
+      "entropy": 1.042240034788847,
+      "epoch": 2.304,
+      "grad_norm": 2.953125,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 0.9889,
+      "mean_token_accuracy": 0.7297652084380388,
+      "num_tokens": 28165624.0,
+      "step": 1440
+    },
+    {
+      "entropy": 1.0128632839769125,
+      "epoch": 2.32,
+      "grad_norm": 3.03125,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 0.9682,
+      "mean_token_accuracy": 0.7353047866374254,
+      "num_tokens": 28363594.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.021491314470768,
+      "epoch": 2.336,
+      "grad_norm": 3.375,
+      "learning_rate": 1.898e-05,
+      "loss": 0.9763,
+      "mean_token_accuracy": 0.7318463895469904,
+      "num_tokens": 28558104.0,
+      "step": 1460
+    },
+    {
+      "entropy": 1.0460473388433456,
+      "epoch": 2.352,
+      "grad_norm": 3.234375,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 0.9934,
+      "mean_token_accuracy": 0.7273747593164444,
+      "num_tokens": 28747265.0,
+      "step": 1470
+    },
+    {
+      "entropy": 1.0281270746141673,
+      "epoch": 2.368,
+      "grad_norm": 2.953125,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 0.9686,
+      "mean_token_accuracy": 0.7312608975917101,
+      "num_tokens": 28944694.0,
+      "step": 1480
+    },
+    {
+      "entropy": 1.052429899200797,
+      "epoch": 2.384,
+      "grad_norm": 3.40625,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 0.9965,
+      "mean_token_accuracy": 0.7276010267436505,
+      "num_tokens": 29136617.0,
+      "step": 1490
+    },
+    {
+      "entropy": 1.0189764112234116,
+      "epoch": 2.4,
+      "grad_norm": 3.25,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 0.969,
+      "mean_token_accuracy": 0.7323620662093162,
+      "num_tokens": 29333526.0,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_biology_entropy": 1.0150307307243347,
+      "eval_biology_loss": 1.2410078048706055,
+      "eval_biology_mean_token_accuracy": 0.7019601583480835,
+      "eval_biology_num_tokens": 29333526.0,
+      "eval_biology_runtime": 39.111,
+      "eval_biology_samples_per_second": 12.784,
+      "eval_biology_steps_per_second": 3.196,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_chemistry_entropy": 0.5947320945262909,
+      "eval_chemistry_loss": 0.735136091709137,
+      "eval_chemistry_mean_token_accuracy": 0.8036991062164307,
+      "eval_chemistry_num_tokens": 29333526.0,
+      "eval_chemistry_runtime": 48.5118,
+      "eval_chemistry_samples_per_second": 10.307,
+      "eval_chemistry_steps_per_second": 2.577,
+      "step": 1500
+    },
+    {
+      "entropy": 1.0211387369781733,
+      "epoch": 2.416,
+      "grad_norm": 3.6875,
+      "learning_rate": 1.886888888888889e-05,
+      "loss": 0.9757,
+      "mean_token_accuracy": 0.7318841896951198,
+      "num_tokens": 29534153.0,
+      "step": 1510
+    },
+    {
+      "entropy": 1.0182897172868253,
+      "epoch": 2.432,
+      "grad_norm": 3.21875,
+      "learning_rate": 1.884666666666667e-05,
+      "loss": 0.9676,
+      "mean_token_accuracy": 0.7337216585874557,
+      "num_tokens": 29736571.0,
+      "step": 1520
+    },
+    {
+      "entropy": 1.055547694116831,
+      "epoch": 2.448,
+      "grad_norm": 3.65625,
+      "learning_rate": 1.8824444444444445e-05,
+      "loss": 1.0122,
+      "mean_token_accuracy": 0.7263809140771628,
+      "num_tokens": 29932423.0,
+      "step": 1530
+    },
+    {
+      "entropy": 1.002600496634841,
+      "epoch": 2.464,
+      "grad_norm": 3.453125,
+      "learning_rate": 1.8802222222222226e-05,
+      "loss": 0.9505,
+      "mean_token_accuracy": 0.736130004748702,
+      "num_tokens": 30137125.0,
+      "step": 1540
+    },
+    {
+      "entropy": 1.015967608243227,
+      "epoch": 2.48,
+      "grad_norm": 3.390625,
+      "learning_rate": 1.878e-05,
+      "loss": 0.9704,
+      "mean_token_accuracy": 0.7336820974946022,
+      "num_tokens": 30341656.0,
+      "step": 1550
+    },
+    {
+      "entropy": 1.0205113902688026,
+      "epoch": 2.496,
+      "grad_norm": 3.046875,
+      "learning_rate": 1.875777777777778e-05,
+      "loss": 0.9609,
+      "mean_token_accuracy": 0.7317427225410938,
+      "num_tokens": 30533732.0,
+      "step": 1560
+    },
+    {
+      "entropy": 1.039156510680914,
+      "epoch": 2.512,
+      "grad_norm": 3.703125,
+      "learning_rate": 1.873555555555556e-05,
+      "loss": 0.9891,
+      "mean_token_accuracy": 0.7297920010983944,
+      "num_tokens": 30730599.0,
+      "step": 1570
+    },
+    {
+      "entropy": 1.0211701802909374,
+      "epoch": 2.528,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.8713333333333336e-05,
+      "loss": 0.9703,
+      "mean_token_accuracy": 0.7342745490372181,
+      "num_tokens": 30927950.0,
+      "step": 1580
+    },
+    {
+      "entropy": 1.0247683700174093,
+      "epoch": 2.544,
+      "grad_norm": 3.21875,
+      "learning_rate": 1.8691111111111114e-05,
+      "loss": 0.9847,
+      "mean_token_accuracy": 0.7314475499093532,
+      "num_tokens": 31125667.0,
+      "step": 1590
+    },
+    {
+      "entropy": 1.0344158336520195,
+      "epoch": 2.56,
+      "grad_norm": 3.65625,
+      "learning_rate": 1.866888888888889e-05,
+      "loss": 0.9852,
+      "mean_token_accuracy": 0.7309044182300568,
+      "num_tokens": 31315742.0,
+      "step": 1600
+    },
+    {
+      "epoch": 2.56,
+      "eval_biology_entropy": 0.9934879779815674,
+      "eval_biology_loss": 1.2355761528015137,
+      "eval_biology_mean_token_accuracy": 0.703362479686737,
+      "eval_biology_num_tokens": 31315742.0,
+      "eval_biology_runtime": 39.1386,
+      "eval_biology_samples_per_second": 12.775,
+      "eval_biology_steps_per_second": 3.194,
+      "step": 1600
+    },
+    {
+      "epoch": 2.56,
+      "eval_chemistry_entropy": 0.5818395533561707,
+      "eval_chemistry_loss": 0.7372699975967407,
+      "eval_chemistry_mean_token_accuracy": 0.8029632153511047,
+      "eval_chemistry_num_tokens": 31315742.0,
+      "eval_chemistry_runtime": 48.7641,
+      "eval_chemistry_samples_per_second": 10.253,
+      "eval_chemistry_steps_per_second": 2.563,
+      "step": 1600
+    },
+    {
+      "entropy": 1.0386214345693587,
+      "epoch": 2.576,
+      "grad_norm": 3.25,
+      "learning_rate": 1.864666666666667e-05,
+      "loss": 0.9889,
+      "mean_token_accuracy": 0.7297458287328482,
+      "num_tokens": 31505374.0,
+      "step": 1610
+    },
+    {
+      "entropy": 1.0596749667078256,
+      "epoch": 2.592,
+      "grad_norm": 3.21875,
+      "learning_rate": 1.8624444444444446e-05,
+      "loss": 1.0091,
+      "mean_token_accuracy": 0.7234568212181329,
+      "num_tokens": 31701680.0,
+      "step": 1620
+    },
+    {
+      "entropy": 1.043595340102911,
+      "epoch": 2.608,
+      "grad_norm": 3.140625,
+      "learning_rate": 1.8602222222222224e-05,
+      "loss": 0.9956,
+      "mean_token_accuracy": 0.7284572850912809,
+      "num_tokens": 31893221.0,
+      "step": 1630
+    },
+    {
+      "entropy": 1.0105513636022807,
+      "epoch": 2.624,
+      "grad_norm": 3.40625,
+      "learning_rate": 1.858e-05,
+      "loss": 0.9594,
+      "mean_token_accuracy": 0.7338220972567797,
+      "num_tokens": 32087176.0,
+      "step": 1640
+    },
+    {
+      "entropy": 1.0368166904896499,
+      "epoch": 2.64,
+      "grad_norm": 3.5625,
+      "learning_rate": 1.855777777777778e-05,
+      "loss": 0.9881,
+      "mean_token_accuracy": 0.7307275418192148,
+      "num_tokens": 32280689.0,
+      "step": 1650
+    },
+    {
+      "entropy": 1.0114492613822221,
+      "epoch": 2.656,
+      "grad_norm": 3.140625,
+      "learning_rate": 1.8535555555555557e-05,
+      "loss": 0.9589,
+      "mean_token_accuracy": 0.7375065598636865,
+      "num_tokens": 32483392.0,
+      "step": 1660
+    },
+    {
+      "entropy": 1.0144252099096775,
+      "epoch": 2.672,
+      "grad_norm": 3.15625,
+      "learning_rate": 1.8513333333333335e-05,
+      "loss": 0.9711,
+      "mean_token_accuracy": 0.734334035590291,
+      "num_tokens": 32683246.0,
+      "step": 1670
+    },
+    {
+      "entropy": 1.0010439176112413,
+      "epoch": 2.6879999999999997,
+      "grad_norm": 3.375,
+      "learning_rate": 1.8491111111111112e-05,
+      "loss": 0.9492,
+      "mean_token_accuracy": 0.7375241097062826,
+      "num_tokens": 32884993.0,
+      "step": 1680
+    },
+    {
+      "entropy": 1.0266988582909107,
+      "epoch": 2.7039999999999997,
+      "grad_norm": 3.078125,
+      "learning_rate": 1.846888888888889e-05,
+      "loss": 0.9742,
+      "mean_token_accuracy": 0.7322238199412823,
+      "num_tokens": 33088388.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.9994403839111328,
+      "epoch": 2.7199999999999998,
+      "grad_norm": 3.453125,
+      "learning_rate": 1.8446666666666667e-05,
+      "loss": 0.9598,
+      "mean_token_accuracy": 0.7361585844308138,
+      "num_tokens": 33287208.0,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "eval_biology_entropy": 1.0237988924980164,
+      "eval_biology_loss": 1.2203679084777832,
+      "eval_biology_mean_token_accuracy": 0.7062526535987854,
+      "eval_biology_num_tokens": 33287208.0,
+      "eval_biology_runtime": 39.1109,
+      "eval_biology_samples_per_second": 12.784,
+      "eval_biology_steps_per_second": 3.196,
+      "step": 1700
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "eval_chemistry_entropy": 0.6041144397258759,
+      "eval_chemistry_loss": 0.7344766855239868,
+      "eval_chemistry_mean_token_accuracy": 0.8033582849502563,
+      "eval_chemistry_num_tokens": 33287208.0,
+      "eval_chemistry_runtime": 48.7203,
+      "eval_chemistry_samples_per_second": 10.263,
+      "eval_chemistry_steps_per_second": 2.566,
+      "step": 1700
+    },
+    {
+      "entropy": 1.0407861892133952,
+      "epoch": 2.7359999999999998,
+      "grad_norm": 3.09375,
+      "learning_rate": 1.842444444444445e-05,
+      "loss": 0.9879,
+      "mean_token_accuracy": 0.7298587996512651,
+      "num_tokens": 33474750.0,
+      "step": 1710
+    },
+    {
+      "entropy": 1.0247716046869755,
+      "epoch": 2.752,
+      "grad_norm": 3.609375,
+      "learning_rate": 1.8402222222222223e-05,
+      "loss": 0.9769,
+      "mean_token_accuracy": 0.730589097738266,
+      "num_tokens": 33664302.0,
+      "step": 1720
+    },
+    {
+      "entropy": 1.0562083158642053,
+      "epoch": 2.768,
+      "grad_norm": 3.390625,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 1.0054,
+      "mean_token_accuracy": 0.7239043671637774,
+      "num_tokens": 33854081.0,
+      "step": 1730
+    },
+    {
+      "entropy": 1.016645783931017,
+      "epoch": 2.784,
+      "grad_norm": 3.0,
+      "learning_rate": 1.8357777777777778e-05,
+      "loss": 0.9691,
+      "mean_token_accuracy": 0.7346661690622568,
+      "num_tokens": 34053149.0,
+      "step": 1740
+    },
+    {
+      "entropy": 1.0153463836759329,
+      "epoch": 2.8,
+      "grad_norm": 3.125,
+      "learning_rate": 1.833555555555556e-05,
+      "loss": 0.969,
+      "mean_token_accuracy": 0.7363858807832002,
+      "num_tokens": 34254105.0,
+      "step": 1750
+    },
+    {
+      "entropy": 1.0121935941278934,
+      "epoch": 2.816,
+      "grad_norm": 3.15625,
+      "learning_rate": 1.8313333333333333e-05,
+      "loss": 0.9579,
+      "mean_token_accuracy": 0.7347628369927406,
+      "num_tokens": 34453342.0,
+      "step": 1760
+    },
+    {
+      "entropy": 1.050192491337657,
+      "epoch": 2.832,
+      "grad_norm": 3.421875,
+      "learning_rate": 1.8291111111111114e-05,
+      "loss": 0.9938,
+      "mean_token_accuracy": 0.7286655321717262,
+      "num_tokens": 34644113.0,
+      "step": 1770
+    },
+    {
+      "entropy": 1.0338923085480929,
+      "epoch": 2.848,
+      "grad_norm": 3.296875,
+      "learning_rate": 1.8268888888888888e-05,
+      "loss": 0.9888,
+      "mean_token_accuracy": 0.7316403832286597,
+      "num_tokens": 34835062.0,
+      "step": 1780
+    },
+    {
+      "entropy": 1.0176855057477951,
+      "epoch": 2.864,
+      "grad_norm": 3.25,
+      "learning_rate": 1.824666666666667e-05,
+      "loss": 0.9754,
+      "mean_token_accuracy": 0.7330775160342455,
+      "num_tokens": 35032361.0,
+      "step": 1790
+    },
+    {
+      "entropy": 1.0399887025356294,
+      "epoch": 2.88,
+      "grad_norm": 3.21875,
+      "learning_rate": 1.8224444444444447e-05,
+      "loss": 0.9921,
+      "mean_token_accuracy": 0.7296446476131677,
+      "num_tokens": 35226653.0,
+      "step": 1800
+    },
+    {
+      "epoch": 2.88,
+      "eval_biology_entropy": 1.0291271986961366,
+      "eval_biology_loss": 1.2109084129333496,
+      "eval_biology_mean_token_accuracy": 0.70716144323349,
+      "eval_biology_num_tokens": 35226653.0,
+      "eval_biology_runtime": 39.2084,
+      "eval_biology_samples_per_second": 12.752,
+      "eval_biology_steps_per_second": 3.188,
+      "step": 1800
+    },
+    {
+      "epoch": 2.88,
+      "eval_chemistry_entropy": 0.5928264305591583,
+      "eval_chemistry_loss": 0.7367877960205078,
+      "eval_chemistry_mean_token_accuracy": 0.8034037079811096,
+      "eval_chemistry_num_tokens": 35226653.0,
+      "eval_chemistry_runtime": 48.8018,
+      "eval_chemistry_samples_per_second": 10.246,
+      "eval_chemistry_steps_per_second": 2.561,
+      "step": 1800
+    },
+    {
+      "entropy": 1.0336259830743075,
+      "epoch": 2.896,
+      "grad_norm": 3.234375,
+      "learning_rate": 1.8202222222222225e-05,
+      "loss": 0.9812,
+      "mean_token_accuracy": 0.7294993791729212,
+      "num_tokens": 35421873.0,
+      "step": 1810
+    },
+    {
+      "entropy": 1.0508763272315265,
+      "epoch": 2.912,
+      "grad_norm": 3.5625,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 1.0003,
+      "mean_token_accuracy": 0.728679533302784,
+      "num_tokens": 35614590.0,
+      "step": 1820
+    },
+    {
+      "entropy": 1.0313632506877184,
+      "epoch": 2.928,
+      "grad_norm": 3.515625,
+      "learning_rate": 1.815777777777778e-05,
+      "loss": 0.9877,
+      "mean_token_accuracy": 0.7299741402268409,
+      "num_tokens": 35806494.0,
+      "step": 1830
+    },
+    {
+      "entropy": 1.023673979192972,
+      "epoch": 2.944,
+      "grad_norm": 3.09375,
+      "learning_rate": 1.8135555555555557e-05,
+      "loss": 0.9814,
+      "mean_token_accuracy": 0.7329835917800664,
+      "num_tokens": 36001867.0,
+      "step": 1840
+    },
+    {
+      "entropy": 1.0165797971189021,
+      "epoch": 2.96,
+      "grad_norm": 3.296875,
+      "learning_rate": 1.8113333333333335e-05,
+      "loss": 0.9588,
+      "mean_token_accuracy": 0.7343671884387731,
+      "num_tokens": 36198806.0,
+      "step": 1850
+    },
+    {
+      "entropy": 1.0112668476998805,
+      "epoch": 2.976,
+      "grad_norm": 3.1875,
+      "learning_rate": 1.8091111111111113e-05,
+      "loss": 0.9622,
+      "mean_token_accuracy": 0.7349605076014996,
+      "num_tokens": 36393098.0,
+      "step": 1860
+    },
+    {
+      "entropy": 1.0263954121619463,
+      "epoch": 2.992,
+      "grad_norm": 3.265625,
+      "learning_rate": 1.806888888888889e-05,
+      "loss": 0.9775,
+      "mean_token_accuracy": 0.7333775267004967,
+      "num_tokens": 36589074.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.938998518884182,
+      "epoch": 3.008,
+      "grad_norm": 4.53125,
+      "learning_rate": 1.8046666666666668e-05,
+      "loss": 0.8348,
+      "mean_token_accuracy": 0.7635289087891579,
+      "num_tokens": 36778710.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.673611904680729,
+      "epoch": 3.024,
+      "grad_norm": 3.609375,
+      "learning_rate": 1.8024444444444445e-05,
+      "loss": 0.6431,
+      "mean_token_accuracy": 0.8048487104475498,
+      "num_tokens": 36966789.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.6563162552192807,
+      "epoch": 3.04,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.8002222222222223e-05,
+      "loss": 0.598,
+      "mean_token_accuracy": 0.8154135558754205,
+      "num_tokens": 37156529.0,
+      "step": 1900
+    },
+    {
+      "epoch": 3.04,
+      "eval_biology_entropy": 0.7205301468372345,
+      "eval_biology_loss": 1.454393744468689,
+      "eval_biology_mean_token_accuracy": 0.6943806533813477,
+      "eval_biology_num_tokens": 37156529.0,
+      "eval_biology_runtime": 39.1135,
+      "eval_biology_samples_per_second": 12.783,
+      "eval_biology_steps_per_second": 3.196,
+      "step": 1900
+    },
+    {
+      "epoch": 3.04,
+      "eval_chemistry_entropy": 0.44404922652244566,
+      "eval_chemistry_loss": 0.8767896890640259,
+      "eval_chemistry_mean_token_accuracy": 0.7932383036613464,
+      "eval_chemistry_num_tokens": 37156529.0,
+      "eval_chemistry_runtime": 48.7267,
+      "eval_chemistry_samples_per_second": 10.261,
+      "eval_chemistry_steps_per_second": 2.565,
+      "step": 1900
+    },
+    {
+      "entropy": 0.6639188259840012,
+      "epoch": 3.056,
+      "grad_norm": 4.125,
+      "learning_rate": 1.798e-05,
+      "loss": 0.6032,
+      "mean_token_accuracy": 0.8127994637936353,
+      "num_tokens": 37351325.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.6422042794525623,
+      "epoch": 3.072,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.7957777777777778e-05,
+      "loss": 0.5921,
+      "mean_token_accuracy": 0.8172261964529752,
+      "num_tokens": 37541069.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.6540521441027523,
+      "epoch": 3.088,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.7935555555555556e-05,
+      "loss": 0.5993,
+      "mean_token_accuracy": 0.8149240974336862,
+      "num_tokens": 37733440.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.6463017119094729,
+      "epoch": 3.104,
+      "grad_norm": 3.71875,
+      "learning_rate": 1.7913333333333337e-05,
+      "loss": 0.5868,
+      "mean_token_accuracy": 0.8175160981714725,
+      "num_tokens": 37933661.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.6483396209776402,
+      "epoch": 3.12,
+      "grad_norm": 4.0,
+      "learning_rate": 1.789111111111111e-05,
+      "loss": 0.6017,
+      "mean_token_accuracy": 0.8143340889364481,
+      "num_tokens": 38126905.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.6592650156468153,
+      "epoch": 3.136,
+      "grad_norm": 4.4375,
+      "learning_rate": 1.7868888888888892e-05,
+      "loss": 0.598,
+      "mean_token_accuracy": 0.8149209085851907,
+      "num_tokens": 38316042.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.6408368192613125,
+      "epoch": 3.152,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.7846666666666666e-05,
+      "loss": 0.5898,
+      "mean_token_accuracy": 0.8172995913773775,
+      "num_tokens": 38514212.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.6699947357177735,
+      "epoch": 3.168,
+      "grad_norm": 4.09375,
+      "learning_rate": 1.7824444444444447e-05,
+      "loss": 0.6136,
+      "mean_token_accuracy": 0.8107931781560183,
+      "num_tokens": 38708822.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.6612202525138855,
+      "epoch": 3.184,
+      "grad_norm": 3.859375,
+      "learning_rate": 1.780222222222222e-05,
+      "loss": 0.6107,
+      "mean_token_accuracy": 0.8108679629862309,
+      "num_tokens": 38900708.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.6577222358435393,
+      "epoch": 3.2,
+      "grad_norm": 4.09375,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 0.6033,
+      "mean_token_accuracy": 0.8127841059118509,
+      "num_tokens": 39102325.0,
+      "step": 2000
+    },
+    {
+      "epoch": 3.2,
+      "eval_biology_entropy": 0.7194305346012115,
+      "eval_biology_loss": 1.458989143371582,
+      "eval_biology_mean_token_accuracy": 0.6955648012161255,
+      "eval_biology_num_tokens": 39102325.0,
+      "eval_biology_runtime": 39.0915,
+      "eval_biology_samples_per_second": 12.791,
+      "eval_biology_steps_per_second": 3.198,
+      "step": 2000
+    },
+    {
+      "epoch": 3.2,
+      "eval_chemistry_entropy": 0.4279054062366486,
+      "eval_chemistry_loss": 0.8939092755317688,
+      "eval_chemistry_mean_token_accuracy": 0.7933331608772278,
+      "eval_chemistry_num_tokens": 39102325.0,
+      "eval_chemistry_runtime": 48.5319,
+      "eval_chemistry_samples_per_second": 10.303,
+      "eval_chemistry_steps_per_second": 2.576,
+      "step": 2000
+    },
+    {
+      "entropy": 0.6628522459417582,
+      "epoch": 3.216,
+      "grad_norm": 4.5,
+      "learning_rate": 1.7757777777777777e-05,
+      "loss": 0.6056,
+      "mean_token_accuracy": 0.8133372392505407,
+      "num_tokens": 39295913.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.6509159937500953,
+      "epoch": 3.232,
+      "grad_norm": 4.21875,
+      "learning_rate": 1.7735555555555558e-05,
+      "loss": 0.5956,
+      "mean_token_accuracy": 0.8132004003971816,
+      "num_tokens": 39495778.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.6906503856182098,
+      "epoch": 3.248,
+      "grad_norm": 4.9375,
+      "learning_rate": 1.7713333333333335e-05,
+      "loss": 0.6307,
+      "mean_token_accuracy": 0.8041897661983967,
+      "num_tokens": 39681539.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.6459178572520614,
+      "epoch": 3.2640000000000002,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.7691111111111113e-05,
+      "loss": 0.5894,
+      "mean_token_accuracy": 0.8152297649532556,
+      "num_tokens": 39887936.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.6582429932430387,
+      "epoch": 3.2800000000000002,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.766888888888889e-05,
+      "loss": 0.6037,
+      "mean_token_accuracy": 0.8127248145639896,
+      "num_tokens": 40081612.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.6647687572985888,
+      "epoch": 3.296,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.7646666666666668e-05,
+      "loss": 0.6134,
+      "mean_token_accuracy": 0.8107028409838677,
+      "num_tokens": 40274536.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.6784954000264406,
+      "epoch": 3.312,
+      "grad_norm": 4.5,
+      "learning_rate": 1.7624444444444446e-05,
+      "loss": 0.6277,
+      "mean_token_accuracy": 0.8066128794103861,
+      "num_tokens": 40465691.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.6526031637564301,
+      "epoch": 3.328,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.7602222222222223e-05,
+      "loss": 0.5965,
+      "mean_token_accuracy": 0.81493533924222,
+      "num_tokens": 40665740.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.6589240668341517,
+      "epoch": 3.344,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.758e-05,
+      "loss": 0.6031,
+      "mean_token_accuracy": 0.8119147256016731,
+      "num_tokens": 40859755.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.6566657887771725,
+      "epoch": 3.36,
+      "grad_norm": 3.984375,
+      "learning_rate": 1.755777777777778e-05,
+      "loss": 0.6065,
+      "mean_token_accuracy": 0.8120837610214948,
+      "num_tokens": 41059360.0,
+      "step": 2100
+    },
+    {
+      "epoch": 3.36,
+      "eval_biology_entropy": 0.7223304212093353,
+      "eval_biology_loss": 1.4520024061203003,
+      "eval_biology_mean_token_accuracy": 0.6971806507110596,
+      "eval_biology_num_tokens": 41059360.0,
+      "eval_biology_runtime": 39.0816,
+      "eval_biology_samples_per_second": 12.794,
+      "eval_biology_steps_per_second": 3.198,
+      "step": 2100
+    },
+    {
+      "epoch": 3.36,
+      "eval_chemistry_entropy": 0.425049998998642,
+      "eval_chemistry_loss": 0.8979185223579407,
+      "eval_chemistry_mean_token_accuracy": 0.7932314896583557,
+      "eval_chemistry_num_tokens": 41059360.0,
+      "eval_chemistry_runtime": 48.685,
+      "eval_chemistry_samples_per_second": 10.27,
+      "eval_chemistry_steps_per_second": 2.568,
+      "step": 2100
+    },
+    {
+      "entropy": 0.6639438619837165,
+      "epoch": 3.376,
+      "grad_norm": 4.21875,
+      "learning_rate": 1.7535555555555556e-05,
+      "loss": 0.6149,
+      "mean_token_accuracy": 0.8104340396821499,
+      "num_tokens": 41256224.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.6947120493277907,
+      "epoch": 3.392,
+      "grad_norm": 4.4375,
+      "learning_rate": 1.7513333333333334e-05,
+      "loss": 0.6371,
+      "mean_token_accuracy": 0.8046501569449902,
+      "num_tokens": 41449411.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.6828485745936632,
+      "epoch": 3.408,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.749111111111111e-05,
+      "loss": 0.6288,
+      "mean_token_accuracy": 0.8066561881452798,
+      "num_tokens": 41636184.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.6725861068814993,
+      "epoch": 3.424,
+      "grad_norm": 3.90625,
+      "learning_rate": 1.746888888888889e-05,
+      "loss": 0.6202,
+      "mean_token_accuracy": 0.81025187112391,
+      "num_tokens": 41831661.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.6702702101320028,
+      "epoch": 3.44,
+      "grad_norm": 4.5,
+      "learning_rate": 1.7446666666666667e-05,
+      "loss": 0.6107,
+      "mean_token_accuracy": 0.8081615891307592,
+      "num_tokens": 42030546.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.6535747822374105,
+      "epoch": 3.456,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.7424444444444444e-05,
+      "loss": 0.6064,
+      "mean_token_accuracy": 0.8117943584918976,
+      "num_tokens": 42232724.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.6701465247198939,
+      "epoch": 3.472,
+      "grad_norm": 4.25,
+      "learning_rate": 1.7402222222222222e-05,
+      "loss": 0.6144,
+      "mean_token_accuracy": 0.8089125752449036,
+      "num_tokens": 42428139.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.6602701621130109,
+      "epoch": 3.488,
+      "grad_norm": 4.09375,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 0.6051,
+      "mean_token_accuracy": 0.8109641902148723,
+      "num_tokens": 42624281.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.6750716870650649,
+      "epoch": 3.504,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.735777777777778e-05,
+      "loss": 0.6224,
+      "mean_token_accuracy": 0.8076240297406911,
+      "num_tokens": 42820999.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.6583628479391337,
+      "epoch": 3.52,
+      "grad_norm": 4.25,
+      "learning_rate": 1.7335555555555558e-05,
+      "loss": 0.6027,
+      "mean_token_accuracy": 0.8134463246911764,
+      "num_tokens": 43024350.0,
+      "step": 2200
+    },
+    {
+      "epoch": 3.52,
+      "eval_biology_entropy": 0.7312813591957092,
+      "eval_biology_loss": 1.448984146118164,
+      "eval_biology_mean_token_accuracy": 0.6975588989257813,
+      "eval_biology_num_tokens": 43024350.0,
+      "eval_biology_runtime": 39.1828,
+      "eval_biology_samples_per_second": 12.761,
+      "eval_biology_steps_per_second": 3.19,
+      "step": 2200
+    },
+    {
+      "epoch": 3.52,
+      "eval_chemistry_entropy": 0.42386009073257447,
+      "eval_chemistry_loss": 0.9041973352432251,
+      "eval_chemistry_mean_token_accuracy": 0.7924366879463196,
+      "eval_chemistry_num_tokens": 43024350.0,
+      "eval_chemistry_runtime": 48.8105,
+      "eval_chemistry_samples_per_second": 10.244,
+      "eval_chemistry_steps_per_second": 2.561,
+      "step": 2200
+    },
+    {
+      "entropy": 0.6752564821392297,
+      "epoch": 3.536,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.7313333333333336e-05,
+      "loss": 0.6237,
+      "mean_token_accuracy": 0.8093563176691532,
+      "num_tokens": 43214925.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.675455342233181,
+      "epoch": 3.552,
+      "grad_norm": 4.375,
+      "learning_rate": 1.7291111111111113e-05,
+      "loss": 0.6187,
+      "mean_token_accuracy": 0.8095767885446549,
+      "num_tokens": 43409227.0,
+      "step": 2220
+    },
+    {
+      "entropy": 0.6612193696200848,
+      "epoch": 3.568,
+      "grad_norm": 4.09375,
+      "learning_rate": 1.726888888888889e-05,
+      "loss": 0.6128,
+      "mean_token_accuracy": 0.8100322619080543,
+      "num_tokens": 43609101.0,
+      "step": 2230
+    },
+    {
+      "entropy": 0.6645467173308134,
+      "epoch": 3.584,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.724666666666667e-05,
+      "loss": 0.61,
+      "mean_token_accuracy": 0.8110004521906375,
+      "num_tokens": 43806883.0,
+      "step": 2240
+    },
+    {
+      "entropy": 0.6700274987146259,
+      "epoch": 3.6,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.7224444444444446e-05,
+      "loss": 0.6151,
+      "mean_token_accuracy": 0.8092441029846669,
+      "num_tokens": 43996466.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.6563864570111037,
+      "epoch": 3.616,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.7202222222222224e-05,
+      "loss": 0.6092,
+      "mean_token_accuracy": 0.8101060092449188,
+      "num_tokens": 44199791.0,
+      "step": 2260
+    },
+    {
+      "entropy": 0.6729124957695604,
+      "epoch": 3.632,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.718e-05,
+      "loss": 0.6165,
+      "mean_token_accuracy": 0.8097267791628837,
+      "num_tokens": 44400793.0,
+      "step": 2270
+    },
+    {
+      "entropy": 0.7002201689407229,
+      "epoch": 3.648,
+      "grad_norm": 4.53125,
+      "learning_rate": 1.715777777777778e-05,
+      "loss": 0.6449,
+      "mean_token_accuracy": 0.8013300366699696,
+      "num_tokens": 44592807.0,
+      "step": 2280
+    },
+    {
+      "entropy": 0.6589280311018229,
+      "epoch": 3.664,
+      "grad_norm": 4.125,
+      "learning_rate": 1.7135555555555557e-05,
+      "loss": 0.6106,
+      "mean_token_accuracy": 0.8104540213942528,
+      "num_tokens": 44796238.0,
+      "step": 2290
+    },
+    {
+      "entropy": 0.6718129109591245,
+      "epoch": 3.68,
+      "grad_norm": 4.09375,
+      "learning_rate": 1.7113333333333334e-05,
+      "loss": 0.6148,
+      "mean_token_accuracy": 0.808036233112216,
+      "num_tokens": 44996201.0,
+      "step": 2300
+    },
+    {
+      "epoch": 3.68,
+      "eval_biology_entropy": 0.7342447080612182,
+      "eval_biology_loss": 1.4424549341201782,
+      "eval_biology_mean_token_accuracy": 0.6978958992958069,
+      "eval_biology_num_tokens": 44996201.0,
+      "eval_biology_runtime": 39.18,
+      "eval_biology_samples_per_second": 12.762,
+      "eval_biology_steps_per_second": 3.19,
+      "step": 2300
+    },
+    {
+      "epoch": 3.68,
+      "eval_chemistry_entropy": 0.42464341497421265,
+      "eval_chemistry_loss": 0.9024158120155334,
+      "eval_chemistry_mean_token_accuracy": 0.7931222615242004,
+      "eval_chemistry_num_tokens": 44996201.0,
+      "eval_chemistry_runtime": 48.8209,
+      "eval_chemistry_samples_per_second": 10.242,
+      "eval_chemistry_steps_per_second": 2.56,
+      "step": 2300
+    },
+    {
+      "entropy": 0.6750511961057782,
+      "epoch": 3.6959999999999997,
+      "grad_norm": 4.53125,
+      "learning_rate": 1.7091111111111112e-05,
+      "loss": 0.6263,
+      "mean_token_accuracy": 0.8063635263592005,
+      "num_tokens": 45195456.0,
+      "step": 2310
+    },
+    {
+      "entropy": 0.6969444552436471,
+      "epoch": 3.7119999999999997,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.706888888888889e-05,
+      "loss": 0.6355,
+      "mean_token_accuracy": 0.8052191227674484,
+      "num_tokens": 45385114.0,
+      "step": 2320
+    },
+    {
+      "entropy": 0.6575890483334661,
+      "epoch": 3.7279999999999998,
+      "grad_norm": 3.96875,
+      "learning_rate": 1.704666666666667e-05,
+      "loss": 0.6079,
+      "mean_token_accuracy": 0.8101479094475508,
+      "num_tokens": 45586032.0,
+      "step": 2330
+    },
+    {
+      "entropy": 0.6878007039427757,
+      "epoch": 3.7439999999999998,
+      "grad_norm": 4.25,
+      "learning_rate": 1.7024444444444445e-05,
+      "loss": 0.6287,
+      "mean_token_accuracy": 0.8045054972171783,
+      "num_tokens": 45777138.0,
+      "step": 2340
+    },
+    {
+      "entropy": 0.6799204269424081,
+      "epoch": 3.76,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.7002222222222226e-05,
+      "loss": 0.6291,
+      "mean_token_accuracy": 0.8070768497884273,
+      "num_tokens": 45971818.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.6708019917830825,
+      "epoch": 3.776,
+      "grad_norm": 4.125,
+      "learning_rate": 1.698e-05,
+      "loss": 0.6237,
+      "mean_token_accuracy": 0.8083038441836834,
+      "num_tokens": 46167844.0,
+      "step": 2360
+    },
+    {
+      "entropy": 0.6993245150893926,
+      "epoch": 3.792,
+      "grad_norm": 4.34375,
+      "learning_rate": 1.695777777777778e-05,
+      "loss": 0.6447,
+      "mean_token_accuracy": 0.8005988635122776,
+      "num_tokens": 46362538.0,
+      "step": 2370
+    },
+    {
+      "entropy": 0.6768058065325022,
+      "epoch": 3.808,
+      "grad_norm": 4.25,
+      "learning_rate": 1.6935555555555555e-05,
+      "loss": 0.6247,
+      "mean_token_accuracy": 0.8075179129838943,
+      "num_tokens": 46553166.0,
+      "step": 2380
+    },
+    {
+      "entropy": 0.6853747364133597,
+      "epoch": 3.824,
+      "grad_norm": 4.125,
+      "learning_rate": 1.6913333333333336e-05,
+      "loss": 0.635,
+      "mean_token_accuracy": 0.8052531398832798,
+      "num_tokens": 46744198.0,
+      "step": 2390
+    },
+    {
+      "entropy": 0.6714792435988783,
+      "epoch": 3.84,
+      "grad_norm": 4.21875,
+      "learning_rate": 1.689111111111111e-05,
+      "loss": 0.6228,
+      "mean_token_accuracy": 0.8093580879271031,
+      "num_tokens": 46941095.0,
+      "step": 2400
+    },
+    {
+      "epoch": 3.84,
+      "eval_biology_entropy": 0.7420620145797729,
+      "eval_biology_loss": 1.4358100891113281,
+      "eval_biology_mean_token_accuracy": 0.698259696006775,
+      "eval_biology_num_tokens": 46941095.0,
+      "eval_biology_runtime": 39.0623,
+      "eval_biology_samples_per_second": 12.8,
+      "eval_biology_steps_per_second": 3.2,
+      "step": 2400
+    },
+    {
+      "epoch": 3.84,
+      "eval_chemistry_entropy": 0.4296411466598511,
+      "eval_chemistry_loss": 0.902924656867981,
+      "eval_chemistry_mean_token_accuracy": 0.7924109597206116,
+      "eval_chemistry_num_tokens": 46941095.0,
+      "eval_chemistry_runtime": 48.6871,
+      "eval_chemistry_samples_per_second": 10.27,
+      "eval_chemistry_steps_per_second": 2.567,
+      "step": 2400
+    },
+    {
+      "entropy": 0.6763832990080119,
+      "epoch": 3.856,
+      "grad_norm": 4.28125,
+      "learning_rate": 1.686888888888889e-05,
+      "loss": 0.622,
+      "mean_token_accuracy": 0.8078614696860313,
+      "num_tokens": 47139983.0,
+      "step": 2410
+    },
+    {
+      "entropy": 0.690857807919383,
+      "epoch": 3.872,
+      "grad_norm": 4.21875,
+      "learning_rate": 1.684666666666667e-05,
+      "loss": 0.6425,
+      "mean_token_accuracy": 0.8026266809552908,
+      "num_tokens": 47337119.0,
+      "step": 2420
+    },
+    {
+      "entropy": 0.6782918659970164,
+      "epoch": 3.888,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.6824444444444447e-05,
+      "loss": 0.6103,
+      "mean_token_accuracy": 0.8091849368065596,
+      "num_tokens": 47535142.0,
+      "step": 2430
+    },
+    {
+      "entropy": 0.6686505619436502,
+      "epoch": 3.904,
+      "grad_norm": 4.25,
+      "learning_rate": 1.6802222222222224e-05,
+      "loss": 0.6245,
+      "mean_token_accuracy": 0.8080017164349556,
+      "num_tokens": 47729505.0,
+      "step": 2440
+    },
+    {
+      "entropy": 0.6681339140981436,
+      "epoch": 3.92,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 0.6189,
+      "mean_token_accuracy": 0.8098367966711522,
+      "num_tokens": 47929285.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.6815527578815818,
+      "epoch": 3.936,
+      "grad_norm": 4.40625,
+      "learning_rate": 1.675777777777778e-05,
+      "loss": 0.6279,
+      "mean_token_accuracy": 0.8059242676943541,
+      "num_tokens": 48127866.0,
+      "step": 2460
+    },
+    {
+      "entropy": 0.6731824316084385,
+      "epoch": 3.952,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.6735555555555557e-05,
+      "loss": 0.6223,
+      "mean_token_accuracy": 0.8080686226487159,
+      "num_tokens": 48325429.0,
+      "step": 2470
+    },
+    {
+      "entropy": 0.7055601105093956,
+      "epoch": 3.968,
+      "grad_norm": 4.8125,
+      "learning_rate": 1.6713333333333335e-05,
+      "loss": 0.6452,
+      "mean_token_accuracy": 0.8004195280373096,
+      "num_tokens": 48522725.0,
+      "step": 2480
+    },
+    {
+      "entropy": 0.6725156113505364,
+      "epoch": 3.984,
+      "grad_norm": 3.921875,
+      "learning_rate": 1.6691111111111112e-05,
+      "loss": 0.6208,
+      "mean_token_accuracy": 0.8087173383682966,
+      "num_tokens": 48719975.0,
+      "step": 2490
+    },
+    {
+      "entropy": 0.6917624410241843,
+      "epoch": 4.0,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.666888888888889e-05,
+      "loss": 0.6448,
+      "mean_token_accuracy": 0.8019643574953079,
+      "num_tokens": 48911088.0,
+      "step": 2500
+    },
+    {
+      "epoch": 4.0,
+      "eval_biology_entropy": 0.7488004591464996,
+      "eval_biology_loss": 1.4288437366485596,
+      "eval_biology_mean_token_accuracy": 0.6984080262184144,
+      "eval_biology_num_tokens": 48911088.0,
+      "eval_biology_runtime": 39.1871,
+      "eval_biology_samples_per_second": 12.759,
+      "eval_biology_steps_per_second": 3.19,
+      "step": 2500
+    },
+    {
+      "epoch": 4.0,
+      "eval_chemistry_entropy": 0.43333416879177095,
+      "eval_chemistry_loss": 0.9011425971984863,
+      "eval_chemistry_mean_token_accuracy": 0.7929084548950195,
+      "eval_chemistry_num_tokens": 48911088.0,
+      "eval_chemistry_runtime": 48.521,
+      "eval_chemistry_samples_per_second": 10.305,
+      "eval_chemistry_steps_per_second": 2.576,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 16,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.102472136037589e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b64358284dc9f1de2e91b4b3a078696ba2b6f857c3894ea502f2b47187d627c
+size 6417