Upload folder using huggingface_hub

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-1000/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-1000/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-1000/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-1000/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-1000/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-1000/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a37864f0950e0e995601549751de76b23a805a91742d2d96f6004ff0f297871
+size 4962221464

checkpoint-1000/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8600037329ff4a33ec3e312e565446636853d17f0ae4f02af6af3f653e85437e
+size 3670322200

checkpoint-1000/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f10d467ee881e1052fe7ca51c6b8ee660bfdfea438411569abb0907f8775ab8
+size 1585596266

checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:471b1c653aabe47d1dda07893496c6ec8f21353cd348bef0e1f9b8b6dd76552c
+size 14645

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f23267b381a42784999e08fdda207042bbe3b9bc2b38cecbe7bd208dc572c4ab
+size 1465

checkpoint-1000/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-1000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-1000/trainer_state.json

@@ -0,0 +1,1474 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.6,
+  "eval_steps": 100,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.1780160423368216,
+      "epoch": 0.016,
+      "grad_norm": 13.0,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 1.8406,
+      "mean_token_accuracy": 0.6489301804453135,
+      "num_tokens": 340696.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.1818634796887637,
+      "epoch": 0.032,
+      "grad_norm": 11.5625,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 1.8191,
+      "mean_token_accuracy": 0.6528046734631061,
+      "num_tokens": 675930.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.195955842360854,
+      "epoch": 0.048,
+      "grad_norm": 9.3125,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 1.7817,
+      "mean_token_accuracy": 0.6578715395182371,
+      "num_tokens": 1007956.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.2672166559845208,
+      "epoch": 0.064,
+      "grad_norm": 6.5625,
+      "learning_rate": 2.6e-06,
+      "loss": 1.7272,
+      "mean_token_accuracy": 0.6625342659652234,
+      "num_tokens": 1340560.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.2964693307876587,
+      "epoch": 0.08,
+      "grad_norm": 5.0,
+      "learning_rate": 3.266666666666667e-06,
+      "loss": 1.5867,
+      "mean_token_accuracy": 0.6811951555311679,
+      "num_tokens": 1679432.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.2539724007248878,
+      "epoch": 0.096,
+      "grad_norm": 3.375,
+      "learning_rate": 3.9333333333333335e-06,
+      "loss": 1.3962,
+      "mean_token_accuracy": 0.7081016473472118,
+      "num_tokens": 2021570.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.2302619956433773,
+      "epoch": 0.112,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 1.2974,
+      "mean_token_accuracy": 0.7239668637514114,
+      "num_tokens": 2359297.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.169797332212329,
+      "epoch": 0.128,
+      "grad_norm": 1.875,
+      "learning_rate": 5.2666666666666665e-06,
+      "loss": 1.2306,
+      "mean_token_accuracy": 0.7328833002597094,
+      "num_tokens": 2683168.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.0165224198251963,
+      "epoch": 0.144,
+      "grad_norm": 1.59375,
+      "learning_rate": 5.933333333333335e-06,
+      "loss": 1.1129,
+      "mean_token_accuracy": 0.7561763934791088,
+      "num_tokens": 3019426.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.9457759071141482,
+      "epoch": 0.16,
+      "grad_norm": 1.546875,
+      "learning_rate": 6.600000000000001e-06,
+      "loss": 1.0448,
+      "mean_token_accuracy": 0.7662712432444095,
+      "num_tokens": 3354390.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 2.4009983863830566,
+      "eval_biology_loss": 3.090766429901123,
+      "eval_biology_mean_token_accuracy": 0.5075433547496796,
+      "eval_biology_num_tokens": 3354390.0,
+      "eval_biology_runtime": 38.801,
+      "eval_biology_samples_per_second": 12.886,
+      "eval_biology_steps_per_second": 3.222,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_chemistry_entropy": 1.1818295245170594,
+      "eval_chemistry_loss": 1.4003204107284546,
+      "eval_chemistry_mean_token_accuracy": 0.7158680257797241,
+      "eval_chemistry_num_tokens": 3354390.0,
+      "eval_chemistry_runtime": 48.2819,
+      "eval_chemistry_samples_per_second": 10.356,
+      "eval_chemistry_steps_per_second": 2.589,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_math_entropy": 0.8902422695159912,
+      "eval_math_loss": 1.2323389053344727,
+      "eval_math_mean_token_accuracy": 0.750218500137329,
+      "eval_math_num_tokens": 3354390.0,
+      "eval_math_runtime": 49.6484,
+      "eval_math_samples_per_second": 10.071,
+      "eval_math_steps_per_second": 2.518,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_physics_entropy": 0.9318590335845948,
+      "eval_physics_loss": 1.0569193363189697,
+      "eval_physics_mean_token_accuracy": 0.7675474114418029,
+      "eval_physics_num_tokens": 3354390.0,
+      "eval_physics_runtime": 57.1057,
+      "eval_physics_samples_per_second": 8.756,
+      "eval_physics_steps_per_second": 2.189,
+      "step": 100
+    },
+    {
+      "entropy": 0.9224597703665495,
+      "epoch": 0.176,
+      "grad_norm": 1.4921875,
+      "learning_rate": 7.266666666666668e-06,
+      "loss": 1.0192,
+      "mean_token_accuracy": 0.7696165222674608,
+      "num_tokens": 3684374.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.8823430232703686,
+      "epoch": 0.192,
+      "grad_norm": 1.125,
+      "learning_rate": 7.933333333333334e-06,
+      "loss": 0.9854,
+      "mean_token_accuracy": 0.7739298477768898,
+      "num_tokens": 4023915.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.8393984897062182,
+      "epoch": 0.208,
+      "grad_norm": 1.4375,
+      "learning_rate": 8.6e-06,
+      "loss": 0.9379,
+      "mean_token_accuracy": 0.7841779347509146,
+      "num_tokens": 4352045.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.8179828137159347,
+      "epoch": 0.224,
+      "grad_norm": 1.234375,
+      "learning_rate": 9.266666666666667e-06,
+      "loss": 0.9191,
+      "mean_token_accuracy": 0.7869349300861359,
+      "num_tokens": 4681371.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.805096386373043,
+      "epoch": 0.24,
+      "grad_norm": 1.09375,
+      "learning_rate": 9.933333333333334e-06,
+      "loss": 0.9012,
+      "mean_token_accuracy": 0.7888233289122581,
+      "num_tokens": 5021784.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.798224457167089,
+      "epoch": 0.256,
+      "grad_norm": 1.2109375,
+      "learning_rate": 1.0600000000000002e-05,
+      "loss": 0.8922,
+      "mean_token_accuracy": 0.7903898701071739,
+      "num_tokens": 5367308.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.805234762467444,
+      "epoch": 0.272,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.1266666666666668e-05,
+      "loss": 0.8993,
+      "mean_token_accuracy": 0.7877147275954485,
+      "num_tokens": 5699101.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.7946224914863705,
+      "epoch": 0.288,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.1933333333333335e-05,
+      "loss": 0.8976,
+      "mean_token_accuracy": 0.7911442808806897,
+      "num_tokens": 6022837.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.7837886592373252,
+      "epoch": 0.304,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.2600000000000001e-05,
+      "loss": 0.8805,
+      "mean_token_accuracy": 0.7934082143008709,
+      "num_tokens": 6343826.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.778077344968915,
+      "epoch": 0.32,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.3266666666666668e-05,
+      "loss": 0.8781,
+      "mean_token_accuracy": 0.7940127164125442,
+      "num_tokens": 6670164.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 2.1669651889801025,
+      "eval_biology_loss": 2.686201810836792,
+      "eval_biology_mean_token_accuracy": 0.5460970797538758,
+      "eval_biology_num_tokens": 6670164.0,
+      "eval_biology_runtime": 38.7901,
+      "eval_biology_samples_per_second": 12.89,
+      "eval_biology_steps_per_second": 3.222,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_chemistry_entropy": 0.9981456413269043,
+      "eval_chemistry_loss": 1.1842883825302124,
+      "eval_chemistry_mean_token_accuracy": 0.7467401041984558,
+      "eval_chemistry_num_tokens": 6670164.0,
+      "eval_chemistry_runtime": 48.3154,
+      "eval_chemistry_samples_per_second": 10.349,
+      "eval_chemistry_steps_per_second": 2.587,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_math_entropy": 0.7862319254875183,
+      "eval_math_loss": 1.0977362394332886,
+      "eval_math_mean_token_accuracy": 0.7683374147415161,
+      "eval_math_num_tokens": 6670164.0,
+      "eval_math_runtime": 49.687,
+      "eval_math_samples_per_second": 10.063,
+      "eval_math_steps_per_second": 2.516,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_physics_entropy": 0.7679825134277344,
+      "eval_physics_loss": 0.886142373085022,
+      "eval_physics_mean_token_accuracy": 0.7957743234634399,
+      "eval_physics_num_tokens": 6670164.0,
+      "eval_physics_runtime": 57.1202,
+      "eval_physics_samples_per_second": 8.753,
+      "eval_physics_steps_per_second": 2.188,
+      "step": 200
+    },
+    {
+      "entropy": 0.7617403082549572,
+      "epoch": 0.336,
+      "grad_norm": 1.03125,
+      "learning_rate": 1.3933333333333334e-05,
+      "loss": 0.8531,
+      "mean_token_accuracy": 0.7977134332060813,
+      "num_tokens": 7003404.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.737130863033235,
+      "epoch": 0.352,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.46e-05,
+      "loss": 0.8339,
+      "mean_token_accuracy": 0.8024804938584567,
+      "num_tokens": 7342593.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.7299130430445075,
+      "epoch": 0.368,
+      "grad_norm": 1.015625,
+      "learning_rate": 1.5266666666666667e-05,
+      "loss": 0.8252,
+      "mean_token_accuracy": 0.805871631577611,
+      "num_tokens": 7675569.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.7597114410251379,
+      "epoch": 0.384,
+      "grad_norm": 1.171875,
+      "learning_rate": 1.5933333333333336e-05,
+      "loss": 0.8558,
+      "mean_token_accuracy": 0.7943590730428696,
+      "num_tokens": 8012218.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.7247350050136447,
+      "epoch": 0.4,
+      "grad_norm": 1.125,
+      "learning_rate": 1.66e-05,
+      "loss": 0.8193,
+      "mean_token_accuracy": 0.8040182612836361,
+      "num_tokens": 8350120.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.7144488081336021,
+      "epoch": 0.416,
+      "grad_norm": 1.0859375,
+      "learning_rate": 1.726666666666667e-05,
+      "loss": 0.7998,
+      "mean_token_accuracy": 0.8067968346178531,
+      "num_tokens": 8688773.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.7226355630904436,
+      "epoch": 0.432,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.7933333333333333e-05,
+      "loss": 0.8125,
+      "mean_token_accuracy": 0.8042349684983492,
+      "num_tokens": 9022094.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.6943683221936225,
+      "epoch": 0.448,
+      "grad_norm": 1.03125,
+      "learning_rate": 1.86e-05,
+      "loss": 0.7924,
+      "mean_token_accuracy": 0.8109067149460316,
+      "num_tokens": 9360608.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.6898288525640964,
+      "epoch": 0.464,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.926666666666667e-05,
+      "loss": 0.7814,
+      "mean_token_accuracy": 0.8110810052603483,
+      "num_tokens": 9688896.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.6964065950363875,
+      "epoch": 0.48,
+      "grad_norm": 1.125,
+      "learning_rate": 1.9933333333333334e-05,
+      "loss": 0.7896,
+      "mean_token_accuracy": 0.8094950247555971,
+      "num_tokens": 10025864.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 2.1021561794281007,
+      "eval_biology_loss": 2.546415090560913,
+      "eval_biology_mean_token_accuracy": 0.5616441056728363,
+      "eval_biology_num_tokens": 10025864.0,
+      "eval_biology_runtime": 38.8255,
+      "eval_biology_samples_per_second": 12.878,
+      "eval_biology_steps_per_second": 3.22,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_chemistry_entropy": 0.9578249802589417,
+      "eval_chemistry_loss": 1.1104036569595337,
+      "eval_chemistry_mean_token_accuracy": 0.7582236580848694,
+      "eval_chemistry_num_tokens": 10025864.0,
+      "eval_chemistry_runtime": 48.5149,
+      "eval_chemistry_samples_per_second": 10.306,
+      "eval_chemistry_steps_per_second": 2.577,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_math_entropy": 0.7676350421905518,
+      "eval_math_loss": 1.0549193620681763,
+      "eval_math_mean_token_accuracy": 0.7745008988380432,
+      "eval_math_num_tokens": 10025864.0,
+      "eval_math_runtime": 49.8707,
+      "eval_math_samples_per_second": 10.026,
+      "eval_math_steps_per_second": 2.506,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_physics_entropy": 0.7307665984630585,
+      "eval_physics_loss": 0.8242064118385315,
+      "eval_physics_mean_token_accuracy": 0.8064617681503295,
+      "eval_physics_num_tokens": 10025864.0,
+      "eval_physics_runtime": 57.1693,
+      "eval_physics_samples_per_second": 8.746,
+      "eval_physics_steps_per_second": 2.186,
+      "step": 300
+    },
+    {
+      "entropy": 0.7168749757111073,
+      "epoch": 0.496,
+      "grad_norm": 0.9921875,
+      "learning_rate": 1.9933333333333334e-05,
+      "loss": 0.8032,
+      "mean_token_accuracy": 0.8051175128668546,
+      "num_tokens": 10362573.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.6993480321019888,
+      "epoch": 0.512,
+      "grad_norm": 1.15625,
+      "learning_rate": 1.985925925925926e-05,
+      "loss": 0.7903,
+      "mean_token_accuracy": 0.8099954195320607,
+      "num_tokens": 10694440.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.7138712629675865,
+      "epoch": 0.528,
+      "grad_norm": 1.046875,
+      "learning_rate": 1.9785185185185187e-05,
+      "loss": 0.8101,
+      "mean_token_accuracy": 0.805506169050932,
+      "num_tokens": 11022357.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.6928766580298543,
+      "epoch": 0.544,
+      "grad_norm": 1.0546875,
+      "learning_rate": 1.971111111111111e-05,
+      "loss": 0.7916,
+      "mean_token_accuracy": 0.8087165944278241,
+      "num_tokens": 11345911.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.682861409150064,
+      "epoch": 0.56,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.963703703703704e-05,
+      "loss": 0.7735,
+      "mean_token_accuracy": 0.8126827124506235,
+      "num_tokens": 11687225.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.6824749782681465,
+      "epoch": 0.576,
+      "grad_norm": 1.015625,
+      "learning_rate": 1.9562962962962964e-05,
+      "loss": 0.7723,
+      "mean_token_accuracy": 0.8146394658833742,
+      "num_tokens": 12015720.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.6805058639496565,
+      "epoch": 0.592,
+      "grad_norm": 1.2109375,
+      "learning_rate": 1.948888888888889e-05,
+      "loss": 0.776,
+      "mean_token_accuracy": 0.811661035567522,
+      "num_tokens": 12343393.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.6700849516317249,
+      "epoch": 0.608,
+      "grad_norm": 0.9921875,
+      "learning_rate": 1.9414814814814817e-05,
+      "loss": 0.7658,
+      "mean_token_accuracy": 0.8141583666205406,
+      "num_tokens": 12671474.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.6893410481512546,
+      "epoch": 0.624,
+      "grad_norm": 0.98046875,
+      "learning_rate": 1.9340740740740743e-05,
+      "loss": 0.7782,
+      "mean_token_accuracy": 0.8111887093633413,
+      "num_tokens": 12996077.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.6852772971615195,
+      "epoch": 0.64,
+      "grad_norm": 0.9609375,
+      "learning_rate": 1.926666666666667e-05,
+      "loss": 0.7799,
+      "mean_token_accuracy": 0.8112590182572603,
+      "num_tokens": 13325291.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 2.0124985208511355,
+      "eval_biology_loss": 2.4155702590942383,
+      "eval_biology_mean_token_accuracy": 0.5735694291591644,
+      "eval_biology_num_tokens": 13325291.0,
+      "eval_biology_runtime": 38.6667,
+      "eval_biology_samples_per_second": 12.931,
+      "eval_biology_steps_per_second": 3.233,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_chemistry_entropy": 0.8911373369693756,
+      "eval_chemistry_loss": 1.0692609548568726,
+      "eval_chemistry_mean_token_accuracy": 0.7647108516693115,
+      "eval_chemistry_num_tokens": 13325291.0,
+      "eval_chemistry_runtime": 48.2089,
+      "eval_chemistry_samples_per_second": 10.372,
+      "eval_chemistry_steps_per_second": 2.593,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_math_entropy": 0.7330298454761505,
+      "eval_math_loss": 1.0336663722991943,
+      "eval_math_mean_token_accuracy": 0.7786638278961182,
+      "eval_math_num_tokens": 13325291.0,
+      "eval_math_runtime": 49.7905,
+      "eval_math_samples_per_second": 10.042,
+      "eval_math_steps_per_second": 2.511,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_physics_entropy": 0.681670261144638,
+      "eval_physics_loss": 0.7883204817771912,
+      "eval_physics_mean_token_accuracy": 0.8129381031990052,
+      "eval_physics_num_tokens": 13325291.0,
+      "eval_physics_runtime": 57.2563,
+      "eval_physics_samples_per_second": 8.733,
+      "eval_physics_steps_per_second": 2.183,
+      "step": 400
+    },
+    {
+      "entropy": 0.6723099524155259,
+      "epoch": 0.656,
+      "grad_norm": 0.953125,
+      "learning_rate": 1.9192592592592593e-05,
+      "loss": 0.757,
+      "mean_token_accuracy": 0.8151150114834309,
+      "num_tokens": 13671434.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.6621538577601314,
+      "epoch": 0.672,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.911851851851852e-05,
+      "loss": 0.7563,
+      "mean_token_accuracy": 0.8172798678278923,
+      "num_tokens": 13993668.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.6673029117286206,
+      "epoch": 0.688,
+      "grad_norm": 0.921875,
+      "learning_rate": 1.9044444444444446e-05,
+      "loss": 0.7529,
+      "mean_token_accuracy": 0.8164214458316564,
+      "num_tokens": 14334907.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.6810694945976138,
+      "epoch": 0.704,
+      "grad_norm": 1.046875,
+      "learning_rate": 1.8970370370370372e-05,
+      "loss": 0.7695,
+      "mean_token_accuracy": 0.8127802673727273,
+      "num_tokens": 14670639.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.6582699194550514,
+      "epoch": 0.72,
+      "grad_norm": 1.0,
+      "learning_rate": 1.8896296296296295e-05,
+      "loss": 0.7544,
+      "mean_token_accuracy": 0.8171826928853989,
+      "num_tokens": 14987995.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.6564427128061652,
+      "epoch": 0.736,
+      "grad_norm": 1.078125,
+      "learning_rate": 1.8822222222222225e-05,
+      "loss": 0.7446,
+      "mean_token_accuracy": 0.8173684533685446,
+      "num_tokens": 15324659.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.6495672106742859,
+      "epoch": 0.752,
+      "grad_norm": 0.94921875,
+      "learning_rate": 1.874814814814815e-05,
+      "loss": 0.7416,
+      "mean_token_accuracy": 0.8195757914334536,
+      "num_tokens": 15657342.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.6738376861438156,
+      "epoch": 0.768,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.8674074074074075e-05,
+      "loss": 0.7642,
+      "mean_token_accuracy": 0.8124835971742869,
+      "num_tokens": 15988993.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.6495399951934815,
+      "epoch": 0.784,
+      "grad_norm": 0.9609375,
+      "learning_rate": 1.86e-05,
+      "loss": 0.7324,
+      "mean_token_accuracy": 0.8208612345159054,
+      "num_tokens": 16333857.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.6567428983747959,
+      "epoch": 0.8,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.8525925925925928e-05,
+      "loss": 0.7488,
+      "mean_token_accuracy": 0.817277068644762,
+      "num_tokens": 16664690.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 2.039887357711792,
+      "eval_biology_loss": 2.4157471656799316,
+      "eval_biology_mean_token_accuracy": 0.5770127189159393,
+      "eval_biology_num_tokens": 16664690.0,
+      "eval_biology_runtime": 38.6815,
+      "eval_biology_samples_per_second": 12.926,
+      "eval_biology_steps_per_second": 3.232,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_chemistry_entropy": 0.8955893061161041,
+      "eval_chemistry_loss": 1.048737645149231,
+      "eval_chemistry_mean_token_accuracy": 0.768093888759613,
+      "eval_chemistry_num_tokens": 16664690.0,
+      "eval_chemistry_runtime": 48.42,
+      "eval_chemistry_samples_per_second": 10.326,
+      "eval_chemistry_steps_per_second": 2.582,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_math_entropy": 0.7318526220321655,
+      "eval_math_loss": 1.0199034214019775,
+      "eval_math_mean_token_accuracy": 0.7799897599220276,
+      "eval_math_num_tokens": 16664690.0,
+      "eval_math_runtime": 49.8882,
+      "eval_math_samples_per_second": 10.022,
+      "eval_math_steps_per_second": 2.506,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_physics_entropy": 0.6712099099159241,
+      "eval_physics_loss": 0.7689476609230042,
+      "eval_physics_mean_token_accuracy": 0.8163677668571472,
+      "eval_physics_num_tokens": 16664690.0,
+      "eval_physics_runtime": 57.4517,
+      "eval_physics_samples_per_second": 8.703,
+      "eval_physics_steps_per_second": 2.176,
+      "step": 500
+    },
+    {
+      "entropy": 0.6573688389733434,
+      "epoch": 0.816,
+      "grad_norm": 0.96875,
+      "learning_rate": 1.8451851851851855e-05,
+      "loss": 0.7477,
+      "mean_token_accuracy": 0.8164514761418105,
+      "num_tokens": 16989307.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.6648645078763366,
+      "epoch": 0.832,
+      "grad_norm": 0.9609375,
+      "learning_rate": 1.8377777777777778e-05,
+      "loss": 0.749,
+      "mean_token_accuracy": 0.8170726090669632,
+      "num_tokens": 17317524.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.6526606786996126,
+      "epoch": 0.848,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.8303703703703704e-05,
+      "loss": 0.7388,
+      "mean_token_accuracy": 0.8195879191160202,
+      "num_tokens": 17650424.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.6565553491935134,
+      "epoch": 0.864,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.822962962962963e-05,
+      "loss": 0.7464,
+      "mean_token_accuracy": 0.8177060768008232,
+      "num_tokens": 17985547.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.6623865978792309,
+      "epoch": 0.88,
+      "grad_norm": 0.9453125,
+      "learning_rate": 1.8155555555555557e-05,
+      "loss": 0.7495,
+      "mean_token_accuracy": 0.8162953305989504,
+      "num_tokens": 18315392.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.6179373754188419,
+      "epoch": 0.896,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.8081481481481484e-05,
+      "loss": 0.7023,
+      "mean_token_accuracy": 0.8264330130070447,
+      "num_tokens": 18667124.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.6555242039263248,
+      "epoch": 0.912,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.800740740740741e-05,
+      "loss": 0.7393,
+      "mean_token_accuracy": 0.8174499638378621,
+      "num_tokens": 19002186.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.6485247412696481,
+      "epoch": 0.928,
+      "grad_norm": 0.98046875,
+      "learning_rate": 1.7933333333333333e-05,
+      "loss": 0.7363,
+      "mean_token_accuracy": 0.8195535041391849,
+      "num_tokens": 19335013.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.63772834520787,
+      "epoch": 0.944,
+      "grad_norm": 0.984375,
+      "learning_rate": 1.785925925925926e-05,
+      "loss": 0.7269,
+      "mean_token_accuracy": 0.821834321692586,
+      "num_tokens": 19662733.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.6372630735859275,
+      "epoch": 0.96,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.7785185185185186e-05,
+      "loss": 0.7318,
+      "mean_token_accuracy": 0.8204116970300674,
+      "num_tokens": 19996789.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_biology_entropy": 2.0030080833435058,
+      "eval_biology_loss": 2.361283302307129,
+      "eval_biology_mean_token_accuracy": 0.5793048655986786,
+      "eval_biology_num_tokens": 19996789.0,
+      "eval_biology_runtime": 38.6981,
+      "eval_biology_samples_per_second": 12.921,
+      "eval_biology_steps_per_second": 3.23,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_chemistry_entropy": 0.8840513801574708,
+      "eval_chemistry_loss": 1.0323643684387207,
+      "eval_chemistry_mean_token_accuracy": 0.7703397722244263,
+      "eval_chemistry_num_tokens": 19996789.0,
+      "eval_chemistry_runtime": 48.2039,
+      "eval_chemistry_samples_per_second": 10.373,
+      "eval_chemistry_steps_per_second": 2.593,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_math_entropy": 0.7270034260749817,
+      "eval_math_loss": 1.009481430053711,
+      "eval_math_mean_token_accuracy": 0.7820751585960388,
+      "eval_math_num_tokens": 19996789.0,
+      "eval_math_runtime": 49.571,
+      "eval_math_samples_per_second": 10.087,
+      "eval_math_steps_per_second": 2.522,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_physics_entropy": 0.6698693735599518,
+      "eval_physics_loss": 0.7564050555229187,
+      "eval_physics_mean_token_accuracy": 0.8185040464401245,
+      "eval_physics_num_tokens": 19996789.0,
+      "eval_physics_runtime": 56.9517,
+      "eval_physics_samples_per_second": 8.779,
+      "eval_physics_steps_per_second": 2.195,
+      "step": 600
+    },
+    {
+      "entropy": 0.6384279150515795,
+      "epoch": 0.976,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.7711111111111113e-05,
+      "loss": 0.7283,
+      "mean_token_accuracy": 0.82187210470438,
+      "num_tokens": 20332636.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.6450063675642014,
+      "epoch": 0.992,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.763703703703704e-05,
+      "loss": 0.7272,
+      "mean_token_accuracy": 0.8209770727902651,
+      "num_tokens": 20662209.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.6167608626186848,
+      "epoch": 1.008,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.7562962962962962e-05,
+      "loss": 0.7008,
+      "mean_token_accuracy": 0.825862829759717,
+      "num_tokens": 21005731.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.6117491278797388,
+      "epoch": 1.024,
+      "grad_norm": 0.93359375,
+      "learning_rate": 1.7488888888888892e-05,
+      "loss": 0.7058,
+      "mean_token_accuracy": 0.8266724064946175,
+      "num_tokens": 21335407.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.6068670526146889,
+      "epoch": 1.04,
+      "grad_norm": 0.97265625,
+      "learning_rate": 1.7414814814814815e-05,
+      "loss": 0.6989,
+      "mean_token_accuracy": 0.8283324401825667,
+      "num_tokens": 21668490.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.601451874896884,
+      "epoch": 1.056,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.7340740740740742e-05,
+      "loss": 0.6864,
+      "mean_token_accuracy": 0.829675118252635,
+      "num_tokens": 22006039.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.6137524953112006,
+      "epoch": 1.072,
+      "grad_norm": 0.94921875,
+      "learning_rate": 1.726666666666667e-05,
+      "loss": 0.7008,
+      "mean_token_accuracy": 0.8262542523443699,
+      "num_tokens": 22348206.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.6093455260619521,
+      "epoch": 1.088,
+      "grad_norm": 0.96875,
+      "learning_rate": 1.7192592592592595e-05,
+      "loss": 0.6956,
+      "mean_token_accuracy": 0.8260304640978575,
+      "num_tokens": 22676163.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.6026462253183127,
+      "epoch": 1.104,
+      "grad_norm": 0.87109375,
+      "learning_rate": 1.711851851851852e-05,
+      "loss": 0.6967,
+      "mean_token_accuracy": 0.8273924000561237,
+      "num_tokens": 23009279.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.6122452523559332,
+      "epoch": 1.12,
+      "grad_norm": 0.9453125,
+      "learning_rate": 1.7044444444444445e-05,
+      "loss": 0.7,
+      "mean_token_accuracy": 0.8257350366562605,
+      "num_tokens": 23346828.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_biology_entropy": 1.8616097135543823,
+      "eval_biology_loss": 2.4014246463775635,
+      "eval_biology_mean_token_accuracy": 0.578062358379364,
+      "eval_biology_num_tokens": 23346828.0,
+      "eval_biology_runtime": 38.9662,
+      "eval_biology_samples_per_second": 12.832,
+      "eval_biology_steps_per_second": 3.208,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_chemistry_entropy": 0.8116880433559418,
+      "eval_chemistry_loss": 1.0317949056625366,
+      "eval_chemistry_mean_token_accuracy": 0.7715164208412171,
+      "eval_chemistry_num_tokens": 23346828.0,
+      "eval_chemistry_runtime": 48.496,
+      "eval_chemistry_samples_per_second": 10.31,
+      "eval_chemistry_steps_per_second": 2.578,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_math_entropy": 0.6895896532535553,
+      "eval_math_loss": 1.0153776407241821,
+      "eval_math_mean_token_accuracy": 0.7820956745147705,
+      "eval_math_num_tokens": 23346828.0,
+      "eval_math_runtime": 49.7815,
+      "eval_math_samples_per_second": 10.044,
+      "eval_math_steps_per_second": 2.511,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_physics_entropy": 0.6192426791191101,
+      "eval_physics_loss": 0.7497905492782593,
+      "eval_physics_mean_token_accuracy": 0.8200045080184937,
+      "eval_physics_num_tokens": 23346828.0,
+      "eval_physics_runtime": 57.1287,
+      "eval_physics_samples_per_second": 8.752,
+      "eval_physics_steps_per_second": 2.188,
+      "step": 700
+    },
+    {
+      "entropy": 0.588855667039752,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 0.9375,
+      "learning_rate": 1.697037037037037e-05,
+      "loss": 0.6772,
+      "mean_token_accuracy": 0.8326364874839782,
+      "num_tokens": 23689161.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.6099351227283478,
+      "epoch": 1.152,
+      "grad_norm": 1.0390625,
+      "learning_rate": 1.6896296296296298e-05,
+      "loss": 0.7039,
+      "mean_token_accuracy": 0.8282962709665298,
+      "num_tokens": 24016439.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.6140455640852451,
+      "epoch": 1.168,
+      "grad_norm": 0.94921875,
+      "learning_rate": 1.6822222222222224e-05,
+      "loss": 0.7027,
+      "mean_token_accuracy": 0.8253309100866317,
+      "num_tokens": 24344431.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.6041248327121138,
+      "epoch": 1.184,
+      "grad_norm": 1.09375,
+      "learning_rate": 1.6748148148148147e-05,
+      "loss": 0.6955,
+      "mean_token_accuracy": 0.8286070462316275,
+      "num_tokens": 24668087.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.5895567566156388,
+      "epoch": 1.2,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.6674074074074077e-05,
+      "loss": 0.6822,
+      "mean_token_accuracy": 0.8305550657212735,
+      "num_tokens": 25000629.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.6073450578376651,
+      "epoch": 1.216,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.66e-05,
+      "loss": 0.7023,
+      "mean_token_accuracy": 0.8261869914829731,
+      "num_tokens": 25324324.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.6059388216584921,
+      "epoch": 1.232,
+      "grad_norm": 0.9453125,
+      "learning_rate": 1.6525925925925927e-05,
+      "loss": 0.6863,
+      "mean_token_accuracy": 0.8298216536641121,
+      "num_tokens": 25658795.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.5957553267478943,
+      "epoch": 1.248,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.6451851851851853e-05,
+      "loss": 0.6864,
+      "mean_token_accuracy": 0.830686765909195,
+      "num_tokens": 25993177.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.5959657493978738,
+      "epoch": 1.264,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.637777777777778e-05,
+      "loss": 0.6826,
+      "mean_token_accuracy": 0.8305899318307638,
+      "num_tokens": 26329726.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.6204709148034453,
+      "epoch": 1.28,
+      "grad_norm": 1.0078125,
+      "learning_rate": 1.6303703703703706e-05,
+      "loss": 0.7071,
+      "mean_token_accuracy": 0.8238335218280554,
+      "num_tokens": 26664093.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_biology_entropy": 1.7922346639633178,
+      "eval_biology_loss": 2.4233291149139404,
+      "eval_biology_mean_token_accuracy": 0.5778127768039704,
+      "eval_biology_num_tokens": 26664093.0,
+      "eval_biology_runtime": 39.1057,
+      "eval_biology_samples_per_second": 12.786,
+      "eval_biology_steps_per_second": 3.196,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_chemistry_entropy": 0.7962674243450165,
+      "eval_chemistry_loss": 1.0286256074905396,
+      "eval_chemistry_mean_token_accuracy": 0.7727626013755798,
+      "eval_chemistry_num_tokens": 26664093.0,
+      "eval_chemistry_runtime": 48.3131,
+      "eval_chemistry_samples_per_second": 10.349,
+      "eval_chemistry_steps_per_second": 2.587,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_math_entropy": 0.6702238636016846,
+      "eval_math_loss": 1.0157709121704102,
+      "eval_math_mean_token_accuracy": 0.7829344019889831,
+      "eval_math_num_tokens": 26664093.0,
+      "eval_math_runtime": 49.7647,
+      "eval_math_samples_per_second": 10.047,
+      "eval_math_steps_per_second": 2.512,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_physics_entropy": 0.6072953283786774,
+      "eval_physics_loss": 0.7434503436088562,
+      "eval_physics_mean_token_accuracy": 0.8212288799285888,
+      "eval_physics_num_tokens": 26664093.0,
+      "eval_physics_runtime": 57.1238,
+      "eval_physics_samples_per_second": 8.753,
+      "eval_physics_steps_per_second": 2.188,
+      "step": 800
+    },
+    {
+      "entropy": 0.593550406768918,
+      "epoch": 1.296,
+      "grad_norm": 0.984375,
+      "learning_rate": 1.622962962962963e-05,
+      "loss": 0.6825,
+      "mean_token_accuracy": 0.8320997886359691,
+      "num_tokens": 26994877.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.6156477816402912,
+      "epoch": 1.312,
+      "grad_norm": 1.046875,
+      "learning_rate": 1.6155555555555556e-05,
+      "loss": 0.703,
+      "mean_token_accuracy": 0.8259445391595364,
+      "num_tokens": 27328198.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.5969830378890038,
+      "epoch": 1.328,
+      "grad_norm": 1.0078125,
+      "learning_rate": 1.6081481481481482e-05,
+      "loss": 0.6901,
+      "mean_token_accuracy": 0.8290002550929785,
+      "num_tokens": 27658818.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.5997996777296066,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 1.1015625,
+      "learning_rate": 1.600740740740741e-05,
+      "loss": 0.6904,
+      "mean_token_accuracy": 0.8284455709159374,
+      "num_tokens": 27991108.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.6052390130236744,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 0.94921875,
+      "learning_rate": 1.5933333333333336e-05,
+      "loss": 0.6852,
+      "mean_token_accuracy": 0.8274706263095141,
+      "num_tokens": 28327775.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.6018602728843689,
+      "epoch": 1.376,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.5859259259259262e-05,
+      "loss": 0.6827,
+      "mean_token_accuracy": 0.8304568257182836,
+      "num_tokens": 28666116.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.5851238770410419,
+      "epoch": 1.392,
+      "grad_norm": 0.98828125,
+      "learning_rate": 1.5785185185185185e-05,
+      "loss": 0.6746,
+      "mean_token_accuracy": 0.8333170894533396,
+      "num_tokens": 29004293.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.58795285820961,
+      "epoch": 1.408,
+      "grad_norm": 0.96875,
+      "learning_rate": 1.571111111111111e-05,
+      "loss": 0.6857,
+      "mean_token_accuracy": 0.8305332105606794,
+      "num_tokens": 29330131.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.6189510561525822,
+      "epoch": 1.424,
+      "grad_norm": 0.90234375,
+      "learning_rate": 1.5637037037037038e-05,
+      "loss": 0.7072,
+      "mean_token_accuracy": 0.8251793116331101,
+      "num_tokens": 29661850.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.5798567572608591,
+      "epoch": 1.44,
+      "grad_norm": 0.90625,
+      "learning_rate": 1.5562962962962965e-05,
+      "loss": 0.6695,
+      "mean_token_accuracy": 0.8342564977705479,
+      "num_tokens": 29999392.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_biology_entropy": 1.8147089042663573,
+      "eval_biology_loss": 2.417410135269165,
+      "eval_biology_mean_token_accuracy": 0.5787473826408386,
+      "eval_biology_num_tokens": 29999392.0,
+      "eval_biology_runtime": 38.9072,
+      "eval_biology_samples_per_second": 12.851,
+      "eval_biology_steps_per_second": 3.213,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_chemistry_entropy": 0.7944381227493286,
+      "eval_chemistry_loss": 1.0228049755096436,
+      "eval_chemistry_mean_token_accuracy": 0.7732948322296143,
+      "eval_chemistry_num_tokens": 29999392.0,
+      "eval_chemistry_runtime": 48.4429,
+      "eval_chemistry_samples_per_second": 10.321,
+      "eval_chemistry_steps_per_second": 2.58,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_math_entropy": 0.6757729785442352,
+      "eval_math_loss": 1.0138108730316162,
+      "eval_math_mean_token_accuracy": 0.7829539861679077,
+      "eval_math_num_tokens": 29999392.0,
+      "eval_math_runtime": 49.813,
+      "eval_math_samples_per_second": 10.038,
+      "eval_math_steps_per_second": 2.509,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_physics_entropy": 0.607404308795929,
+      "eval_physics_loss": 0.737089991569519,
+      "eval_physics_mean_token_accuracy": 0.8226114134788514,
+      "eval_physics_num_tokens": 29999392.0,
+      "eval_physics_runtime": 57.3115,
+      "eval_physics_samples_per_second": 8.724,
+      "eval_physics_steps_per_second": 2.181,
+      "step": 900
+    },
+    {
+      "entropy": 0.5827234297990799,
+      "epoch": 1.456,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.548888888888889e-05,
+      "loss": 0.6728,
+      "mean_token_accuracy": 0.8342432040721178,
+      "num_tokens": 30341004.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.613773494027555,
+      "epoch": 1.472,
+      "grad_norm": 1.015625,
+      "learning_rate": 1.5414814814814814e-05,
+      "loss": 0.7035,
+      "mean_token_accuracy": 0.8268327709287405,
+      "num_tokens": 30667689.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.5988023646175862,
+      "epoch": 1.488,
+      "grad_norm": 0.93359375,
+      "learning_rate": 1.5340740740740744e-05,
+      "loss": 0.6897,
+      "mean_token_accuracy": 0.8293631616979837,
+      "num_tokens": 30999697.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.5896453000605106,
+      "epoch": 1.504,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.5266666666666667e-05,
+      "loss": 0.6728,
+      "mean_token_accuracy": 0.8324251122772693,
+      "num_tokens": 31332775.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.605005569756031,
+      "epoch": 1.52,
+      "grad_norm": 1.0390625,
+      "learning_rate": 1.5192592592592594e-05,
+      "loss": 0.6942,
+      "mean_token_accuracy": 0.8275358382612467,
+      "num_tokens": 31666521.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.5999037871137262,
+      "epoch": 1.536,
+      "grad_norm": 1.0390625,
+      "learning_rate": 1.5118518518518519e-05,
+      "loss": 0.6904,
+      "mean_token_accuracy": 0.8276842717081309,
+      "num_tokens": 31998232.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.5867987772449851,
+      "epoch": 1.552,
+      "grad_norm": 1.0703125,
+      "learning_rate": 1.5044444444444445e-05,
+      "loss": 0.6754,
+      "mean_token_accuracy": 0.8337812848389149,
+      "num_tokens": 32328085.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.5942975046113134,
+      "epoch": 1.568,
+      "grad_norm": 1.0,
+      "learning_rate": 1.497037037037037e-05,
+      "loss": 0.6807,
+      "mean_token_accuracy": 0.831771444156766,
+      "num_tokens": 32656603.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.6016290852800011,
+      "epoch": 1.584,
+      "grad_norm": 0.98828125,
+      "learning_rate": 1.4896296296296298e-05,
+      "loss": 0.6947,
+      "mean_token_accuracy": 0.8264373868703843,
+      "num_tokens": 32983769.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.5999371835961937,
+      "epoch": 1.6,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.4822222222222225e-05,
+      "loss": 0.6882,
+      "mean_token_accuracy": 0.8293469067662954,
+      "num_tokens": 33309282.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_biology_entropy": 1.788435049057007,
+      "eval_biology_loss": 2.369239330291748,
+      "eval_biology_mean_token_accuracy": 0.582524644613266,
+      "eval_biology_num_tokens": 33309282.0,
+      "eval_biology_runtime": 38.8968,
+      "eval_biology_samples_per_second": 12.855,
+      "eval_biology_steps_per_second": 3.214,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_chemistry_entropy": 0.7873289968967437,
+      "eval_chemistry_loss": 1.0177081823349,
+      "eval_chemistry_mean_token_accuracy": 0.774118812084198,
+      "eval_chemistry_num_tokens": 33309282.0,
+      "eval_chemistry_runtime": 48.3649,
+      "eval_chemistry_samples_per_second": 10.338,
+      "eval_chemistry_steps_per_second": 2.585,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_math_entropy": 0.6668027784824372,
+      "eval_math_loss": 1.0126802921295166,
+      "eval_math_mean_token_accuracy": 0.7838437123298645,
+      "eval_math_num_tokens": 33309282.0,
+      "eval_math_runtime": 49.8108,
+      "eval_math_samples_per_second": 10.038,
+      "eval_math_steps_per_second": 2.509,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_physics_entropy": 0.5990195829868317,
+      "eval_physics_loss": 0.7323749661445618,
+      "eval_physics_mean_token_accuracy": 0.8238483490943909,
+      "eval_physics_num_tokens": 33309282.0,
+      "eval_physics_runtime": 57.3705,
+      "eval_physics_samples_per_second": 8.715,
+      "eval_physics_steps_per_second": 2.179,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.473218768217043e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:206bae3974561467a723a0a2f09ffcd0596b30445368e1e721e80cf45c549cd9
+size 6417