Upload folder using huggingface_hub

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-2000/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-2000/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-2000/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-2000/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-2000/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-2000/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8eb73b3c715e194172ec2ccf840c7477e7fd72b80cb017111396a2b0caffdf4a
+size 4962221464

checkpoint-2000/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08a148d7aa98f0874c2c7287b5fa817b22072dfeac226fbdddb502e444d795e3
+size 3670322200

checkpoint-2000/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1087c756a431865fa3f5e5ab3e8fe78358ab260116d062d19237a66898c8a0d
+size 1585596266

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14c74faa02adea60668114d02c466dcfd82efc22d7bc2201f6e6de0bf552fbda
+size 14645

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e95ad2f3950cee1b83b106cf88750c8604566acab211d78973b3b5c4917b1a30
+size 1465

checkpoint-2000/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-2000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-2000/trainer_state.json

@@ -0,0 +1,2474 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 6.3904,
+  "eval_steps": 100,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.9365717947483063,
+      "epoch": 0.032,
+      "grad_norm": 245.0,
+      "learning_rate": 1.8e-07,
+      "loss": 6.202,
+      "mean_token_accuracy": 0.2543207791633904,
+      "num_tokens": 144208.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.9173382006585598,
+      "epoch": 0.064,
+      "grad_norm": 230.0,
+      "learning_rate": 3.8e-07,
+      "loss": 6.1081,
+      "mean_token_accuracy": 0.25344993276521566,
+      "num_tokens": 285280.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.9177092142403125,
+      "epoch": 0.096,
+      "grad_norm": 246.0,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 5.9977,
+      "mean_token_accuracy": 0.25974713498726487,
+      "num_tokens": 430075.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.9969916075468064,
+      "epoch": 0.128,
+      "grad_norm": 218.0,
+      "learning_rate": 7.8e-07,
+      "loss": 5.7736,
+      "mean_token_accuracy": 0.25647984803654256,
+      "num_tokens": 574626.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.0409375332295894,
+      "epoch": 0.16,
+      "grad_norm": 198.0,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 5.4968,
+      "mean_token_accuracy": 0.25714964754879477,
+      "num_tokens": 721064.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.080457562953234,
+      "epoch": 0.192,
+      "grad_norm": 141.0,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 4.9089,
+      "mean_token_accuracy": 0.2747303694486618,
+      "num_tokens": 865275.0,
+      "step": 60
+    },
+    {
+      "entropy": 2.1661732770502566,
+      "epoch": 0.224,
+      "grad_norm": 95.0,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 4.2411,
+      "mean_token_accuracy": 0.3079248811118305,
+      "num_tokens": 1005142.0,
+      "step": 70
+    },
+    {
+      "entropy": 2.440109082311392,
+      "epoch": 0.256,
+      "grad_norm": 65.5,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 3.7265,
+      "mean_token_accuracy": 0.32527065742760897,
+      "num_tokens": 1147262.0,
+      "step": 80
+    },
+    {
+      "entropy": 2.6965860120952128,
+      "epoch": 0.288,
+      "grad_norm": 43.75,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 3.3189,
+      "mean_token_accuracy": 0.3538851010613143,
+      "num_tokens": 1283845.0,
+      "step": 90
+    },
+    {
+      "entropy": 2.7403977632522585,
+      "epoch": 0.32,
+      "grad_norm": 26.25,
+      "learning_rate": 1.98e-06,
+      "loss": 3.0218,
+      "mean_token_accuracy": 0.3812081998214126,
+      "num_tokens": 1430551.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.32,
+      "eval_coding_entropy": 2.1376109790802,
+      "eval_coding_loss": 2.211063861846924,
+      "eval_coding_mean_token_accuracy": 0.4966651816368103,
+      "eval_coding_num_tokens": 1430551.0,
+      "eval_coding_runtime": 81.7119,
+      "eval_coding_samples_per_second": 6.119,
+      "eval_coding_steps_per_second": 3.06,
+      "step": 100
+    },
+    {
+      "epoch": 0.32,
+      "eval_math_entropy": 2.7154639225006103,
+      "eval_math_loss": 2.8518309593200684,
+      "eval_math_mean_token_accuracy": 0.402160507440567,
+      "eval_math_num_tokens": 1430551.0,
+      "eval_math_runtime": 46.4902,
+      "eval_math_samples_per_second": 10.755,
+      "eval_math_steps_per_second": 5.377,
+      "step": 100
+    },
+    {
+      "entropy": 2.6445168428123,
+      "epoch": 0.352,
+      "grad_norm": 22.25,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 2.7264,
+      "mean_token_accuracy": 0.41560478005558255,
+      "num_tokens": 1573121.0,
+      "step": 110
+    },
+    {
+      "entropy": 2.5231464847922327,
+      "epoch": 0.384,
+      "grad_norm": 17.5,
+      "learning_rate": 2.38e-06,
+      "loss": 2.597,
+      "mean_token_accuracy": 0.4375463806092739,
+      "num_tokens": 1714945.0,
+      "step": 120
+    },
+    {
+      "entropy": 2.426661425083876,
+      "epoch": 0.416,
+      "grad_norm": 18.875,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 2.4805,
+      "mean_token_accuracy": 0.45005974899977447,
+      "num_tokens": 1851617.0,
+      "step": 130
+    },
+    {
+      "entropy": 2.3277198508381844,
+      "epoch": 0.448,
+      "grad_norm": 22.0,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 2.3694,
+      "mean_token_accuracy": 0.4707961905747652,
+      "num_tokens": 1994075.0,
+      "step": 140
+    },
+    {
+      "entropy": 2.2954090014100075,
+      "epoch": 0.48,
+      "grad_norm": 20.875,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 2.3273,
+      "mean_token_accuracy": 0.4766052259132266,
+      "num_tokens": 2136748.0,
+      "step": 150
+    },
+    {
+      "entropy": 2.2432560376822948,
+      "epoch": 0.512,
+      "grad_norm": 15.1875,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 2.2433,
+      "mean_token_accuracy": 0.4955187674611807,
+      "num_tokens": 2276430.0,
+      "step": 160
+    },
+    {
+      "entropy": 2.1506235644221308,
+      "epoch": 0.544,
+      "grad_norm": 20.375,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 2.1835,
+      "mean_token_accuracy": 0.5024478340521454,
+      "num_tokens": 2420005.0,
+      "step": 170
+    },
+    {
+      "entropy": 2.173030565679073,
+      "epoch": 0.576,
+      "grad_norm": 21.25,
+      "learning_rate": 3.58e-06,
+      "loss": 2.2005,
+      "mean_token_accuracy": 0.5029806947335601,
+      "num_tokens": 2567140.0,
+      "step": 180
+    },
+    {
+      "entropy": 2.067853280156851,
+      "epoch": 0.608,
+      "grad_norm": 15.625,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 2.0934,
+      "mean_token_accuracy": 0.52086176071316,
+      "num_tokens": 2707769.0,
+      "step": 190
+    },
+    {
+      "entropy": 2.0607680074870585,
+      "epoch": 0.64,
+      "grad_norm": 16.125,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 2.0413,
+      "mean_token_accuracy": 0.5280526082962751,
+      "num_tokens": 2851391.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_coding_entropy": 2.082627496242523,
+      "eval_coding_loss": 2.0564982891082764,
+      "eval_coding_mean_token_accuracy": 0.5250193012952804,
+      "eval_coding_num_tokens": 2851391.0,
+      "eval_coding_runtime": 81.149,
+      "eval_coding_samples_per_second": 6.162,
+      "eval_coding_steps_per_second": 3.081,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_math_entropy": 2.0227903814315797,
+      "eval_math_loss": 2.056239604949951,
+      "eval_math_mean_token_accuracy": 0.5309821690320968,
+      "eval_math_num_tokens": 2851391.0,
+      "eval_math_runtime": 46.4944,
+      "eval_math_samples_per_second": 10.754,
+      "eval_math_steps_per_second": 5.377,
+      "step": 200
+    },
+    {
+      "entropy": 2.0263434551656245,
+      "epoch": 0.672,
+      "grad_norm": 15.6875,
+      "learning_rate": 4.18e-06,
+      "loss": 2.0178,
+      "mean_token_accuracy": 0.5313251735642552,
+      "num_tokens": 2997713.0,
+      "step": 210
+    },
+    {
+      "entropy": 2.0123437851667405,
+      "epoch": 0.704,
+      "grad_norm": 17.375,
+      "learning_rate": 4.38e-06,
+      "loss": 2.0443,
+      "mean_token_accuracy": 0.5295042948797345,
+      "num_tokens": 3141249.0,
+      "step": 220
+    },
+    {
+      "entropy": 2.039450883120298,
+      "epoch": 0.736,
+      "grad_norm": 18.5,
+      "learning_rate": 4.58e-06,
+      "loss": 2.0519,
+      "mean_token_accuracy": 0.5338158709928393,
+      "num_tokens": 3279135.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.9529575437307358,
+      "epoch": 0.768,
+      "grad_norm": 15.4375,
+      "learning_rate": 4.78e-06,
+      "loss": 1.9503,
+      "mean_token_accuracy": 0.5456137385219335,
+      "num_tokens": 3429580.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.9847303189337253,
+      "epoch": 0.8,
+      "grad_norm": 18.875,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 1.9973,
+      "mean_token_accuracy": 0.5430868744850159,
+      "num_tokens": 3574102.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.941353114694357,
+      "epoch": 0.832,
+      "grad_norm": 17.625,
+      "learning_rate": 5.18e-06,
+      "loss": 1.9468,
+      "mean_token_accuracy": 0.5475187944248319,
+      "num_tokens": 3711682.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.8798863902688026,
+      "epoch": 0.864,
+      "grad_norm": 21.0,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 1.873,
+      "mean_token_accuracy": 0.5597963750362396,
+      "num_tokens": 3855113.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.835599235445261,
+      "epoch": 0.896,
+      "grad_norm": 14.375,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 1.8228,
+      "mean_token_accuracy": 0.5651934513822198,
+      "num_tokens": 3995349.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.8507938131690025,
+      "epoch": 0.928,
+      "grad_norm": 16.875,
+      "learning_rate": 5.78e-06,
+      "loss": 1.8394,
+      "mean_token_accuracy": 0.5661519970744848,
+      "num_tokens": 4135852.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.8499364234507083,
+      "epoch": 0.96,
+      "grad_norm": 16.75,
+      "learning_rate": 5.98e-06,
+      "loss": 1.8409,
+      "mean_token_accuracy": 0.5702653925865888,
+      "num_tokens": 4274655.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "eval_coding_entropy": 2.037804766178131,
+      "eval_coding_loss": 2.022986650466919,
+      "eval_coding_mean_token_accuracy": 0.5313419116735458,
+      "eval_coding_num_tokens": 4274655.0,
+      "eval_coding_runtime": 81.1157,
+      "eval_coding_samples_per_second": 6.164,
+      "eval_coding_steps_per_second": 3.082,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "eval_math_entropy": 1.802053134918213,
+      "eval_math_loss": 1.8271188735961914,
+      "eval_math_mean_token_accuracy": 0.5726956763267517,
+      "eval_math_num_tokens": 4274655.0,
+      "eval_math_runtime": 46.5128,
+      "eval_math_samples_per_second": 10.75,
+      "eval_math_steps_per_second": 5.375,
+      "step": 300
+    },
+    {
+      "entropy": 1.8301428437232972,
+      "epoch": 0.992,
+      "grad_norm": 17.25,
+      "learning_rate": 6.18e-06,
+      "loss": 1.8206,
+      "mean_token_accuracy": 0.569915696978569,
+      "num_tokens": 4415600.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.8003562704512948,
+      "epoch": 1.0224,
+      "grad_norm": 12.75,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 1.7748,
+      "mean_token_accuracy": 0.5778473768579332,
+      "num_tokens": 4551683.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.7793153546750546,
+      "epoch": 1.0544,
+      "grad_norm": 15.125,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 1.7851,
+      "mean_token_accuracy": 0.579111011326313,
+      "num_tokens": 4693696.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.7894929252564906,
+      "epoch": 1.0864,
+      "grad_norm": 14.625,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 1.7635,
+      "mean_token_accuracy": 0.5829346276819706,
+      "num_tokens": 4834396.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.7216685689985751,
+      "epoch": 1.1184,
+      "grad_norm": 15.1875,
+      "learning_rate": 6.98e-06,
+      "loss": 1.7294,
+      "mean_token_accuracy": 0.5859142445027828,
+      "num_tokens": 4971857.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.795823823660612,
+      "epoch": 1.1504,
+      "grad_norm": 19.0,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 1.7665,
+      "mean_token_accuracy": 0.5832353200763464,
+      "num_tokens": 5110683.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.7300004258751869,
+      "epoch": 1.1824,
+      "grad_norm": 13.625,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 1.6956,
+      "mean_token_accuracy": 0.589705759473145,
+      "num_tokens": 5252621.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.6924258664250373,
+      "epoch": 1.2144,
+      "grad_norm": 18.875,
+      "learning_rate": 7.58e-06,
+      "loss": 1.687,
+      "mean_token_accuracy": 0.5896685650572181,
+      "num_tokens": 5394768.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.754713100194931,
+      "epoch": 1.2464,
+      "grad_norm": 12.5,
+      "learning_rate": 7.78e-06,
+      "loss": 1.7225,
+      "mean_token_accuracy": 0.5889439214020967,
+      "num_tokens": 5533439.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.704547866433859,
+      "epoch": 1.2784,
+      "grad_norm": 20.5,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 1.6608,
+      "mean_token_accuracy": 0.599461050145328,
+      "num_tokens": 5670712.0,
+      "step": 400
+    },
+    {
+      "epoch": 1.2784,
+      "eval_coding_entropy": 1.9745532732009887,
+      "eval_coding_loss": 2.004408359527588,
+      "eval_coding_mean_token_accuracy": 0.5342881647348404,
+      "eval_coding_num_tokens": 5670712.0,
+      "eval_coding_runtime": 81.3362,
+      "eval_coding_samples_per_second": 6.147,
+      "eval_coding_steps_per_second": 3.074,
+      "step": 400
+    },
+    {
+      "epoch": 1.2784,
+      "eval_math_entropy": 1.6563503637313843,
+      "eval_math_loss": 1.6785366535186768,
+      "eval_math_mean_token_accuracy": 0.6000865260362626,
+      "eval_math_num_tokens": 5670712.0,
+      "eval_math_runtime": 46.6244,
+      "eval_math_samples_per_second": 10.724,
+      "eval_math_steps_per_second": 5.362,
+      "step": 400
+    },
+    {
+      "entropy": 1.6839238464832307,
+      "epoch": 1.3104,
+      "grad_norm": 16.625,
+      "learning_rate": 8.18e-06,
+      "loss": 1.6869,
+      "mean_token_accuracy": 0.5953959075734019,
+      "num_tokens": 5813458.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.6726863503456115,
+      "epoch": 1.3424,
+      "grad_norm": 12.75,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 1.6315,
+      "mean_token_accuracy": 0.6037707706913352,
+      "num_tokens": 5951880.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.6084760576486588,
+      "epoch": 1.3744,
+      "grad_norm": 14.3125,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 1.5881,
+      "mean_token_accuracy": 0.6148429989814759,
+      "num_tokens": 6098256.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.6628840889781713,
+      "epoch": 1.4064,
+      "grad_norm": 15.6875,
+      "learning_rate": 8.78e-06,
+      "loss": 1.6761,
+      "mean_token_accuracy": 0.6000858852639794,
+      "num_tokens": 6242646.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.6300812467932702,
+      "epoch": 1.4384000000000001,
+      "grad_norm": 21.25,
+      "learning_rate": 8.98e-06,
+      "loss": 1.6003,
+      "mean_token_accuracy": 0.6083745462819934,
+      "num_tokens": 6386025.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.6391606278717519,
+      "epoch": 1.4704,
+      "grad_norm": 20.375,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 1.6273,
+      "mean_token_accuracy": 0.6097741341218352,
+      "num_tokens": 6533075.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.6905927129089833,
+      "epoch": 1.5024,
+      "grad_norm": 15.375,
+      "learning_rate": 9.38e-06,
+      "loss": 1.6572,
+      "mean_token_accuracy": 0.6046760326251388,
+      "num_tokens": 6677208.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.6522716350853444,
+      "epoch": 1.5344,
+      "grad_norm": 13.5,
+      "learning_rate": 9.58e-06,
+      "loss": 1.6494,
+      "mean_token_accuracy": 0.6073551412671805,
+      "num_tokens": 6814455.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.6329746030271053,
+      "epoch": 1.5664,
+      "grad_norm": 13.6875,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 1.5924,
+      "mean_token_accuracy": 0.614310959354043,
+      "num_tokens": 6958149.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.5567305848002433,
+      "epoch": 1.5984,
+      "grad_norm": 12.6875,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 1.5416,
+      "mean_token_accuracy": 0.6263186866417527,
+      "num_tokens": 7099528.0,
+      "step": 500
+    },
+    {
+      "epoch": 1.5984,
+      "eval_coding_entropy": 1.9465989785194397,
+      "eval_coding_loss": 1.990442156791687,
+      "eval_coding_mean_token_accuracy": 0.5386206238269806,
+      "eval_coding_num_tokens": 7099528.0,
+      "eval_coding_runtime": 81.2403,
+      "eval_coding_samples_per_second": 6.155,
+      "eval_coding_steps_per_second": 3.077,
+      "step": 500
+    },
+    {
+      "epoch": 1.5984,
+      "eval_math_entropy": 1.5453263101577759,
+      "eval_math_loss": 1.5754268169403076,
+      "eval_math_mean_token_accuracy": 0.6199949022531509,
+      "eval_math_num_tokens": 7099528.0,
+      "eval_math_runtime": 45.9145,
+      "eval_math_samples_per_second": 10.89,
+      "eval_math_steps_per_second": 5.445,
+      "step": 500
+    },
+    {
+      "entropy": 1.5644792415201665,
+      "epoch": 1.6303999999999998,
+      "grad_norm": 15.5625,
+      "learning_rate": 1.018e-05,
+      "loss": 1.5511,
+      "mean_token_accuracy": 0.6213703533634544,
+      "num_tokens": 7241222.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.6044137500226499,
+      "epoch": 1.6623999999999999,
+      "grad_norm": 14.6875,
+      "learning_rate": 1.038e-05,
+      "loss": 1.5761,
+      "mean_token_accuracy": 0.6146464478224516,
+      "num_tokens": 7387899.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.5174766711890697,
+      "epoch": 1.6944,
+      "grad_norm": 14.4375,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 1.515,
+      "mean_token_accuracy": 0.6280366148799658,
+      "num_tokens": 7535577.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.5012709453701973,
+      "epoch": 1.7264,
+      "grad_norm": 18.0,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 1.4795,
+      "mean_token_accuracy": 0.6314780458807945,
+      "num_tokens": 7679139.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.557135234773159,
+      "epoch": 1.7584,
+      "grad_norm": 21.0,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 1.5279,
+      "mean_token_accuracy": 0.6282323149964213,
+      "num_tokens": 7821116.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.5534105684608221,
+      "epoch": 1.7904,
+      "grad_norm": 15.9375,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 1.5329,
+      "mean_token_accuracy": 0.6273708928376436,
+      "num_tokens": 7955945.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.5024470299482346,
+      "epoch": 1.8224,
+      "grad_norm": 16.75,
+      "learning_rate": 1.138e-05,
+      "loss": 1.5027,
+      "mean_token_accuracy": 0.6360822120681405,
+      "num_tokens": 8098517.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.468005495518446,
+      "epoch": 1.8544,
+      "grad_norm": 14.75,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 1.4433,
+      "mean_token_accuracy": 0.6402354558929801,
+      "num_tokens": 8247652.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.4894821517169476,
+      "epoch": 1.8864,
+      "grad_norm": 15.125,
+      "learning_rate": 1.178e-05,
+      "loss": 1.4651,
+      "mean_token_accuracy": 0.6387512655928731,
+      "num_tokens": 8390969.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.5070936933159829,
+      "epoch": 1.9184,
+      "grad_norm": 14.625,
+      "learning_rate": 1.198e-05,
+      "loss": 1.4912,
+      "mean_token_accuracy": 0.6379801956936717,
+      "num_tokens": 8532905.0,
+      "step": 600
+    },
+    {
+      "epoch": 1.9184,
+      "eval_coding_entropy": 1.9700443706512452,
+      "eval_coding_loss": 1.9896337985992432,
+      "eval_coding_mean_token_accuracy": 0.5387515650987625,
+      "eval_coding_num_tokens": 8532905.0,
+      "eval_coding_runtime": 81.1982,
+      "eval_coding_samples_per_second": 6.158,
+      "eval_coding_steps_per_second": 3.079,
+      "step": 600
+    },
+    {
+      "epoch": 1.9184,
+      "eval_math_entropy": 1.479466641664505,
+      "eval_math_loss": 1.4867653846740723,
+      "eval_math_mean_token_accuracy": 0.6389840010404587,
+      "eval_math_num_tokens": 8532905.0,
+      "eval_math_runtime": 46.5533,
+      "eval_math_samples_per_second": 10.74,
+      "eval_math_steps_per_second": 5.37,
+      "step": 600
+    },
+    {
+      "entropy": 1.5151923824101687,
+      "epoch": 1.9504000000000001,
+      "grad_norm": 17.125,
+      "learning_rate": 1.218e-05,
+      "loss": 1.4863,
+      "mean_token_accuracy": 0.6339302582666277,
+      "num_tokens": 8676028.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.468616184964776,
+      "epoch": 1.9824000000000002,
+      "grad_norm": 14.5,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 1.4732,
+      "mean_token_accuracy": 0.6385495683178306,
+      "num_tokens": 8817323.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.5188211822196056,
+      "epoch": 2.0128,
+      "grad_norm": 14.4375,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 1.4835,
+      "mean_token_accuracy": 0.634301284426137,
+      "num_tokens": 8949173.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.3755222257226705,
+      "epoch": 2.0448,
+      "grad_norm": 12.9375,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 1.3601,
+      "mean_token_accuracy": 0.6608490329235792,
+      "num_tokens": 9085612.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.4207779351621865,
+      "epoch": 2.0768,
+      "grad_norm": 13.9375,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 1.4097,
+      "mean_token_accuracy": 0.6495957912877202,
+      "num_tokens": 9233211.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.437428993731737,
+      "epoch": 2.1088,
+      "grad_norm": 17.5,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 1.4124,
+      "mean_token_accuracy": 0.6517477501183748,
+      "num_tokens": 9376523.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.3855586621910334,
+      "epoch": 2.1408,
+      "grad_norm": 14.75,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 1.3542,
+      "mean_token_accuracy": 0.6590309431776404,
+      "num_tokens": 9523963.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.4285461019724608,
+      "epoch": 2.1728,
+      "grad_norm": 16.375,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 1.4063,
+      "mean_token_accuracy": 0.6481046233326196,
+      "num_tokens": 9670446.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.389345682412386,
+      "epoch": 2.2048,
+      "grad_norm": 14.5625,
+      "learning_rate": 1.378e-05,
+      "loss": 1.368,
+      "mean_token_accuracy": 0.6584958679974079,
+      "num_tokens": 9809324.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.4275502871721983,
+      "epoch": 2.2368,
+      "grad_norm": 14.875,
+      "learning_rate": 1.398e-05,
+      "loss": 1.4041,
+      "mean_token_accuracy": 0.6543006511405111,
+      "num_tokens": 9943194.0,
+      "step": 700
+    },
+    {
+      "epoch": 2.2368,
+      "eval_coding_entropy": 1.8811444640159607,
+      "eval_coding_loss": 1.999657154083252,
+      "eval_coding_mean_token_accuracy": 0.5383623929023743,
+      "eval_coding_num_tokens": 9943194.0,
+      "eval_coding_runtime": 80.9539,
+      "eval_coding_samples_per_second": 6.176,
+      "eval_coding_steps_per_second": 3.088,
+      "step": 700
+    },
+    {
+      "epoch": 2.2368,
+      "eval_math_entropy": 1.3948393285274505,
+      "eval_math_loss": 1.4332354068756104,
+      "eval_math_mean_token_accuracy": 0.6499490721225738,
+      "eval_math_num_tokens": 9943194.0,
+      "eval_math_runtime": 46.0836,
+      "eval_math_samples_per_second": 10.85,
+      "eval_math_steps_per_second": 5.425,
+      "step": 700
+    },
+    {
+      "entropy": 1.407985271140933,
+      "epoch": 2.2688,
+      "grad_norm": 16.625,
+      "learning_rate": 1.418e-05,
+      "loss": 1.3837,
+      "mean_token_accuracy": 0.6551688494160771,
+      "num_tokens": 10087633.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.378972141817212,
+      "epoch": 2.3008,
+      "grad_norm": 13.375,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 1.3544,
+      "mean_token_accuracy": 0.6556477602571249,
+      "num_tokens": 10229382.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.354974314570427,
+      "epoch": 2.3327999999999998,
+      "grad_norm": 14.9375,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 1.3344,
+      "mean_token_accuracy": 0.6645016176626086,
+      "num_tokens": 10376268.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.3568570345640183,
+      "epoch": 2.3648,
+      "grad_norm": 17.75,
+      "learning_rate": 1.478e-05,
+      "loss": 1.3345,
+      "mean_token_accuracy": 0.6638497773557901,
+      "num_tokens": 10518064.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.418501327931881,
+      "epoch": 2.3968,
+      "grad_norm": 15.5625,
+      "learning_rate": 1.498e-05,
+      "loss": 1.3993,
+      "mean_token_accuracy": 0.6545212138444185,
+      "num_tokens": 10654090.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.3498825568705797,
+      "epoch": 2.4288,
+      "grad_norm": 14.8125,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 1.3302,
+      "mean_token_accuracy": 0.6608284536749125,
+      "num_tokens": 10803539.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.3939791657030582,
+      "epoch": 2.4608,
+      "grad_norm": 15.1875,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 1.3721,
+      "mean_token_accuracy": 0.6571606319397688,
+      "num_tokens": 10951500.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.4095406230539083,
+      "epoch": 2.4928,
+      "grad_norm": 12.6875,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 1.3827,
+      "mean_token_accuracy": 0.6542418278753758,
+      "num_tokens": 11089421.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.3628040976822375,
+      "epoch": 2.5248,
+      "grad_norm": 16.0,
+      "learning_rate": 1.578e-05,
+      "loss": 1.3434,
+      "mean_token_accuracy": 0.6625945869833231,
+      "num_tokens": 11233056.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.353104681521654,
+      "epoch": 2.5568,
+      "grad_norm": 13.5,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 1.3441,
+      "mean_token_accuracy": 0.663878696039319,
+      "num_tokens": 11382658.0,
+      "step": 800
+    },
+    {
+      "epoch": 2.5568,
+      "eval_coding_entropy": 1.88368141412735,
+      "eval_coding_loss": 2.0113229751586914,
+      "eval_coding_mean_token_accuracy": 0.5368727495670319,
+      "eval_coding_num_tokens": 11382658.0,
+      "eval_coding_runtime": 80.7647,
+      "eval_coding_samples_per_second": 6.191,
+      "eval_coding_steps_per_second": 3.095,
+      "step": 800
+    },
+    {
+      "epoch": 2.5568,
+      "eval_math_entropy": 1.3451528911590576,
+      "eval_math_loss": 1.3778893947601318,
+      "eval_math_mean_token_accuracy": 0.6607881124019623,
+      "eval_math_num_tokens": 11382658.0,
+      "eval_math_runtime": 46.1509,
+      "eval_math_samples_per_second": 10.834,
+      "eval_math_steps_per_second": 5.417,
+      "step": 800
+    },
+    {
+      "entropy": 1.3698780447244645,
+      "epoch": 2.5888,
+      "grad_norm": 14.0625,
+      "learning_rate": 1.618e-05,
+      "loss": 1.3446,
+      "mean_token_accuracy": 0.667889142036438,
+      "num_tokens": 11520248.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.3439954832196235,
+      "epoch": 2.6208,
+      "grad_norm": 13.25,
+      "learning_rate": 1.638e-05,
+      "loss": 1.3411,
+      "mean_token_accuracy": 0.6649587988853455,
+      "num_tokens": 11662426.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.3382667150348424,
+      "epoch": 2.6528,
+      "grad_norm": 15.125,
+      "learning_rate": 1.658e-05,
+      "loss": 1.2968,
+      "mean_token_accuracy": 0.667837655171752,
+      "num_tokens": 11801529.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.3607995260506869,
+      "epoch": 2.6848,
+      "grad_norm": 12.8125,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 1.3342,
+      "mean_token_accuracy": 0.6632724912837148,
+      "num_tokens": 11945073.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.340226661413908,
+      "epoch": 2.7168,
+      "grad_norm": 18.375,
+      "learning_rate": 1.698e-05,
+      "loss": 1.3183,
+      "mean_token_accuracy": 0.6692224076017738,
+      "num_tokens": 12086334.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.3257515206933022,
+      "epoch": 2.7488,
+      "grad_norm": 17.75,
+      "learning_rate": 1.718e-05,
+      "loss": 1.3204,
+      "mean_token_accuracy": 0.6661742279306054,
+      "num_tokens": 12227580.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.2998232152312994,
+      "epoch": 2.7808,
+      "grad_norm": 14.0,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 1.2841,
+      "mean_token_accuracy": 0.6723095076158643,
+      "num_tokens": 12366568.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.321110926195979,
+      "epoch": 2.8128,
+      "grad_norm": 14.1875,
+      "learning_rate": 1.758e-05,
+      "loss": 1.2809,
+      "mean_token_accuracy": 0.6734412474557757,
+      "num_tokens": 12508616.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.3441704627126456,
+      "epoch": 2.8448,
+      "grad_norm": 14.5625,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 1.3264,
+      "mean_token_accuracy": 0.6696716655045748,
+      "num_tokens": 12651142.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.350888105481863,
+      "epoch": 2.8768000000000002,
+      "grad_norm": 13.125,
+      "learning_rate": 1.798e-05,
+      "loss": 1.3305,
+      "mean_token_accuracy": 0.6628765376284719,
+      "num_tokens": 12790697.0,
+      "step": 900
+    },
+    {
+      "epoch": 2.8768000000000002,
+      "eval_coding_entropy": 1.923588216304779,
+      "eval_coding_loss": 2.011129379272461,
+      "eval_coding_mean_token_accuracy": 0.5363702869415283,
+      "eval_coding_num_tokens": 12790697.0,
+      "eval_coding_runtime": 80.767,
+      "eval_coding_samples_per_second": 6.191,
+      "eval_coding_steps_per_second": 3.095,
+      "step": 900
+    },
+    {
+      "epoch": 2.8768000000000002,
+      "eval_math_entropy": 1.3416466426849365,
+      "eval_math_loss": 1.3308217525482178,
+      "eval_math_mean_token_accuracy": 0.6704786785840988,
+      "eval_math_num_tokens": 12790697.0,
+      "eval_math_runtime": 46.0849,
+      "eval_math_samples_per_second": 10.85,
+      "eval_math_steps_per_second": 5.425,
+      "step": 900
+    },
+    {
+      "entropy": 1.3483149357140065,
+      "epoch": 2.9088000000000003,
+      "grad_norm": 11.875,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 1.3227,
+      "mean_token_accuracy": 0.6711992308497429,
+      "num_tokens": 12931565.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.2966109592467547,
+      "epoch": 2.9408,
+      "grad_norm": 13.4375,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 1.2763,
+      "mean_token_accuracy": 0.6791517404839397,
+      "num_tokens": 13073417.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.3717037633061409,
+      "epoch": 2.9728,
+      "grad_norm": 13.0,
+      "learning_rate": 1.858e-05,
+      "loss": 1.3564,
+      "mean_token_accuracy": 0.6620129646733404,
+      "num_tokens": 13220708.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.3031759885580916,
+      "epoch": 3.0032,
+      "grad_norm": 12.5,
+      "learning_rate": 1.878e-05,
+      "loss": 1.2593,
+      "mean_token_accuracy": 0.6780896206435404,
+      "num_tokens": 13357917.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.2132978197187185,
+      "epoch": 3.0352,
+      "grad_norm": 12.125,
+      "learning_rate": 1.898e-05,
+      "loss": 1.2061,
+      "mean_token_accuracy": 0.6939747478812933,
+      "num_tokens": 13504598.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.2069394052028657,
+      "epoch": 3.0672,
+      "grad_norm": 11.625,
+      "learning_rate": 1.918e-05,
+      "loss": 1.1756,
+      "mean_token_accuracy": 0.695431136712432,
+      "num_tokens": 13654124.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2009814836084842,
+      "epoch": 3.0992,
+      "grad_norm": 11.0625,
+      "learning_rate": 1.938e-05,
+      "loss": 1.1898,
+      "mean_token_accuracy": 0.6938818197697401,
+      "num_tokens": 13797243.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.2126072812825441,
+      "epoch": 3.1312,
+      "grad_norm": 12.0,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 1.1748,
+      "mean_token_accuracy": 0.6926786806434393,
+      "num_tokens": 13944269.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.2055565655231475,
+      "epoch": 3.1632,
+      "grad_norm": 10.75,
+      "learning_rate": 1.978e-05,
+      "loss": 1.1736,
+      "mean_token_accuracy": 0.6961859103292227,
+      "num_tokens": 14084426.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.2099793013185263,
+      "epoch": 3.1952,
+      "grad_norm": 12.8125,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 1.1884,
+      "mean_token_accuracy": 0.6956615813076497,
+      "num_tokens": 14225578.0,
+      "step": 1000
+    },
+    {
+      "epoch": 3.1952,
+      "eval_coding_entropy": 1.8568685698509215,
+      "eval_coding_loss": 2.0376784801483154,
+      "eval_coding_mean_token_accuracy": 0.5336039383411407,
+      "eval_coding_num_tokens": 14225578.0,
+      "eval_coding_runtime": 81.5817,
+      "eval_coding_samples_per_second": 6.129,
+      "eval_coding_steps_per_second": 3.064,
+      "step": 1000
+    },
+    {
+      "epoch": 3.1952,
+      "eval_math_entropy": 1.2256325538158417,
+      "eval_math_loss": 1.2980027198791504,
+      "eval_math_mean_token_accuracy": 0.6779666702747345,
+      "eval_math_num_tokens": 14225578.0,
+      "eval_math_runtime": 46.295,
+      "eval_math_samples_per_second": 10.8,
+      "eval_math_steps_per_second": 5.4,
+      "step": 1000
+    },
+    {
+      "entropy": 1.2310455348342657,
+      "epoch": 3.2272,
+      "grad_norm": 11.75,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 1.2069,
+      "mean_token_accuracy": 0.6908577650785446,
+      "num_tokens": 14371313.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.2366832181811334,
+      "epoch": 3.2592,
+      "grad_norm": 9.9375,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 1.2053,
+      "mean_token_accuracy": 0.6909629859030246,
+      "num_tokens": 14510372.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.1998371604830027,
+      "epoch": 3.2912,
+      "grad_norm": 11.1875,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 1.1928,
+      "mean_token_accuracy": 0.6971310704946518,
+      "num_tokens": 14654048.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.2052522465586661,
+      "epoch": 3.3232,
+      "grad_norm": 10.6875,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 1.1736,
+      "mean_token_accuracy": 0.6949702069163323,
+      "num_tokens": 14799061.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.2304670121520758,
+      "epoch": 3.3552,
+      "grad_norm": 9.125,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 1.208,
+      "mean_token_accuracy": 0.6886732917279005,
+      "num_tokens": 14948615.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.195398748293519,
+      "epoch": 3.3872,
+      "grad_norm": 10.4375,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 1.1855,
+      "mean_token_accuracy": 0.7008309479802847,
+      "num_tokens": 15091830.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.1796185825020076,
+      "epoch": 3.4192,
+      "grad_norm": 11.375,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 1.1623,
+      "mean_token_accuracy": 0.7005136132240295,
+      "num_tokens": 15223941.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.1732951115816832,
+      "epoch": 3.4512,
+      "grad_norm": 13.8125,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 1.1437,
+      "mean_token_accuracy": 0.7002243742346763,
+      "num_tokens": 15364119.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.1816517032682896,
+      "epoch": 3.4832,
+      "grad_norm": 11.125,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 1.1425,
+      "mean_token_accuracy": 0.6995426103472709,
+      "num_tokens": 15508933.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.2135872263461351,
+      "epoch": 3.5152,
+      "grad_norm": 8.75,
+      "learning_rate": 1.978e-05,
+      "loss": 1.1847,
+      "mean_token_accuracy": 0.6952282149344683,
+      "num_tokens": 15654554.0,
+      "step": 1100
+    },
+    {
+      "epoch": 3.5152,
+      "eval_coding_entropy": 1.8463355026245116,
+      "eval_coding_loss": 2.0515615940093994,
+      "eval_coding_mean_token_accuracy": 0.5331288994550705,
+      "eval_coding_num_tokens": 15654554.0,
+      "eval_coding_runtime": 80.9144,
+      "eval_coding_samples_per_second": 6.179,
+      "eval_coding_steps_per_second": 3.09,
+      "step": 1100
+    },
+    {
+      "epoch": 3.5152,
+      "eval_math_entropy": 1.2076501297950744,
+      "eval_math_loss": 1.240770697593689,
+      "eval_math_mean_token_accuracy": 0.6900603187084198,
+      "eval_math_num_tokens": 15654554.0,
+      "eval_math_runtime": 46.9328,
+      "eval_math_samples_per_second": 10.654,
+      "eval_math_steps_per_second": 5.327,
+      "step": 1100
+    },
+    {
+      "entropy": 1.212555281817913,
+      "epoch": 3.5472,
+      "grad_norm": 9.4375,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 1.1833,
+      "mean_token_accuracy": 0.6934925394132734,
+      "num_tokens": 15793364.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.207034119591117,
+      "epoch": 3.5792,
+      "grad_norm": 10.875,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 1.1711,
+      "mean_token_accuracy": 0.697374514490366,
+      "num_tokens": 15935657.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.2222992777824402,
+      "epoch": 3.6112,
+      "grad_norm": 9.9375,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 1.2014,
+      "mean_token_accuracy": 0.6935698345303536,
+      "num_tokens": 16077275.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.1657866805791854,
+      "epoch": 3.6432,
+      "grad_norm": 9.4375,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 1.1546,
+      "mean_token_accuracy": 0.7033114738762378,
+      "num_tokens": 16222318.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.1633354667574167,
+      "epoch": 3.6752000000000002,
+      "grad_norm": 9.25,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 1.1251,
+      "mean_token_accuracy": 0.7043058726936579,
+      "num_tokens": 16361855.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.1620853256434203,
+      "epoch": 3.7072000000000003,
+      "grad_norm": 9.75,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 1.1342,
+      "mean_token_accuracy": 0.7022159334272147,
+      "num_tokens": 16500375.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.1738709833472967,
+      "epoch": 3.7392,
+      "grad_norm": 10.0625,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 1.1561,
+      "mean_token_accuracy": 0.7008879505097866,
+      "num_tokens": 16643092.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.1810582045465707,
+      "epoch": 3.7712,
+      "grad_norm": 8.5625,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 1.1661,
+      "mean_token_accuracy": 0.6994068440049886,
+      "num_tokens": 16789527.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.1981235884130002,
+      "epoch": 3.8032,
+      "grad_norm": 9.25,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 1.184,
+      "mean_token_accuracy": 0.7009957425296307,
+      "num_tokens": 16930493.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.1709371741861105,
+      "epoch": 3.8352,
+      "grad_norm": 9.25,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 1.1328,
+      "mean_token_accuracy": 0.7039620807394386,
+      "num_tokens": 17066856.0,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8352,
+      "eval_coding_entropy": 1.7909702219963073,
+      "eval_coding_loss": 2.0417468547821045,
+      "eval_coding_mean_token_accuracy": 0.5346206701993942,
+      "eval_coding_num_tokens": 17066856.0,
+      "eval_coding_runtime": 81.4914,
+      "eval_coding_samples_per_second": 6.136,
+      "eval_coding_steps_per_second": 3.068,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8352,
+      "eval_math_entropy": 1.1532045022249222,
+      "eval_math_loss": 1.2045187950134277,
+      "eval_math_mean_token_accuracy": 0.6980784633159638,
+      "eval_math_num_tokens": 17066856.0,
+      "eval_math_runtime": 46.8505,
+      "eval_math_samples_per_second": 10.672,
+      "eval_math_steps_per_second": 5.336,
+      "step": 1200
+    },
+    {
+      "entropy": 1.156894364207983,
+      "epoch": 3.8672,
+      "grad_norm": 9.0625,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 1.1334,
+      "mean_token_accuracy": 0.7076525427401066,
+      "num_tokens": 17205604.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.174615489691496,
+      "epoch": 3.8992,
+      "grad_norm": 8.375,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 1.165,
+      "mean_token_accuracy": 0.6988233763724565,
+      "num_tokens": 17348293.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.1527796313166618,
+      "epoch": 3.9312,
+      "grad_norm": 9.3125,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 1.1257,
+      "mean_token_accuracy": 0.707377451658249,
+      "num_tokens": 17491463.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.1771091677248477,
+      "epoch": 3.9632,
+      "grad_norm": 10.0625,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 1.1595,
+      "mean_token_accuracy": 0.7032928232103586,
+      "num_tokens": 17628099.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.1742881268262864,
+      "epoch": 3.9952,
+      "grad_norm": 10.5625,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 1.1505,
+      "mean_token_accuracy": 0.7026802890002728,
+      "num_tokens": 17768068.0,
+      "step": 1250
+    },
+    {
+      "entropy": 1.0864369343770177,
+      "epoch": 4.0256,
+      "grad_norm": 7.90625,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 1.0635,
+      "mean_token_accuracy": 0.7234101566044908,
+      "num_tokens": 17904915.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.0292031805962325,
+      "epoch": 4.0576,
+      "grad_norm": 10.375,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 0.9885,
+      "mean_token_accuracy": 0.7318673226982355,
+      "num_tokens": 18050621.0,
+      "step": 1270
+    },
+    {
+      "entropy": 1.0440782070159913,
+      "epoch": 4.0896,
+      "grad_norm": 10.75,
+      "learning_rate": 1.938e-05,
+      "loss": 1.0119,
+      "mean_token_accuracy": 0.7287190053611994,
+      "num_tokens": 18195139.0,
+      "step": 1280
+    },
+    {
+      "entropy": 1.0936193302273751,
+      "epoch": 4.1216,
+      "grad_norm": 9.5,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 1.0587,
+      "mean_token_accuracy": 0.7225902825593948,
+      "num_tokens": 18335483.0,
+      "step": 1290
+    },
+    {
+      "entropy": 1.0716597214341164,
+      "epoch": 4.1536,
+      "grad_norm": 8.0,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 1.0521,
+      "mean_token_accuracy": 0.7205776769667864,
+      "num_tokens": 18471464.0,
+      "step": 1300
+    },
+    {
+      "epoch": 4.1536,
+      "eval_coding_entropy": 1.666171245574951,
+      "eval_coding_loss": 2.089735984802246,
+      "eval_coding_mean_token_accuracy": 0.5322686601877212,
+      "eval_coding_num_tokens": 18471464.0,
+      "eval_coding_runtime": 81.1644,
+      "eval_coding_samples_per_second": 6.16,
+      "eval_coding_steps_per_second": 3.08,
+      "step": 1300
+    },
+    {
+      "epoch": 4.1536,
+      "eval_math_entropy": 1.060988509297371,
+      "eval_math_loss": 1.184299111366272,
+      "eval_math_mean_token_accuracy": 0.7041985726356507,
+      "eval_math_num_tokens": 18471464.0,
+      "eval_math_runtime": 46.7403,
+      "eval_math_samples_per_second": 10.697,
+      "eval_math_steps_per_second": 5.349,
+      "step": 1300
+    },
+    {
+      "entropy": 1.042165393382311,
+      "epoch": 4.1856,
+      "grad_norm": 8.375,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 1.0117,
+      "mean_token_accuracy": 0.7323273476213217,
+      "num_tokens": 18613144.0,
+      "step": 1310
+    },
+    {
+      "entropy": 1.0549738246947526,
+      "epoch": 4.2176,
+      "grad_norm": 12.0625,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 1.0356,
+      "mean_token_accuracy": 0.7262615174055099,
+      "num_tokens": 18757354.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.9945293262600898,
+      "epoch": 4.2496,
+      "grad_norm": 8.5625,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 0.9893,
+      "mean_token_accuracy": 0.737324832752347,
+      "num_tokens": 18901730.0,
+      "step": 1330
+    },
+    {
+      "entropy": 1.0042558293789625,
+      "epoch": 4.2816,
+      "grad_norm": 8.1875,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 0.976,
+      "mean_token_accuracy": 0.7383205603808165,
+      "num_tokens": 19044725.0,
+      "step": 1340
+    },
+    {
+      "entropy": 1.0393657265231013,
+      "epoch": 4.3136,
+      "grad_norm": 8.625,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 1.0086,
+      "mean_token_accuracy": 0.7331446453928947,
+      "num_tokens": 19193912.0,
+      "step": 1350
+    },
+    {
+      "entropy": 1.0386454924941062,
+      "epoch": 4.3456,
+      "grad_norm": 9.0,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 1.0083,
+      "mean_token_accuracy": 0.7298929043114185,
+      "num_tokens": 19336463.0,
+      "step": 1360
+    },
+    {
+      "entropy": 1.0017262276262044,
+      "epoch": 4.3776,
+      "grad_norm": 8.3125,
+      "learning_rate": 1.918e-05,
+      "loss": 0.979,
+      "mean_token_accuracy": 0.7369209341704845,
+      "num_tokens": 19482758.0,
+      "step": 1370
+    },
+    {
+      "entropy": 1.0357928197830915,
+      "epoch": 4.4096,
+      "grad_norm": 11.25,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 1.0003,
+      "mean_token_accuracy": 0.7308139387518168,
+      "num_tokens": 19624149.0,
+      "step": 1380
+    },
+    {
+      "entropy": 1.0128012642264366,
+      "epoch": 4.4416,
+      "grad_norm": 9.6875,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 0.994,
+      "mean_token_accuracy": 0.7347184054553508,
+      "num_tokens": 19762641.0,
+      "step": 1390
+    },
+    {
+      "entropy": 1.0030905462801456,
+      "epoch": 4.4736,
+      "grad_norm": 7.96875,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 0.9852,
+      "mean_token_accuracy": 0.7376307789236307,
+      "num_tokens": 19906714.0,
+      "step": 1400
+    },
+    {
+      "epoch": 4.4736,
+      "eval_coding_entropy": 1.6414423942565919,
+      "eval_coding_loss": 2.086019277572632,
+      "eval_coding_mean_token_accuracy": 0.533721309542656,
+      "eval_coding_num_tokens": 19906714.0,
+      "eval_coding_runtime": 81.4357,
+      "eval_coding_samples_per_second": 6.14,
+      "eval_coding_steps_per_second": 3.07,
+      "step": 1400
+    },
+    {
+      "epoch": 4.4736,
+      "eval_math_entropy": 1.023936753153801,
+      "eval_math_loss": 1.1772490739822388,
+      "eval_math_mean_token_accuracy": 0.707464236497879,
+      "eval_math_num_tokens": 19906714.0,
+      "eval_math_runtime": 46.7296,
+      "eval_math_samples_per_second": 10.7,
+      "eval_math_steps_per_second": 5.35,
+      "step": 1400
+    },
+    {
+      "entropy": 1.0678872428834438,
+      "epoch": 4.5056,
+      "grad_norm": 8.5,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 1.0421,
+      "mean_token_accuracy": 0.7248407501727343,
+      "num_tokens": 20042824.0,
+      "step": 1410
+    },
+    {
+      "entropy": 1.0317748140543699,
+      "epoch": 4.5376,
+      "grad_norm": 7.28125,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 1.0118,
+      "mean_token_accuracy": 0.7290458243340254,
+      "num_tokens": 20184198.0,
+      "step": 1420
+    },
+    {
+      "entropy": 1.0234299018979072,
+      "epoch": 4.5696,
+      "grad_norm": 8.1875,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 0.9974,
+      "mean_token_accuracy": 0.734089670330286,
+      "num_tokens": 20326769.0,
+      "step": 1430
+    },
+    {
+      "entropy": 1.0432322915643453,
+      "epoch": 4.6016,
+      "grad_norm": 10.4375,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 1.0359,
+      "mean_token_accuracy": 0.7288251578807831,
+      "num_tokens": 20463248.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.9960782427340746,
+      "epoch": 4.6336,
+      "grad_norm": 7.75,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 0.9673,
+      "mean_token_accuracy": 0.7419588100165129,
+      "num_tokens": 20606156.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.0214475937187673,
+      "epoch": 4.6655999999999995,
+      "grad_norm": 9.75,
+      "learning_rate": 1.898e-05,
+      "loss": 1.0073,
+      "mean_token_accuracy": 0.7355681076645851,
+      "num_tokens": 20742663.0,
+      "step": 1460
+    },
+    {
+      "entropy": 1.042779802531004,
+      "epoch": 4.6975999999999996,
+      "grad_norm": 8.3125,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 1.015,
+      "mean_token_accuracy": 0.7308492448180914,
+      "num_tokens": 20888422.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.9938499752432108,
+      "epoch": 4.7296,
+      "grad_norm": 8.4375,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 0.9681,
+      "mean_token_accuracy": 0.7414834558963775,
+      "num_tokens": 21032183.0,
+      "step": 1480
+    },
+    {
+      "entropy": 1.016091812774539,
+      "epoch": 4.7616,
+      "grad_norm": 10.3125,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 0.9861,
+      "mean_token_accuracy": 0.7363665714859963,
+      "num_tokens": 21171363.0,
+      "step": 1490
+    },
+    {
+      "entropy": 1.0135044915601612,
+      "epoch": 4.7936,
+      "grad_norm": 7.5625,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 0.9953,
+      "mean_token_accuracy": 0.7363950561732053,
+      "num_tokens": 21316124.0,
+      "step": 1500
+    },
+    {
+      "epoch": 4.7936,
+      "eval_coding_entropy": 1.6775614714622498,
+      "eval_coding_loss": 2.1003665924072266,
+      "eval_coding_mean_token_accuracy": 0.5311660279035568,
+      "eval_coding_num_tokens": 21316124.0,
+      "eval_coding_runtime": 82.243,
+      "eval_coding_samples_per_second": 6.08,
+      "eval_coding_steps_per_second": 3.04,
+      "step": 1500
+    },
+    {
+      "epoch": 4.7936,
+      "eval_math_entropy": 1.0237463138103484,
+      "eval_math_loss": 1.1396057605743408,
+      "eval_math_mean_token_accuracy": 0.715588841676712,
+      "eval_math_num_tokens": 21316124.0,
+      "eval_math_runtime": 47.1978,
+      "eval_math_samples_per_second": 10.594,
+      "eval_math_steps_per_second": 5.297,
+      "step": 1500
+    },
+    {
+      "entropy": 1.0302676513791085,
+      "epoch": 4.8256,
+      "grad_norm": 7.5625,
+      "learning_rate": 1.886888888888889e-05,
+      "loss": 1.0034,
+      "mean_token_accuracy": 0.7323874134570361,
+      "num_tokens": 21456811.0,
+      "step": 1510
+    },
+    {
+      "entropy": 1.0087147317826748,
+      "epoch": 4.8576,
+      "grad_norm": 10.375,
+      "learning_rate": 1.884666666666667e-05,
+      "loss": 0.9966,
+      "mean_token_accuracy": 0.7373051200062036,
+      "num_tokens": 21598174.0,
+      "step": 1520
+    },
+    {
+      "entropy": 1.0385818116366863,
+      "epoch": 4.8896,
+      "grad_norm": 8.75,
+      "learning_rate": 1.8824444444444445e-05,
+      "loss": 1.0234,
+      "mean_token_accuracy": 0.7324004743248225,
+      "num_tokens": 21749526.0,
+      "step": 1530
+    },
+    {
+      "entropy": 1.0246109385043383,
+      "epoch": 4.9216,
+      "grad_norm": 8.125,
+      "learning_rate": 1.8802222222222226e-05,
+      "loss": 0.9977,
+      "mean_token_accuracy": 0.7329407773911953,
+      "num_tokens": 21885800.0,
+      "step": 1540
+    },
+    {
+      "entropy": 1.005864929407835,
+      "epoch": 4.9536,
+      "grad_norm": 9.4375,
+      "learning_rate": 1.878e-05,
+      "loss": 0.9762,
+      "mean_token_accuracy": 0.7406043078750372,
+      "num_tokens": 22028858.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.9871604397892952,
+      "epoch": 4.9856,
+      "grad_norm": 7.84375,
+      "learning_rate": 1.875777777777778e-05,
+      "loss": 0.9605,
+      "mean_token_accuracy": 0.7413943439722062,
+      "num_tokens": 22175575.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.9552774119533991,
+      "epoch": 5.016,
+      "grad_norm": 7.96875,
+      "learning_rate": 1.873555555555556e-05,
+      "loss": 0.9271,
+      "mean_token_accuracy": 0.7519231669996914,
+      "num_tokens": 22313061.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.908423101902008,
+      "epoch": 5.048,
+      "grad_norm": 6.65625,
+      "learning_rate": 1.8713333333333336e-05,
+      "loss": 0.8928,
+      "mean_token_accuracy": 0.7576735183596611,
+      "num_tokens": 22457889.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.9394706435501575,
+      "epoch": 5.08,
+      "grad_norm": 7.5,
+      "learning_rate": 1.8691111111111114e-05,
+      "loss": 0.9083,
+      "mean_token_accuracy": 0.7530260667204857,
+      "num_tokens": 22594618.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.9052924428135156,
+      "epoch": 5.112,
+      "grad_norm": 6.4375,
+      "learning_rate": 1.866888888888889e-05,
+      "loss": 0.876,
+      "mean_token_accuracy": 0.7579276110976935,
+      "num_tokens": 22735034.0,
+      "step": 1600
+    },
+    {
+      "epoch": 5.112,
+      "eval_coding_entropy": 1.5108720426559448,
+      "eval_coding_loss": 2.168074131011963,
+      "eval_coding_mean_token_accuracy": 0.5293694581985474,
+      "eval_coding_num_tokens": 22735034.0,
+      "eval_coding_runtime": 80.6791,
+      "eval_coding_samples_per_second": 6.197,
+      "eval_coding_steps_per_second": 3.099,
+      "step": 1600
+    },
+    {
+      "epoch": 5.112,
+      "eval_math_entropy": 0.9210035672187805,
+      "eval_math_loss": 1.1561440229415894,
+      "eval_math_mean_token_accuracy": 0.7158858454227448,
+      "eval_math_num_tokens": 22735034.0,
+      "eval_math_runtime": 46.0157,
+      "eval_math_samples_per_second": 10.866,
+      "eval_math_steps_per_second": 5.433,
+      "step": 1600
+    },
+    {
+      "entropy": 0.8932699026539922,
+      "epoch": 5.144,
+      "grad_norm": 6.5,
+      "learning_rate": 1.864666666666667e-05,
+      "loss": 0.8714,
+      "mean_token_accuracy": 0.762346514314413,
+      "num_tokens": 22875863.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.8730874558910727,
+      "epoch": 5.176,
+      "grad_norm": 7.3125,
+      "learning_rate": 1.8624444444444446e-05,
+      "loss": 0.8248,
+      "mean_token_accuracy": 0.767206709459424,
+      "num_tokens": 23019827.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.9459875440225005,
+      "epoch": 5.208,
+      "grad_norm": 7.625,
+      "learning_rate": 1.8602222222222224e-05,
+      "loss": 0.9073,
+      "mean_token_accuracy": 0.7505842503160238,
+      "num_tokens": 23160295.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.8937128365039826,
+      "epoch": 5.24,
+      "grad_norm": 6.375,
+      "learning_rate": 1.858e-05,
+      "loss": 0.87,
+      "mean_token_accuracy": 0.7625281598418951,
+      "num_tokens": 23302273.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.8976007211953402,
+      "epoch": 5.272,
+      "grad_norm": 6.75,
+      "learning_rate": 1.855777777777778e-05,
+      "loss": 0.8753,
+      "mean_token_accuracy": 0.7627121699973941,
+      "num_tokens": 23449683.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.8997082270681858,
+      "epoch": 5.304,
+      "grad_norm": 6.5625,
+      "learning_rate": 1.8535555555555557e-05,
+      "loss": 0.888,
+      "mean_token_accuracy": 0.7588404752314091,
+      "num_tokens": 23588004.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.9175663651898504,
+      "epoch": 5.336,
+      "grad_norm": 7.5,
+      "learning_rate": 1.8513333333333335e-05,
+      "loss": 0.8712,
+      "mean_token_accuracy": 0.7588019911199808,
+      "num_tokens": 23735695.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.8917087573558092,
+      "epoch": 5.368,
+      "grad_norm": 6.75,
+      "learning_rate": 1.8491111111111112e-05,
+      "loss": 0.872,
+      "mean_token_accuracy": 0.7614140216261148,
+      "num_tokens": 23878624.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.9097595578059554,
+      "epoch": 5.4,
+      "grad_norm": 6.8125,
+      "learning_rate": 1.846888888888889e-05,
+      "loss": 0.8898,
+      "mean_token_accuracy": 0.7600292358547449,
+      "num_tokens": 24021664.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.8914705088362098,
+      "epoch": 5.432,
+      "grad_norm": 8.75,
+      "learning_rate": 1.8446666666666667e-05,
+      "loss": 0.8629,
+      "mean_token_accuracy": 0.7618724539875984,
+      "num_tokens": 24162453.0,
+      "step": 1700
+    },
+    {
+      "epoch": 5.432,
+      "eval_coding_entropy": 1.5077661447525024,
+      "eval_coding_loss": 2.1740472316741943,
+      "eval_coding_mean_token_accuracy": 0.5293574998378754,
+      "eval_coding_num_tokens": 24162453.0,
+      "eval_coding_runtime": 81.2553,
+      "eval_coding_samples_per_second": 6.153,
+      "eval_coding_steps_per_second": 3.077,
+      "step": 1700
+    },
+    {
+      "epoch": 5.432,
+      "eval_math_entropy": 0.907505418419838,
+      "eval_math_loss": 1.1403419971466064,
+      "eval_math_mean_token_accuracy": 0.7196878392696381,
+      "eval_math_num_tokens": 24162453.0,
+      "eval_math_runtime": 46.3752,
+      "eval_math_samples_per_second": 10.782,
+      "eval_math_steps_per_second": 5.391,
+      "step": 1700
+    },
+    {
+      "entropy": 0.8794696152210235,
+      "epoch": 5.464,
+      "grad_norm": 7.8125,
+      "learning_rate": 1.842444444444445e-05,
+      "loss": 0.8491,
+      "mean_token_accuracy": 0.7637434259057045,
+      "num_tokens": 24302751.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.8770243087783456,
+      "epoch": 5.496,
+      "grad_norm": 6.90625,
+      "learning_rate": 1.8402222222222223e-05,
+      "loss": 0.8569,
+      "mean_token_accuracy": 0.7662156637758016,
+      "num_tokens": 24447316.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.9151918144896627,
+      "epoch": 5.5280000000000005,
+      "grad_norm": 7.9375,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 0.8896,
+      "mean_token_accuracy": 0.7570474561303854,
+      "num_tokens": 24588812.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.9098712190985679,
+      "epoch": 5.5600000000000005,
+      "grad_norm": 6.96875,
+      "learning_rate": 1.8357777777777778e-05,
+      "loss": 0.8752,
+      "mean_token_accuracy": 0.7589137274771929,
+      "num_tokens": 24729042.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.9058109503239393,
+      "epoch": 5.592,
+      "grad_norm": 7.21875,
+      "learning_rate": 1.833555555555556e-05,
+      "loss": 0.8724,
+      "mean_token_accuracy": 0.7591247230768203,
+      "num_tokens": 24861573.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.8967058792710304,
+      "epoch": 5.624,
+      "grad_norm": 7.65625,
+      "learning_rate": 1.8313333333333333e-05,
+      "loss": 0.8772,
+      "mean_token_accuracy": 0.7598790679126978,
+      "num_tokens": 25009913.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.8895885275676847,
+      "epoch": 5.656,
+      "grad_norm": 7.125,
+      "learning_rate": 1.8291111111111114e-05,
+      "loss": 0.8706,
+      "mean_token_accuracy": 0.7640030801296234,
+      "num_tokens": 25148190.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.9247543184086681,
+      "epoch": 5.688,
+      "grad_norm": 7.75,
+      "learning_rate": 1.8268888888888888e-05,
+      "loss": 0.899,
+      "mean_token_accuracy": 0.7559223342686892,
+      "num_tokens": 25291745.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.9217993386089802,
+      "epoch": 5.72,
+      "grad_norm": 7.8125,
+      "learning_rate": 1.824666666666667e-05,
+      "loss": 0.895,
+      "mean_token_accuracy": 0.7565487738698721,
+      "num_tokens": 25434441.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.8966394849121571,
+      "epoch": 5.752,
+      "grad_norm": 8.3125,
+      "learning_rate": 1.8224444444444447e-05,
+      "loss": 0.8758,
+      "mean_token_accuracy": 0.7607072979211807,
+      "num_tokens": 25573574.0,
+      "step": 1800
+    },
+    {
+      "epoch": 5.752,
+      "eval_coding_entropy": 1.5287488384246826,
+      "eval_coding_loss": 2.1693997383117676,
+      "eval_coding_mean_token_accuracy": 0.5300438596010209,
+      "eval_coding_num_tokens": 25573574.0,
+      "eval_coding_runtime": 81.0569,
+      "eval_coding_samples_per_second": 6.169,
+      "eval_coding_steps_per_second": 3.084,
+      "step": 1800
+    },
+    {
+      "epoch": 5.752,
+      "eval_math_entropy": 0.9324754639863968,
+      "eval_math_loss": 1.1225923299789429,
+      "eval_math_mean_token_accuracy": 0.7223285484313965,
+      "eval_math_num_tokens": 25573574.0,
+      "eval_math_runtime": 46.1182,
+      "eval_math_samples_per_second": 10.842,
+      "eval_math_steps_per_second": 5.421,
+      "step": 1800
+    },
+    {
+      "entropy": 0.8990588784217834,
+      "epoch": 5.784,
+      "grad_norm": 6.6875,
+      "learning_rate": 1.8202222222222225e-05,
+      "loss": 0.8611,
+      "mean_token_accuracy": 0.7620632901787758,
+      "num_tokens": 25720633.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.9017492324113846,
+      "epoch": 5.816,
+      "grad_norm": 7.0,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 0.8795,
+      "mean_token_accuracy": 0.7607263036072254,
+      "num_tokens": 25860386.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.8764655807986855,
+      "epoch": 5.848,
+      "grad_norm": 7.09375,
+      "learning_rate": 1.815777777777778e-05,
+      "loss": 0.8454,
+      "mean_token_accuracy": 0.7668800290673972,
+      "num_tokens": 26003831.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.9092245759442449,
+      "epoch": 5.88,
+      "grad_norm": 7.28125,
+      "learning_rate": 1.8135555555555557e-05,
+      "loss": 0.9039,
+      "mean_token_accuracy": 0.7570795770734549,
+      "num_tokens": 26141832.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.9221448406577111,
+      "epoch": 5.912,
+      "grad_norm": 7.6875,
+      "learning_rate": 1.8113333333333335e-05,
+      "loss": 0.889,
+      "mean_token_accuracy": 0.7561331510543823,
+      "num_tokens": 26285027.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.8948194550350308,
+      "epoch": 5.944,
+      "grad_norm": 8.6875,
+      "learning_rate": 1.8091111111111113e-05,
+      "loss": 0.8728,
+      "mean_token_accuracy": 0.760323528200388,
+      "num_tokens": 26432812.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.8912792162969708,
+      "epoch": 5.976,
+      "grad_norm": 7.4375,
+      "learning_rate": 1.806888888888889e-05,
+      "loss": 0.8637,
+      "mean_token_accuracy": 0.7622829657047987,
+      "num_tokens": 26576198.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.8932076808261243,
+      "epoch": 6.0064,
+      "grad_norm": 6.59375,
+      "learning_rate": 1.8046666666666668e-05,
+      "loss": 0.8527,
+      "mean_token_accuracy": 0.7719342869363333,
+      "num_tokens": 26714418.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.7547721007838846,
+      "epoch": 6.0384,
+      "grad_norm": 7.78125,
+      "learning_rate": 1.8024444444444445e-05,
+      "loss": 0.7466,
+      "mean_token_accuracy": 0.7900280602276325,
+      "num_tokens": 26854471.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.8012739174067974,
+      "epoch": 6.0704,
+      "grad_norm": 6.9375,
+      "learning_rate": 1.8002222222222223e-05,
+      "loss": 0.7694,
+      "mean_token_accuracy": 0.7837659023702145,
+      "num_tokens": 26997548.0,
+      "step": 1900
+    },
+    {
+      "epoch": 6.0704,
+      "eval_coding_entropy": 1.4375424737930298,
+      "eval_coding_loss": 2.2412030696868896,
+      "eval_coding_mean_token_accuracy": 0.5258033001422882,
+      "eval_coding_num_tokens": 26997548.0,
+      "eval_coding_runtime": 80.7826,
+      "eval_coding_samples_per_second": 6.189,
+      "eval_coding_steps_per_second": 3.095,
+      "step": 1900
+    },
+    {
+      "epoch": 6.0704,
+      "eval_math_entropy": 0.8488073872327805,
+      "eval_math_loss": 1.1494157314300537,
+      "eval_math_mean_token_accuracy": 0.7230218532085418,
+      "eval_math_num_tokens": 26997548.0,
+      "eval_math_runtime": 46.1166,
+      "eval_math_samples_per_second": 10.842,
+      "eval_math_steps_per_second": 5.421,
+      "step": 1900
+    },
+    {
+      "entropy": 0.7635609801858664,
+      "epoch": 6.1024,
+      "grad_norm": 7.59375,
+      "learning_rate": 1.798e-05,
+      "loss": 0.7384,
+      "mean_token_accuracy": 0.7916049119085073,
+      "num_tokens": 27142854.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.756542045250535,
+      "epoch": 6.1344,
+      "grad_norm": 7.84375,
+      "learning_rate": 1.7957777777777778e-05,
+      "loss": 0.7285,
+      "mean_token_accuracy": 0.7928035575896502,
+      "num_tokens": 27287562.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.7951679166406393,
+      "epoch": 6.1664,
+      "grad_norm": 8.3125,
+      "learning_rate": 1.7935555555555556e-05,
+      "loss": 0.7664,
+      "mean_token_accuracy": 0.7882673852145672,
+      "num_tokens": 27424818.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.7970327842980623,
+      "epoch": 6.1984,
+      "grad_norm": 7.75,
+      "learning_rate": 1.7913333333333337e-05,
+      "loss": 0.7689,
+      "mean_token_accuracy": 0.7828843638300895,
+      "num_tokens": 27562593.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.7904769157990813,
+      "epoch": 6.2304,
+      "grad_norm": 6.84375,
+      "learning_rate": 1.789111111111111e-05,
+      "loss": 0.7573,
+      "mean_token_accuracy": 0.7857714165002108,
+      "num_tokens": 27704822.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.7835376903414726,
+      "epoch": 6.2624,
+      "grad_norm": 6.3125,
+      "learning_rate": 1.7868888888888892e-05,
+      "loss": 0.7641,
+      "mean_token_accuracy": 0.7862238857895136,
+      "num_tokens": 27850462.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.7813543854281306,
+      "epoch": 6.2943999999999996,
+      "grad_norm": 6.09375,
+      "learning_rate": 1.7846666666666666e-05,
+      "loss": 0.7619,
+      "mean_token_accuracy": 0.7848350748419761,
+      "num_tokens": 27995867.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.7751486446708441,
+      "epoch": 6.3264,
+      "grad_norm": 6.53125,
+      "learning_rate": 1.7824444444444447e-05,
+      "loss": 0.7454,
+      "mean_token_accuracy": 0.7895105637609958,
+      "num_tokens": 28141531.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.8145087849348783,
+      "epoch": 6.3584,
+      "grad_norm": 7.1875,
+      "learning_rate": 1.780222222222222e-05,
+      "loss": 0.7932,
+      "mean_token_accuracy": 0.7784016776829958,
+      "num_tokens": 28283236.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.7936712250113487,
+      "epoch": 6.3904,
+      "grad_norm": 7.5625,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 0.7684,
+      "mean_token_accuracy": 0.7840494114905596,
+      "num_tokens": 28424091.0,
+      "step": 2000
+    },
+    {
+      "epoch": 6.3904,
+      "eval_coding_entropy": 1.3973456344604491,
+      "eval_coding_loss": 2.2944459915161133,
+      "eval_coding_mean_token_accuracy": 0.523113137125969,
+      "eval_coding_num_tokens": 28424091.0,
+      "eval_coding_runtime": 80.7861,
+      "eval_coding_samples_per_second": 6.189,
+      "eval_coding_steps_per_second": 3.095,
+      "step": 2000
+    },
+    {
+      "epoch": 6.3904,
+      "eval_math_entropy": 0.8243015093803405,
+      "eval_math_loss": 1.1564151048660278,
+      "eval_math_mean_token_accuracy": 0.7230146683454514,
+      "eval_math_num_tokens": 28424091.0,
+      "eval_math_runtime": 45.6525,
+      "eval_math_samples_per_second": 10.952,
+      "eval_math_steps_per_second": 5.476,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 32,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.8029443350942536e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84264f5831ba25072b080a74ec83ba1e77c8ba298f8cb3e79129fcc274dc2fc0
+size 6289