Upload folder using huggingface_hub

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-1500/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-1500/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-1500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-1500/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-1500/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-1500/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0275679cc6c286897858f4cfda6eafffc1c4d69f8b09e2f41d4f57972c9d9a62
+size 4962221464

checkpoint-1500/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da605e267d18e8bc89c2b293f634aa2b9a76bbd00da98ac2f3bab6410ded9f15
+size 3670322200

checkpoint-1500/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-1500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a378afabf9fa6eac4ea1608a93cd918a53d2d962a3958210522f0856577dd421
+size 7927918951

checkpoint-1500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac3fcfa57afbc11557a4a5c770cfa945864eadeadc093e7e5231772aed0c1ada
+size 14645

checkpoint-1500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb8afd6405fc2f72bcdc3e89147a10597d475bf4457b8f98d9c27aa5442f3ef9
+size 1465

checkpoint-1500/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-1500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-1500/trainer_state.json

@@ -0,0 +1,1864 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.2,
+  "eval_steps": 100,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 3.1496359556913376,
+      "epoch": 0.008,
+      "grad_norm": 298.0,
+      "learning_rate": 1.8e-07,
+      "loss": 7.8456,
+      "mean_token_accuracy": 0.18150889375247062,
+      "num_tokens": 103189.0,
+      "step": 10
+    },
+    {
+      "entropy": 3.2677677035331727,
+      "epoch": 0.016,
+      "grad_norm": 274.0,
+      "learning_rate": 3.8e-07,
+      "loss": 7.8029,
+      "mean_token_accuracy": 0.17940039467066526,
+      "num_tokens": 195510.0,
+      "step": 20
+    },
+    {
+      "entropy": 3.4018275901675223,
+      "epoch": 0.024,
+      "grad_norm": 221.0,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 7.5603,
+      "mean_token_accuracy": 0.18447871217504144,
+      "num_tokens": 297075.0,
+      "step": 30
+    },
+    {
+      "entropy": 3.904408423602581,
+      "epoch": 0.032,
+      "grad_norm": 183.0,
+      "learning_rate": 7.8e-07,
+      "loss": 7.0974,
+      "mean_token_accuracy": 0.1905162119306624,
+      "num_tokens": 391008.0,
+      "step": 40
+    },
+    {
+      "entropy": 4.471200747787952,
+      "epoch": 0.04,
+      "grad_norm": 116.5,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 6.474,
+      "mean_token_accuracy": 0.19736560098826886,
+      "num_tokens": 488721.0,
+      "step": 50
+    },
+    {
+      "entropy": 4.936916092038155,
+      "epoch": 0.048,
+      "grad_norm": 61.75,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 5.8305,
+      "mean_token_accuracy": 0.2095636417157948,
+      "num_tokens": 590136.0,
+      "step": 60
+    },
+    {
+      "entropy": 5.065159273147583,
+      "epoch": 0.056,
+      "grad_norm": 40.75,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 5.3279,
+      "mean_token_accuracy": 0.23331616390496493,
+      "num_tokens": 691017.0,
+      "step": 70
+    },
+    {
+      "entropy": 4.822714896500111,
+      "epoch": 0.064,
+      "grad_norm": 30.625,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 4.93,
+      "mean_token_accuracy": 0.2618246308527887,
+      "num_tokens": 791432.0,
+      "step": 80
+    },
+    {
+      "entropy": 4.613001684844494,
+      "epoch": 0.072,
+      "grad_norm": 30.5,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 4.6595,
+      "mean_token_accuracy": 0.28834385639056564,
+      "num_tokens": 890431.0,
+      "step": 90
+    },
+    {
+      "entropy": 4.42924614995718,
+      "epoch": 0.08,
+      "grad_norm": 25.0,
+      "learning_rate": 1.98e-06,
+      "loss": 4.4406,
+      "mean_token_accuracy": 0.30723202889785173,
+      "num_tokens": 990342.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_coding_entropy": 2.019406507015228,
+      "eval_coding_loss": 2.2580299377441406,
+      "eval_coding_mean_token_accuracy": 0.49473730635643004,
+      "eval_coding_num_tokens": 990342.0,
+      "eval_coding_runtime": 81.2027,
+      "eval_coding_samples_per_second": 6.157,
+      "eval_coding_steps_per_second": 3.079,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_biology_entropy": 4.3503482704162595,
+      "eval_biology_loss": 4.414401531219482,
+      "eval_biology_mean_token_accuracy": 0.3150999557375908,
+      "eval_biology_num_tokens": 990342.0,
+      "eval_biology_runtime": 34.5542,
+      "eval_biology_samples_per_second": 14.47,
+      "eval_biology_steps_per_second": 7.235,
+      "step": 100
+    },
+    {
+      "entropy": 4.333127842843533,
+      "epoch": 0.088,
+      "grad_norm": 24.75,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 4.33,
+      "mean_token_accuracy": 0.3205883141607046,
+      "num_tokens": 1087616.0,
+      "step": 110
+    },
+    {
+      "entropy": 4.196488052606583,
+      "epoch": 0.096,
+      "grad_norm": 21.25,
+      "learning_rate": 2.38e-06,
+      "loss": 4.1941,
+      "mean_token_accuracy": 0.3275640026666224,
+      "num_tokens": 1182339.0,
+      "step": 120
+    },
+    {
+      "entropy": 4.199820376932621,
+      "epoch": 0.104,
+      "grad_norm": 22.25,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 4.1709,
+      "mean_token_accuracy": 0.33598021846264603,
+      "num_tokens": 1284449.0,
+      "step": 130
+    },
+    {
+      "entropy": 4.0606200903654095,
+      "epoch": 0.112,
+      "grad_norm": 22.375,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 4.0429,
+      "mean_token_accuracy": 0.34365201108157634,
+      "num_tokens": 1386193.0,
+      "step": 140
+    },
+    {
+      "entropy": 3.976654475927353,
+      "epoch": 0.12,
+      "grad_norm": 25.125,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 3.9428,
+      "mean_token_accuracy": 0.35409670416265726,
+      "num_tokens": 1484528.0,
+      "step": 150
+    },
+    {
+      "entropy": 3.9824412897229196,
+      "epoch": 0.128,
+      "grad_norm": 26.5,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 3.9225,
+      "mean_token_accuracy": 0.35787415150552987,
+      "num_tokens": 1583261.0,
+      "step": 160
+    },
+    {
+      "entropy": 3.90925519913435,
+      "epoch": 0.136,
+      "grad_norm": 27.75,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 3.8876,
+      "mean_token_accuracy": 0.36578934341669084,
+      "num_tokens": 1678138.0,
+      "step": 170
+    },
+    {
+      "entropy": 3.846222287416458,
+      "epoch": 0.144,
+      "grad_norm": 22.5,
+      "learning_rate": 3.58e-06,
+      "loss": 3.7638,
+      "mean_token_accuracy": 0.37533934097737076,
+      "num_tokens": 1774826.0,
+      "step": 180
+    },
+    {
+      "entropy": 3.7961452692747115,
+      "epoch": 0.152,
+      "grad_norm": 20.625,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 3.7673,
+      "mean_token_accuracy": 0.3747658669948578,
+      "num_tokens": 1874332.0,
+      "step": 190
+    },
+    {
+      "entropy": 3.8614122182130814,
+      "epoch": 0.16,
+      "grad_norm": 23.5,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 3.7987,
+      "mean_token_accuracy": 0.3740364293567836,
+      "num_tokens": 1971310.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_coding_entropy": 1.94223806142807,
+      "eval_coding_loss": 2.1925477981567383,
+      "eval_coding_mean_token_accuracy": 0.5086279969215393,
+      "eval_coding_num_tokens": 1971310.0,
+      "eval_coding_runtime": 80.6501,
+      "eval_coding_samples_per_second": 6.2,
+      "eval_coding_steps_per_second": 3.1,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 3.7338000392913817,
+      "eval_biology_loss": 3.729203939437866,
+      "eval_biology_mean_token_accuracy": 0.38531523847579957,
+      "eval_biology_num_tokens": 1971310.0,
+      "eval_biology_runtime": 34.5728,
+      "eval_biology_samples_per_second": 14.462,
+      "eval_biology_steps_per_second": 7.231,
+      "step": 200
+    },
+    {
+      "entropy": 3.7424988850951193,
+      "epoch": 0.168,
+      "grad_norm": 20.0,
+      "learning_rate": 4.18e-06,
+      "loss": 3.6677,
+      "mean_token_accuracy": 0.38609125521034005,
+      "num_tokens": 2068010.0,
+      "step": 210
+    },
+    {
+      "entropy": 3.7136047944426536,
+      "epoch": 0.176,
+      "grad_norm": 24.5,
+      "learning_rate": 4.38e-06,
+      "loss": 3.6613,
+      "mean_token_accuracy": 0.38672342915087937,
+      "num_tokens": 2169453.0,
+      "step": 220
+    },
+    {
+      "entropy": 3.6596395045518877,
+      "epoch": 0.184,
+      "grad_norm": 23.875,
+      "learning_rate": 4.58e-06,
+      "loss": 3.6248,
+      "mean_token_accuracy": 0.38720098566263916,
+      "num_tokens": 2271716.0,
+      "step": 230
+    },
+    {
+      "entropy": 3.696573592722416,
+      "epoch": 0.192,
+      "grad_norm": 21.25,
+      "learning_rate": 4.78e-06,
+      "loss": 3.6063,
+      "mean_token_accuracy": 0.3891244037076831,
+      "num_tokens": 2366844.0,
+      "step": 240
+    },
+    {
+      "entropy": 3.589663599431515,
+      "epoch": 0.2,
+      "grad_norm": 24.5,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 3.5393,
+      "mean_token_accuracy": 0.3962410934269428,
+      "num_tokens": 2460881.0,
+      "step": 250
+    },
+    {
+      "entropy": 3.576136104762554,
+      "epoch": 0.208,
+      "grad_norm": 21.5,
+      "learning_rate": 5.18e-06,
+      "loss": 3.5108,
+      "mean_token_accuracy": 0.40014535561203957,
+      "num_tokens": 2559708.0,
+      "step": 260
+    },
+    {
+      "entropy": 3.5436186090111734,
+      "epoch": 0.216,
+      "grad_norm": 19.0,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 3.4656,
+      "mean_token_accuracy": 0.40589779019355776,
+      "num_tokens": 2655858.0,
+      "step": 270
+    },
+    {
+      "entropy": 3.516196443140507,
+      "epoch": 0.224,
+      "grad_norm": 27.125,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 3.4628,
+      "mean_token_accuracy": 0.40603156946599483,
+      "num_tokens": 2756533.0,
+      "step": 280
+    },
+    {
+      "entropy": 3.474540175497532,
+      "epoch": 0.232,
+      "grad_norm": 20.75,
+      "learning_rate": 5.78e-06,
+      "loss": 3.3983,
+      "mean_token_accuracy": 0.410215818695724,
+      "num_tokens": 2853451.0,
+      "step": 290
+    },
+    {
+      "entropy": 3.386892372369766,
+      "epoch": 0.24,
+      "grad_norm": 23.125,
+      "learning_rate": 5.98e-06,
+      "loss": 3.3306,
+      "mean_token_accuracy": 0.41694455314427614,
+      "num_tokens": 2948600.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_coding_entropy": 1.9961137461662293,
+      "eval_coding_loss": 2.172008752822876,
+      "eval_coding_mean_token_accuracy": 0.5127531417608261,
+      "eval_coding_num_tokens": 2948600.0,
+      "eval_coding_runtime": 80.7019,
+      "eval_coding_samples_per_second": 6.196,
+      "eval_coding_steps_per_second": 3.098,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_biology_entropy": 3.484758809089661,
+      "eval_biology_loss": 3.406900644302368,
+      "eval_biology_mean_token_accuracy": 0.41982992458343504,
+      "eval_biology_num_tokens": 2948600.0,
+      "eval_biology_runtime": 34.5611,
+      "eval_biology_samples_per_second": 14.467,
+      "eval_biology_steps_per_second": 7.234,
+      "step": 300
+    },
+    {
+      "entropy": 3.495779898762703,
+      "epoch": 0.248,
+      "grad_norm": 21.75,
+      "learning_rate": 6.18e-06,
+      "loss": 3.3982,
+      "mean_token_accuracy": 0.41361558344215155,
+      "num_tokens": 3044471.0,
+      "step": 310
+    },
+    {
+      "entropy": 3.3510731413960455,
+      "epoch": 0.256,
+      "grad_norm": 26.375,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 3.3239,
+      "mean_token_accuracy": 0.41568741872906684,
+      "num_tokens": 3141473.0,
+      "step": 320
+    },
+    {
+      "entropy": 3.40442940890789,
+      "epoch": 0.264,
+      "grad_norm": 25.5,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 3.3236,
+      "mean_token_accuracy": 0.41933552622795106,
+      "num_tokens": 3233662.0,
+      "step": 330
+    },
+    {
+      "entropy": 3.321935287117958,
+      "epoch": 0.272,
+      "grad_norm": 26.75,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 3.2912,
+      "mean_token_accuracy": 0.42016132585704324,
+      "num_tokens": 3337285.0,
+      "step": 340
+    },
+    {
+      "entropy": 3.317689270526171,
+      "epoch": 0.28,
+      "grad_norm": 30.125,
+      "learning_rate": 6.98e-06,
+      "loss": 3.2355,
+      "mean_token_accuracy": 0.42757263109087945,
+      "num_tokens": 3438579.0,
+      "step": 350
+    },
+    {
+      "entropy": 3.284139819443226,
+      "epoch": 0.288,
+      "grad_norm": 25.0,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 3.2066,
+      "mean_token_accuracy": 0.42989938110113146,
+      "num_tokens": 3541221.0,
+      "step": 360
+    },
+    {
+      "entropy": 3.3082104876637457,
+      "epoch": 0.296,
+      "grad_norm": 23.875,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 3.2212,
+      "mean_token_accuracy": 0.43165368046611546,
+      "num_tokens": 3634910.0,
+      "step": 370
+    },
+    {
+      "entropy": 3.1951300844550135,
+      "epoch": 0.304,
+      "grad_norm": 23.25,
+      "learning_rate": 7.58e-06,
+      "loss": 3.1248,
+      "mean_token_accuracy": 0.43907446004450323,
+      "num_tokens": 3734952.0,
+      "step": 380
+    },
+    {
+      "entropy": 3.2739955455064775,
+      "epoch": 0.312,
+      "grad_norm": 30.25,
+      "learning_rate": 7.78e-06,
+      "loss": 3.1763,
+      "mean_token_accuracy": 0.42868839371949435,
+      "num_tokens": 3826750.0,
+      "step": 390
+    },
+    {
+      "entropy": 3.265377101302147,
+      "epoch": 0.32,
+      "grad_norm": 21.75,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 3.1998,
+      "mean_token_accuracy": 0.4301817687228322,
+      "num_tokens": 3922209.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_coding_entropy": 2.0029401898384096,
+      "eval_coding_loss": 2.171020030975342,
+      "eval_coding_mean_token_accuracy": 0.5134941293001175,
+      "eval_coding_num_tokens": 3922209.0,
+      "eval_coding_runtime": 80.6648,
+      "eval_coding_samples_per_second": 6.198,
+      "eval_coding_steps_per_second": 3.099,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 3.2002078475952147,
+      "eval_biology_loss": 3.1781411170959473,
+      "eval_biology_mean_token_accuracy": 0.440620893239975,
+      "eval_biology_num_tokens": 3922209.0,
+      "eval_biology_runtime": 34.4923,
+      "eval_biology_samples_per_second": 14.496,
+      "eval_biology_steps_per_second": 7.248,
+      "step": 400
+    },
+    {
+      "entropy": 3.150459985435009,
+      "epoch": 0.328,
+      "grad_norm": 24.5,
+      "learning_rate": 8.18e-06,
+      "loss": 3.0904,
+      "mean_token_accuracy": 0.44143912978470323,
+      "num_tokens": 4018384.0,
+      "step": 410
+    },
+    {
+      "entropy": 3.1615461230278017,
+      "epoch": 0.336,
+      "grad_norm": 24.75,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 3.1239,
+      "mean_token_accuracy": 0.44076176043599846,
+      "num_tokens": 4115941.0,
+      "step": 420
+    },
+    {
+      "entropy": 3.1629561871290206,
+      "epoch": 0.344,
+      "grad_norm": 23.625,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 3.1118,
+      "mean_token_accuracy": 0.4393952677026391,
+      "num_tokens": 4212212.0,
+      "step": 430
+    },
+    {
+      "entropy": 3.173071324825287,
+      "epoch": 0.352,
+      "grad_norm": 23.375,
+      "learning_rate": 8.78e-06,
+      "loss": 3.0404,
+      "mean_token_accuracy": 0.4486429067328572,
+      "num_tokens": 4308755.0,
+      "step": 440
+    },
+    {
+      "entropy": 3.11899806112051,
+      "epoch": 0.36,
+      "grad_norm": 21.0,
+      "learning_rate": 8.98e-06,
+      "loss": 3.0634,
+      "mean_token_accuracy": 0.4478369506075978,
+      "num_tokens": 4407923.0,
+      "step": 450
+    },
+    {
+      "entropy": 3.0697645485401153,
+      "epoch": 0.368,
+      "grad_norm": 23.375,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 2.9971,
+      "mean_token_accuracy": 0.4510502349585295,
+      "num_tokens": 4505910.0,
+      "step": 460
+    },
+    {
+      "entropy": 3.0845467180013655,
+      "epoch": 0.376,
+      "grad_norm": 22.5,
+      "learning_rate": 9.38e-06,
+      "loss": 2.9916,
+      "mean_token_accuracy": 0.45388368237763643,
+      "num_tokens": 4599609.0,
+      "step": 470
+    },
+    {
+      "entropy": 3.0334658533334733,
+      "epoch": 0.384,
+      "grad_norm": 26.375,
+      "learning_rate": 9.58e-06,
+      "loss": 2.965,
+      "mean_token_accuracy": 0.4527210883796215,
+      "num_tokens": 4696101.0,
+      "step": 480
+    },
+    {
+      "entropy": 3.0365326166152955,
+      "epoch": 0.392,
+      "grad_norm": 24.75,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 2.9627,
+      "mean_token_accuracy": 0.45046965070068834,
+      "num_tokens": 4793234.0,
+      "step": 490
+    },
+    {
+      "entropy": 3.022104251384735,
+      "epoch": 0.4,
+      "grad_norm": 26.625,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 2.9389,
+      "mean_token_accuracy": 0.4577130895107985,
+      "num_tokens": 4889445.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_coding_entropy": 2.023589345932007,
+      "eval_coding_loss": 2.1741902828216553,
+      "eval_coding_mean_token_accuracy": 0.5137698065042495,
+      "eval_coding_num_tokens": 4889445.0,
+      "eval_coding_runtime": 80.6183,
+      "eval_coding_samples_per_second": 6.202,
+      "eval_coding_steps_per_second": 3.101,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_biology_entropy": 3.0067848119735716,
+      "eval_biology_loss": 2.9910171031951904,
+      "eval_biology_mean_token_accuracy": 0.4611110507249832,
+      "eval_biology_num_tokens": 4889445.0,
+      "eval_biology_runtime": 34.2723,
+      "eval_biology_samples_per_second": 14.589,
+      "eval_biology_steps_per_second": 7.295,
+      "step": 500
+    },
+    {
+      "entropy": 3.043118818104267,
+      "epoch": 0.408,
+      "grad_norm": 21.875,
+      "learning_rate": 1.018e-05,
+      "loss": 2.9591,
+      "mean_token_accuracy": 0.4550719428807497,
+      "num_tokens": 4987724.0,
+      "step": 510
+    },
+    {
+      "entropy": 3.051181846857071,
+      "epoch": 0.416,
+      "grad_norm": 23.0,
+      "learning_rate": 1.038e-05,
+      "loss": 2.9554,
+      "mean_token_accuracy": 0.45290321484208107,
+      "num_tokens": 5087624.0,
+      "step": 520
+    },
+    {
+      "entropy": 3.023839880526066,
+      "epoch": 0.424,
+      "grad_norm": 37.25,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 2.9739,
+      "mean_token_accuracy": 0.45466579012572766,
+      "num_tokens": 5178478.0,
+      "step": 530
+    },
+    {
+      "entropy": 2.9455194897949695,
+      "epoch": 0.432,
+      "grad_norm": 24.875,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 2.8345,
+      "mean_token_accuracy": 0.4659645749256015,
+      "num_tokens": 5273794.0,
+      "step": 540
+    },
+    {
+      "entropy": 3.00012748837471,
+      "epoch": 0.44,
+      "grad_norm": 27.5,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 2.9082,
+      "mean_token_accuracy": 0.46043862625956533,
+      "num_tokens": 5365232.0,
+      "step": 550
+    },
+    {
+      "entropy": 2.951145689189434,
+      "epoch": 0.448,
+      "grad_norm": 25.75,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 2.8758,
+      "mean_token_accuracy": 0.4649818094447255,
+      "num_tokens": 5463288.0,
+      "step": 560
+    },
+    {
+      "entropy": 2.9195503376424314,
+      "epoch": 0.456,
+      "grad_norm": 24.5,
+      "learning_rate": 1.138e-05,
+      "loss": 2.8283,
+      "mean_token_accuracy": 0.46871429681777954,
+      "num_tokens": 5561317.0,
+      "step": 570
+    },
+    {
+      "entropy": 2.895768219232559,
+      "epoch": 0.464,
+      "grad_norm": 23.0,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 2.8347,
+      "mean_token_accuracy": 0.4678240044042468,
+      "num_tokens": 5656827.0,
+      "step": 580
+    },
+    {
+      "entropy": 2.919021227955818,
+      "epoch": 0.472,
+      "grad_norm": 21.875,
+      "learning_rate": 1.178e-05,
+      "loss": 2.8239,
+      "mean_token_accuracy": 0.46788553427904844,
+      "num_tokens": 5754593.0,
+      "step": 590
+    },
+    {
+      "entropy": 2.9184277661144735,
+      "epoch": 0.48,
+      "grad_norm": 19.375,
+      "learning_rate": 1.198e-05,
+      "loss": 2.8284,
+      "mean_token_accuracy": 0.46977027915418146,
+      "num_tokens": 5853311.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_coding_entropy": 1.9910136852264404,
+      "eval_coding_loss": 2.1933281421661377,
+      "eval_coding_mean_token_accuracy": 0.5116564213037491,
+      "eval_coding_num_tokens": 5853311.0,
+      "eval_coding_runtime": 80.6686,
+      "eval_coding_samples_per_second": 6.198,
+      "eval_coding_steps_per_second": 3.099,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 2.7489197278022766,
+      "eval_biology_loss": 2.839024305343628,
+      "eval_biology_mean_token_accuracy": 0.478046000957489,
+      "eval_biology_num_tokens": 5853311.0,
+      "eval_biology_runtime": 34.524,
+      "eval_biology_samples_per_second": 14.483,
+      "eval_biology_steps_per_second": 7.241,
+      "step": 600
+    },
+    {
+      "entropy": 2.761022947728634,
+      "epoch": 0.488,
+      "grad_norm": 23.125,
+      "learning_rate": 1.218e-05,
+      "loss": 2.7114,
+      "mean_token_accuracy": 0.48137810807675124,
+      "num_tokens": 5952079.0,
+      "step": 610
+    },
+    {
+      "entropy": 2.802033407241106,
+      "epoch": 0.496,
+      "grad_norm": 24.875,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 2.7172,
+      "mean_token_accuracy": 0.480432684533298,
+      "num_tokens": 6049788.0,
+      "step": 620
+    },
+    {
+      "entropy": 2.8570902943611145,
+      "epoch": 0.504,
+      "grad_norm": 24.375,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 2.7682,
+      "mean_token_accuracy": 0.4740563785657287,
+      "num_tokens": 6142124.0,
+      "step": 630
+    },
+    {
+      "entropy": 2.803235235065222,
+      "epoch": 0.512,
+      "grad_norm": 19.25,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 2.7612,
+      "mean_token_accuracy": 0.48143365047872066,
+      "num_tokens": 6243825.0,
+      "step": 640
+    },
+    {
+      "entropy": 2.8179355546832086,
+      "epoch": 0.52,
+      "grad_norm": 24.375,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 2.7405,
+      "mean_token_accuracy": 0.47674270141869785,
+      "num_tokens": 6337416.0,
+      "step": 650
+    },
+    {
+      "entropy": 2.783262898027897,
+      "epoch": 0.528,
+      "grad_norm": 23.5,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 2.6905,
+      "mean_token_accuracy": 0.48337399754673244,
+      "num_tokens": 6434188.0,
+      "step": 660
+    },
+    {
+      "entropy": 2.7997302234172823,
+      "epoch": 0.536,
+      "grad_norm": 18.25,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 2.7293,
+      "mean_token_accuracy": 0.4824898593127728,
+      "num_tokens": 6536879.0,
+      "step": 670
+    },
+    {
+      "entropy": 2.8198735982179643,
+      "epoch": 0.544,
+      "grad_norm": 23.75,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 2.7448,
+      "mean_token_accuracy": 0.48089918941259385,
+      "num_tokens": 6629832.0,
+      "step": 680
+    },
+    {
+      "entropy": 2.7923016041517257,
+      "epoch": 0.552,
+      "grad_norm": 21.125,
+      "learning_rate": 1.378e-05,
+      "loss": 2.7259,
+      "mean_token_accuracy": 0.4828041112050414,
+      "num_tokens": 6723756.0,
+      "step": 690
+    },
+    {
+      "entropy": 2.6786952562630177,
+      "epoch": 0.56,
+      "grad_norm": 17.625,
+      "learning_rate": 1.398e-05,
+      "loss": 2.6311,
+      "mean_token_accuracy": 0.49184530321508646,
+      "num_tokens": 6824668.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_coding_entropy": 2.0318734068870543,
+      "eval_coding_loss": 2.1769535541534424,
+      "eval_coding_mean_token_accuracy": 0.5146223112344742,
+      "eval_coding_num_tokens": 6824668.0,
+      "eval_coding_runtime": 80.6202,
+      "eval_coding_samples_per_second": 6.202,
+      "eval_coding_steps_per_second": 3.101,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_biology_entropy": 2.7879175729751586,
+      "eval_biology_loss": 2.7142910957336426,
+      "eval_biology_mean_token_accuracy": 0.49161617302894595,
+      "eval_biology_num_tokens": 6824668.0,
+      "eval_biology_runtime": 34.5513,
+      "eval_biology_samples_per_second": 14.471,
+      "eval_biology_steps_per_second": 7.236,
+      "step": 700
+    },
+    {
+      "entropy": 2.731781490147114,
+      "epoch": 0.568,
+      "grad_norm": 22.0,
+      "learning_rate": 1.418e-05,
+      "loss": 2.6484,
+      "mean_token_accuracy": 0.4882859470322728,
+      "num_tokens": 6928939.0,
+      "step": 710
+    },
+    {
+      "entropy": 2.7027835391461847,
+      "epoch": 0.576,
+      "grad_norm": 20.375,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 2.6258,
+      "mean_token_accuracy": 0.48718241080641744,
+      "num_tokens": 7025923.0,
+      "step": 720
+    },
+    {
+      "entropy": 2.697667721658945,
+      "epoch": 0.584,
+      "grad_norm": 24.25,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 2.6369,
+      "mean_token_accuracy": 0.4901735274121165,
+      "num_tokens": 7122614.0,
+      "step": 730
+    },
+    {
+      "entropy": 2.6893031299114227,
+      "epoch": 0.592,
+      "grad_norm": 20.625,
+      "learning_rate": 1.478e-05,
+      "loss": 2.5922,
+      "mean_token_accuracy": 0.49478939212858675,
+      "num_tokens": 7218224.0,
+      "step": 740
+    },
+    {
+      "entropy": 2.7066168434917928,
+      "epoch": 0.6,
+      "grad_norm": 31.375,
+      "learning_rate": 1.498e-05,
+      "loss": 2.6573,
+      "mean_token_accuracy": 0.4925660764798522,
+      "num_tokens": 7322313.0,
+      "step": 750
+    },
+    {
+      "entropy": 2.69825324267149,
+      "epoch": 0.608,
+      "grad_norm": 21.25,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 2.6359,
+      "mean_token_accuracy": 0.48993428722023963,
+      "num_tokens": 7421216.0,
+      "step": 760
+    },
+    {
+      "entropy": 2.649106864631176,
+      "epoch": 0.616,
+      "grad_norm": 22.375,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 2.5961,
+      "mean_token_accuracy": 0.4944638149812818,
+      "num_tokens": 7518453.0,
+      "step": 770
+    },
+    {
+      "entropy": 2.6871195189654826,
+      "epoch": 0.624,
+      "grad_norm": 25.0,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 2.5837,
+      "mean_token_accuracy": 0.4975929221138358,
+      "num_tokens": 7617258.0,
+      "step": 780
+    },
+    {
+      "entropy": 2.6687942549586294,
+      "epoch": 0.632,
+      "grad_norm": 20.25,
+      "learning_rate": 1.578e-05,
+      "loss": 2.607,
+      "mean_token_accuracy": 0.4943814314901829,
+      "num_tokens": 7714799.0,
+      "step": 790
+    },
+    {
+      "entropy": 2.679428345710039,
+      "epoch": 0.64,
+      "grad_norm": 22.125,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 2.5938,
+      "mean_token_accuracy": 0.4952407155185938,
+      "num_tokens": 7806173.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_coding_entropy": 1.9768212442398072,
+      "eval_coding_loss": 2.224097967147827,
+      "eval_coding_mean_token_accuracy": 0.5107745778560638,
+      "eval_coding_num_tokens": 7806173.0,
+      "eval_coding_runtime": 80.6252,
+      "eval_coding_samples_per_second": 6.202,
+      "eval_coding_steps_per_second": 3.101,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 2.545660098552704,
+      "eval_biology_loss": 2.5940585136413574,
+      "eval_biology_mean_token_accuracy": 0.5062511616945267,
+      "eval_biology_num_tokens": 7806173.0,
+      "eval_biology_runtime": 34.513,
+      "eval_biology_samples_per_second": 14.487,
+      "eval_biology_steps_per_second": 7.244,
+      "step": 800
+    },
+    {
+      "entropy": 2.6028101302683355,
+      "epoch": 0.648,
+      "grad_norm": 21.125,
+      "learning_rate": 1.618e-05,
+      "loss": 2.5534,
+      "mean_token_accuracy": 0.49852073900401594,
+      "num_tokens": 7903614.0,
+      "step": 810
+    },
+    {
+      "entropy": 2.665095742046833,
+      "epoch": 0.656,
+      "grad_norm": 20.5,
+      "learning_rate": 1.638e-05,
+      "loss": 2.6054,
+      "mean_token_accuracy": 0.49321497566998007,
+      "num_tokens": 8000514.0,
+      "step": 820
+    },
+    {
+      "entropy": 2.5793248772621156,
+      "epoch": 0.664,
+      "grad_norm": 17.875,
+      "learning_rate": 1.658e-05,
+      "loss": 2.5159,
+      "mean_token_accuracy": 0.5033486964181065,
+      "num_tokens": 8096743.0,
+      "step": 830
+    },
+    {
+      "entropy": 2.641881722956896,
+      "epoch": 0.672,
+      "grad_norm": 23.875,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 2.5868,
+      "mean_token_accuracy": 0.4976224903017282,
+      "num_tokens": 8187884.0,
+      "step": 840
+    },
+    {
+      "entropy": 2.5881785310804846,
+      "epoch": 0.68,
+      "grad_norm": 15.3125,
+      "learning_rate": 1.698e-05,
+      "loss": 2.5281,
+      "mean_token_accuracy": 0.504815262183547,
+      "num_tokens": 8288387.0,
+      "step": 850
+    },
+    {
+      "entropy": 2.5754016518592833,
+      "epoch": 0.688,
+      "grad_norm": 16.25,
+      "learning_rate": 1.718e-05,
+      "loss": 2.499,
+      "mean_token_accuracy": 0.5069523887708783,
+      "num_tokens": 8390643.0,
+      "step": 860
+    },
+    {
+      "entropy": 2.6078361086547375,
+      "epoch": 0.696,
+      "grad_norm": 20.625,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 2.5317,
+      "mean_token_accuracy": 0.5026106856763363,
+      "num_tokens": 8484952.0,
+      "step": 870
+    },
+    {
+      "entropy": 2.5303950123488903,
+      "epoch": 0.704,
+      "grad_norm": 21.5,
+      "learning_rate": 1.758e-05,
+      "loss": 2.4753,
+      "mean_token_accuracy": 0.5120382806286216,
+      "num_tokens": 8583266.0,
+      "step": 880
+    },
+    {
+      "entropy": 2.4733798749744893,
+      "epoch": 0.712,
+      "grad_norm": 18.875,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 2.3787,
+      "mean_token_accuracy": 0.5168206626549363,
+      "num_tokens": 8684764.0,
+      "step": 890
+    },
+    {
+      "entropy": 2.4695195190608503,
+      "epoch": 0.72,
+      "grad_norm": 19.5,
+      "learning_rate": 1.798e-05,
+      "loss": 2.4147,
+      "mean_token_accuracy": 0.5180861415341497,
+      "num_tokens": 8786233.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_coding_entropy": 1.9488533411026001,
+      "eval_coding_loss": 2.2393529415130615,
+      "eval_coding_mean_token_accuracy": 0.5095039163827896,
+      "eval_coding_num_tokens": 8786233.0,
+      "eval_coding_runtime": 80.6302,
+      "eval_coding_samples_per_second": 6.201,
+      "eval_coding_steps_per_second": 3.101,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_biology_entropy": 2.453756254196167,
+      "eval_biology_loss": 2.5033159255981445,
+      "eval_biology_mean_token_accuracy": 0.5165660305023193,
+      "eval_biology_num_tokens": 8786233.0,
+      "eval_biology_runtime": 34.5573,
+      "eval_biology_samples_per_second": 14.469,
+      "eval_biology_steps_per_second": 7.234,
+      "step": 900
+    },
+    {
+      "entropy": 2.4767140075564384,
+      "epoch": 0.728,
+      "grad_norm": 16.625,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 2.4331,
+      "mean_token_accuracy": 0.5123517647385597,
+      "num_tokens": 8881233.0,
+      "step": 910
+    },
+    {
+      "entropy": 2.475684958696365,
+      "epoch": 0.736,
+      "grad_norm": 22.5,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 2.3936,
+      "mean_token_accuracy": 0.5183122556656599,
+      "num_tokens": 8982898.0,
+      "step": 920
+    },
+    {
+      "entropy": 2.495747657865286,
+      "epoch": 0.744,
+      "grad_norm": 17.5,
+      "learning_rate": 1.858e-05,
+      "loss": 2.3898,
+      "mean_token_accuracy": 0.5161861320957541,
+      "num_tokens": 9078921.0,
+      "step": 930
+    },
+    {
+      "entropy": 2.4943869531154634,
+      "epoch": 0.752,
+      "grad_norm": 17.0,
+      "learning_rate": 1.878e-05,
+      "loss": 2.4483,
+      "mean_token_accuracy": 0.5111985813826323,
+      "num_tokens": 9174361.0,
+      "step": 940
+    },
+    {
+      "entropy": 2.4575762204825877,
+      "epoch": 0.76,
+      "grad_norm": 16.0,
+      "learning_rate": 1.898e-05,
+      "loss": 2.4013,
+      "mean_token_accuracy": 0.5162310445681214,
+      "num_tokens": 9272571.0,
+      "step": 950
+    },
+    {
+      "entropy": 2.431351166218519,
+      "epoch": 0.768,
+      "grad_norm": 16.875,
+      "learning_rate": 1.918e-05,
+      "loss": 2.3378,
+      "mean_token_accuracy": 0.521942549943924,
+      "num_tokens": 9373430.0,
+      "step": 960
+    },
+    {
+      "entropy": 2.4587918162345885,
+      "epoch": 0.776,
+      "grad_norm": 15.125,
+      "learning_rate": 1.938e-05,
+      "loss": 2.4059,
+      "mean_token_accuracy": 0.5153441710397602,
+      "num_tokens": 9468903.0,
+      "step": 970
+    },
+    {
+      "entropy": 2.425259476900101,
+      "epoch": 0.784,
+      "grad_norm": 16.875,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 2.3285,
+      "mean_token_accuracy": 0.5206136658787728,
+      "num_tokens": 9570409.0,
+      "step": 980
+    },
+    {
+      "entropy": 2.42026736587286,
+      "epoch": 0.792,
+      "grad_norm": 15.9375,
+      "learning_rate": 1.978e-05,
+      "loss": 2.3534,
+      "mean_token_accuracy": 0.5174617141485214,
+      "num_tokens": 9669925.0,
+      "step": 990
+    },
+    {
+      "entropy": 2.4630366936326027,
+      "epoch": 0.8,
+      "grad_norm": 22.125,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 2.426,
+      "mean_token_accuracy": 0.5160385722294449,
+      "num_tokens": 9766360.0,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_coding_entropy": 2.0347842559814455,
+      "eval_coding_loss": 2.295905351638794,
+      "eval_coding_mean_token_accuracy": 0.5001156101226807,
+      "eval_coding_num_tokens": 9766360.0,
+      "eval_coding_runtime": 80.6454,
+      "eval_coding_samples_per_second": 6.2,
+      "eval_coding_steps_per_second": 3.1,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 2.336130635738373,
+      "eval_biology_loss": 2.4337234497070312,
+      "eval_biology_mean_token_accuracy": 0.5239072642326355,
+      "eval_biology_num_tokens": 9766360.0,
+      "eval_biology_runtime": 34.2599,
+      "eval_biology_samples_per_second": 14.594,
+      "eval_biology_steps_per_second": 7.297,
+      "step": 1000
+    },
+    {
+      "entropy": 2.3636431239545344,
+      "epoch": 0.808,
+      "grad_norm": 18.125,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 2.3056,
+      "mean_token_accuracy": 0.5313040925189852,
+      "num_tokens": 9868563.0,
+      "step": 1010
+    },
+    {
+      "entropy": 2.395121469348669,
+      "epoch": 0.816,
+      "grad_norm": 19.25,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 2.3484,
+      "mean_token_accuracy": 0.5250098271295428,
+      "num_tokens": 9963834.0,
+      "step": 1020
+    },
+    {
+      "entropy": 2.4034232169389727,
+      "epoch": 0.824,
+      "grad_norm": 15.75,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 2.3392,
+      "mean_token_accuracy": 0.5230668963864445,
+      "num_tokens": 10064222.0,
+      "step": 1030
+    },
+    {
+      "entropy": 2.351021596044302,
+      "epoch": 0.832,
+      "grad_norm": 25.5,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 2.2799,
+      "mean_token_accuracy": 0.5308501249179244,
+      "num_tokens": 10160950.0,
+      "step": 1040
+    },
+    {
+      "entropy": 2.3506353706121446,
+      "epoch": 0.84,
+      "grad_norm": 17.375,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 2.2904,
+      "mean_token_accuracy": 0.5276814445853233,
+      "num_tokens": 10261255.0,
+      "step": 1050
+    },
+    {
+      "entropy": 2.330299597978592,
+      "epoch": 0.848,
+      "grad_norm": 14.625,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 2.2807,
+      "mean_token_accuracy": 0.533601225912571,
+      "num_tokens": 10363131.0,
+      "step": 1060
+    },
+    {
+      "entropy": 2.310487486422062,
+      "epoch": 0.856,
+      "grad_norm": 12.6875,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 2.2289,
+      "mean_token_accuracy": 0.5375016804784536,
+      "num_tokens": 10459821.0,
+      "step": 1070
+    },
+    {
+      "entropy": 2.3588565312325955,
+      "epoch": 0.864,
+      "grad_norm": 17.5,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 2.3075,
+      "mean_token_accuracy": 0.5270110530778765,
+      "num_tokens": 10558003.0,
+      "step": 1080
+    },
+    {
+      "entropy": 2.3483712531626226,
+      "epoch": 0.872,
+      "grad_norm": 15.6875,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 2.2938,
+      "mean_token_accuracy": 0.5279141725972295,
+      "num_tokens": 10657009.0,
+      "step": 1090
+    },
+    {
+      "entropy": 2.301337753236294,
+      "epoch": 0.88,
+      "grad_norm": 16.125,
+      "learning_rate": 1.978e-05,
+      "loss": 2.2388,
+      "mean_token_accuracy": 0.5370641218498349,
+      "num_tokens": 10754470.0,
+      "step": 1100
+    },
+    {
+      "epoch": 0.88,
+      "eval_coding_entropy": 2.0812352204322817,
+      "eval_coding_loss": 2.270843744277954,
+      "eval_coding_mean_token_accuracy": 0.501986554980278,
+      "eval_coding_num_tokens": 10754470.0,
+      "eval_coding_runtime": 80.6786,
+      "eval_coding_samples_per_second": 6.197,
+      "eval_coding_steps_per_second": 3.099,
+      "step": 1100
+    },
+    {
+      "epoch": 0.88,
+      "eval_biology_entropy": 2.3388767604827883,
+      "eval_biology_loss": 2.3078601360321045,
+      "eval_biology_mean_token_accuracy": 0.5390308572053909,
+      "eval_biology_num_tokens": 10754470.0,
+      "eval_biology_runtime": 34.486,
+      "eval_biology_samples_per_second": 14.499,
+      "eval_biology_steps_per_second": 7.249,
+      "step": 1100
+    },
+    {
+      "entropy": 2.318739663809538,
+      "epoch": 0.888,
+      "grad_norm": 14.0,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 2.2605,
+      "mean_token_accuracy": 0.5366495896130801,
+      "num_tokens": 10858512.0,
+      "step": 1110
+    },
+    {
+      "entropy": 2.345344864577055,
+      "epoch": 0.896,
+      "grad_norm": 19.25,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 2.3028,
+      "mean_token_accuracy": 0.5301774218678474,
+      "num_tokens": 10948943.0,
+      "step": 1120
+    },
+    {
+      "entropy": 2.305903819203377,
+      "epoch": 0.904,
+      "grad_norm": 13.6875,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 2.2382,
+      "mean_token_accuracy": 0.5397843992337584,
+      "num_tokens": 11043458.0,
+      "step": 1130
+    },
+    {
+      "entropy": 2.2818943060934544,
+      "epoch": 0.912,
+      "grad_norm": 14.0625,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 2.193,
+      "mean_token_accuracy": 0.5407467754557729,
+      "num_tokens": 11142613.0,
+      "step": 1140
+    },
+    {
+      "entropy": 2.3112077616155147,
+      "epoch": 0.92,
+      "grad_norm": 13.0,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 2.2325,
+      "mean_token_accuracy": 0.5370767971500754,
+      "num_tokens": 11243618.0,
+      "step": 1150
+    },
+    {
+      "entropy": 2.293438446521759,
+      "epoch": 0.928,
+      "grad_norm": 13.75,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 2.2228,
+      "mean_token_accuracy": 0.5370372215285897,
+      "num_tokens": 11336779.0,
+      "step": 1160
+    },
+    {
+      "entropy": 2.2055763468146323,
+      "epoch": 0.936,
+      "grad_norm": 11.5,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 2.1619,
+      "mean_token_accuracy": 0.5508569484576583,
+      "num_tokens": 11441397.0,
+      "step": 1170
+    },
+    {
+      "entropy": 2.2443367682397364,
+      "epoch": 0.944,
+      "grad_norm": 14.0,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 2.1774,
+      "mean_token_accuracy": 0.5465354908257722,
+      "num_tokens": 11536381.0,
+      "step": 1180
+    },
+    {
+      "entropy": 2.220568550378084,
+      "epoch": 0.952,
+      "grad_norm": 12.3125,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 2.143,
+      "mean_token_accuracy": 0.5484613511711359,
+      "num_tokens": 11634931.0,
+      "step": 1190
+    },
+    {
+      "entropy": 2.252422795444727,
+      "epoch": 0.96,
+      "grad_norm": 15.0,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 2.1688,
+      "mean_token_accuracy": 0.5444858035072684,
+      "num_tokens": 11735527.0,
+      "step": 1200
+    },
+    {
+      "epoch": 0.96,
+      "eval_coding_entropy": 2.028546452522278,
+      "eval_coding_loss": 2.313232898712158,
+      "eval_coding_mean_token_accuracy": 0.4952474715709686,
+      "eval_coding_num_tokens": 11735527.0,
+      "eval_coding_runtime": 80.6645,
+      "eval_coding_samples_per_second": 6.199,
+      "eval_coding_steps_per_second": 3.099,
+      "step": 1200
+    },
+    {
+      "epoch": 0.96,
+      "eval_biology_entropy": 2.2140285110473634,
+      "eval_biology_loss": 2.2470810413360596,
+      "eval_biology_mean_token_accuracy": 0.5471095204353332,
+      "eval_biology_num_tokens": 11735527.0,
+      "eval_biology_runtime": 34.5057,
+      "eval_biology_samples_per_second": 14.49,
+      "eval_biology_steps_per_second": 7.245,
+      "step": 1200
+    },
+    {
+      "entropy": 2.200479466468096,
+      "epoch": 0.968,
+      "grad_norm": 12.0,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 2.1626,
+      "mean_token_accuracy": 0.5465732211247086,
+      "num_tokens": 11829912.0,
+      "step": 1210
+    },
+    {
+      "entropy": 2.210404744744301,
+      "epoch": 0.976,
+      "grad_norm": 12.875,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 2.1098,
+      "mean_token_accuracy": 0.5487013891339302,
+      "num_tokens": 11930935.0,
+      "step": 1220
+    },
+    {
+      "entropy": 2.2029160290956495,
+      "epoch": 0.984,
+      "grad_norm": 11.5625,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 2.1556,
+      "mean_token_accuracy": 0.5468338681384921,
+      "num_tokens": 12028616.0,
+      "step": 1230
+    },
+    {
+      "entropy": 2.2255366176366804,
+      "epoch": 0.992,
+      "grad_norm": 12.0,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 2.1711,
+      "mean_token_accuracy": 0.547148628346622,
+      "num_tokens": 12129585.0,
+      "step": 1240
+    },
+    {
+      "entropy": 2.2456682115793227,
+      "epoch": 1.0,
+      "grad_norm": 11.5,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 2.1789,
+      "mean_token_accuracy": 0.544199176505208,
+      "num_tokens": 12227772.0,
+      "step": 1250
+    },
+    {
+      "entropy": 2.04770105406642,
+      "epoch": 1.008,
+      "grad_norm": 10.5,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 1.9599,
+      "mean_token_accuracy": 0.5732083506882191,
+      "num_tokens": 12325847.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.9857938677072524,
+      "epoch": 1.016,
+      "grad_norm": 10.6875,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 1.9223,
+      "mean_token_accuracy": 0.5803239606320858,
+      "num_tokens": 12428864.0,
+      "step": 1270
+    },
+    {
+      "entropy": 2.0265946321189405,
+      "epoch": 1.024,
+      "grad_norm": 14.0625,
+      "learning_rate": 1.938e-05,
+      "loss": 1.9706,
+      "mean_token_accuracy": 0.5730046048760414,
+      "num_tokens": 12530820.0,
+      "step": 1280
+    },
+    {
+      "entropy": 2.033829855918884,
+      "epoch": 1.032,
+      "grad_norm": 12.125,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 1.9723,
+      "mean_token_accuracy": 0.5693843860179186,
+      "num_tokens": 12628246.0,
+      "step": 1290
+    },
+    {
+      "entropy": 2.1004927776753903,
+      "epoch": 1.04,
+      "grad_norm": 11.6875,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 2.0295,
+      "mean_token_accuracy": 0.5636359706521035,
+      "num_tokens": 12725384.0,
+      "step": 1300
+    },
+    {
+      "epoch": 1.04,
+      "eval_coding_entropy": 1.9919977383613587,
+      "eval_coding_loss": 2.314964532852173,
+      "eval_coding_mean_token_accuracy": 0.49944881129264834,
+      "eval_coding_num_tokens": 12725384.0,
+      "eval_coding_runtime": 80.772,
+      "eval_coding_samples_per_second": 6.19,
+      "eval_coding_steps_per_second": 3.095,
+      "step": 1300
+    },
+    {
+      "epoch": 1.04,
+      "eval_biology_entropy": 2.0739205145835875,
+      "eval_biology_loss": 2.1763668060302734,
+      "eval_biology_mean_token_accuracy": 0.5573548142910003,
+      "eval_biology_num_tokens": 12725384.0,
+      "eval_biology_runtime": 34.5814,
+      "eval_biology_samples_per_second": 14.459,
+      "eval_biology_steps_per_second": 7.229,
+      "step": 1300
+    },
+    {
+      "entropy": 2.0259235687553883,
+      "epoch": 1.048,
+      "grad_norm": 11.0625,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 1.9459,
+      "mean_token_accuracy": 0.5764682356268167,
+      "num_tokens": 12824389.0,
+      "step": 1310
+    },
+    {
+      "entropy": 1.9896535106003284,
+      "epoch": 1.056,
+      "grad_norm": 10.9375,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 1.9418,
+      "mean_token_accuracy": 0.5765218947082758,
+      "num_tokens": 12924592.0,
+      "step": 1320
+    },
+    {
+      "entropy": 2.024248744547367,
+      "epoch": 1.064,
+      "grad_norm": 14.0,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 1.9566,
+      "mean_token_accuracy": 0.5732226436957717,
+      "num_tokens": 13019864.0,
+      "step": 1330
+    },
+    {
+      "entropy": 2.0493085965514184,
+      "epoch": 1.072,
+      "grad_norm": 11.6875,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 1.9728,
+      "mean_token_accuracy": 0.5697184054180979,
+      "num_tokens": 13112848.0,
+      "step": 1340
+    },
+    {
+      "entropy": 2.009561914205551,
+      "epoch": 1.08,
+      "grad_norm": 12.5625,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 1.9588,
+      "mean_token_accuracy": 0.5739864658564329,
+      "num_tokens": 13208593.0,
+      "step": 1350
+    },
+    {
+      "entropy": 2.0082264699041845,
+      "epoch": 1.088,
+      "grad_norm": 12.3125,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 1.9356,
+      "mean_token_accuracy": 0.5787085667252541,
+      "num_tokens": 13305725.0,
+      "step": 1360
+    },
+    {
+      "entropy": 2.050854667276144,
+      "epoch": 1.096,
+      "grad_norm": 13.25,
+      "learning_rate": 1.918e-05,
+      "loss": 2.0031,
+      "mean_token_accuracy": 0.568497353605926,
+      "num_tokens": 13397615.0,
+      "step": 1370
+    },
+    {
+      "entropy": 1.9909487955272198,
+      "epoch": 1.104,
+      "grad_norm": 11.5,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 1.9246,
+      "mean_token_accuracy": 0.5775162002071739,
+      "num_tokens": 13498685.0,
+      "step": 1380
+    },
+    {
+      "entropy": 2.022812781482935,
+      "epoch": 1.112,
+      "grad_norm": 10.4375,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 1.9621,
+      "mean_token_accuracy": 0.5735303290188313,
+      "num_tokens": 13587398.0,
+      "step": 1390
+    },
+    {
+      "entropy": 1.9650254637002944,
+      "epoch": 1.12,
+      "grad_norm": 12.1875,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 1.8716,
+      "mean_token_accuracy": 0.5869596699252725,
+      "num_tokens": 13682146.0,
+      "step": 1400
+    },
+    {
+      "epoch": 1.12,
+      "eval_coding_entropy": 1.8874374437332153,
+      "eval_coding_loss": 2.3130838871002197,
+      "eval_coding_mean_token_accuracy": 0.4985515549182892,
+      "eval_coding_num_tokens": 13682146.0,
+      "eval_coding_runtime": 80.6107,
+      "eval_coding_samples_per_second": 6.203,
+      "eval_coding_steps_per_second": 3.101,
+      "step": 1400
+    },
+    {
+      "epoch": 1.12,
+      "eval_biology_entropy": 1.9534716877937317,
+      "eval_biology_loss": 2.117032051086426,
+      "eval_biology_mean_token_accuracy": 0.5663710038661957,
+      "eval_biology_num_tokens": 13682146.0,
+      "eval_biology_runtime": 34.5491,
+      "eval_biology_samples_per_second": 14.472,
+      "eval_biology_steps_per_second": 7.236,
+      "step": 1400
+    },
+    {
+      "entropy": 1.9430220112204553,
+      "epoch": 1.1280000000000001,
+      "grad_norm": 12.5625,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 1.9125,
+      "mean_token_accuracy": 0.5838606007397175,
+      "num_tokens": 13781647.0,
+      "step": 1410
+    },
+    {
+      "entropy": 1.9991029907017945,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 13.25,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 1.9323,
+      "mean_token_accuracy": 0.5773349234834313,
+      "num_tokens": 13876656.0,
+      "step": 1420
+    },
+    {
+      "entropy": 1.9905764922499656,
+      "epoch": 1.144,
+      "grad_norm": 11.875,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 1.9036,
+      "mean_token_accuracy": 0.5791543116793036,
+      "num_tokens": 13981024.0,
+      "step": 1430
+    },
+    {
+      "entropy": 1.938994688540697,
+      "epoch": 1.152,
+      "grad_norm": 11.4375,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 1.8861,
+      "mean_token_accuracy": 0.5848018407821656,
+      "num_tokens": 14086104.0,
+      "step": 1440
+    },
+    {
+      "entropy": 2.0158420264720918,
+      "epoch": 1.16,
+      "grad_norm": 12.8125,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 1.9323,
+      "mean_token_accuracy": 0.5754542848095298,
+      "num_tokens": 14179152.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.9549302212893962,
+      "epoch": 1.168,
+      "grad_norm": 11.5,
+      "learning_rate": 1.898e-05,
+      "loss": 1.9101,
+      "mean_token_accuracy": 0.582378326356411,
+      "num_tokens": 14274520.0,
+      "step": 1460
+    },
+    {
+      "entropy": 1.9402282141149043,
+      "epoch": 1.176,
+      "grad_norm": 10.875,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 1.8703,
+      "mean_token_accuracy": 0.5832702025771141,
+      "num_tokens": 14378506.0,
+      "step": 1470
+    },
+    {
+      "entropy": 2.0116791941225527,
+      "epoch": 1.184,
+      "grad_norm": 13.4375,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 1.9544,
+      "mean_token_accuracy": 0.5769317414611578,
+      "num_tokens": 14473623.0,
+      "step": 1480
+    },
+    {
+      "entropy": 1.9328050471842289,
+      "epoch": 1.192,
+      "grad_norm": 12.4375,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 1.8569,
+      "mean_token_accuracy": 0.5865075359120965,
+      "num_tokens": 14567216.0,
+      "step": 1490
+    },
+    {
+      "entropy": 1.984081008285284,
+      "epoch": 1.2,
+      "grad_norm": 12.125,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 1.9149,
+      "mean_token_accuracy": 0.578367224521935,
+      "num_tokens": 14661238.0,
+      "step": 1500
+    },
+    {
+      "epoch": 1.2,
+      "eval_coding_entropy": 1.9068009638786316,
+      "eval_coding_loss": 2.345546245574951,
+      "eval_coding_mean_token_accuracy": 0.49546057629585266,
+      "eval_coding_num_tokens": 14661238.0,
+      "eval_coding_runtime": 80.6257,
+      "eval_coding_samples_per_second": 6.201,
+      "eval_coding_steps_per_second": 3.101,
+      "step": 1500
+    },
+    {
+      "epoch": 1.2,
+      "eval_biology_entropy": 1.9247408218383788,
+      "eval_biology_loss": 2.0717103481292725,
+      "eval_biology_mean_token_accuracy": 0.5721474689245224,
+      "eval_biology_num_tokens": 14661238.0,
+      "eval_biology_runtime": 34.3306,
+      "eval_biology_samples_per_second": 14.564,
+      "eval_biology_steps_per_second": 7.282,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.739183467016335e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22b7fdeb2a653eeefefb7c83bf9eeb4fb9aa6d67df5a4bc7cdd313fb117a48a0
+size 6353