Upload folder using huggingface_hub

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-1500/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-1500/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-1500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-1500/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-1500/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-1500/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1abdd851a3cc3becbf2c609f00174172cd8e05772c35f7d5bce1587efe090d0
+size 4962221464

checkpoint-1500/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:488d91c21dc723a40fe3aa5c22f1c6bdc3c6d24e9fe15d6d0c3a821dcaed5913
+size 3670322200

checkpoint-1500/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-1500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd553b93429633bd841a6c40a256f03c644dd6c850679ca5fbb97f7b911002b8
+size 1585596266

checkpoint-1500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99e7802ecd0254b77d02dfe2d0c929e7e2dcedac1a05dc82ba2d39f9afc4821c
+size 14645

checkpoint-1500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb8afd6405fc2f72bcdc3e89147a10597d475bf4457b8f98d9c27aa5442f3ef9
+size 1465

checkpoint-1500/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-1500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-1500/trainer_state.json

@@ -0,0 +1,1864 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.7936,
+  "eval_steps": 100,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 3.238465867936611,
+      "epoch": 0.032,
+      "grad_norm": 209.0,
+      "learning_rate": 1.8e-07,
+      "loss": 7.9136,
+      "mean_token_accuracy": 0.17908816603012384,
+      "num_tokens": 99789.0,
+      "step": 10
+    },
+    {
+      "entropy": 3.293700896203518,
+      "epoch": 0.064,
+      "grad_norm": 198.0,
+      "learning_rate": 3.8e-07,
+      "loss": 7.9121,
+      "mean_token_accuracy": 0.17522176005877554,
+      "num_tokens": 190891.0,
+      "step": 20
+    },
+    {
+      "entropy": 3.2724268689751623,
+      "epoch": 0.096,
+      "grad_norm": 195.0,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 7.7884,
+      "mean_token_accuracy": 0.18085262291133403,
+      "num_tokens": 286198.0,
+      "step": 30
+    },
+    {
+      "entropy": 3.4351663440465927,
+      "epoch": 0.128,
+      "grad_norm": 172.0,
+      "learning_rate": 7.8e-07,
+      "loss": 7.5047,
+      "mean_token_accuracy": 0.18496066313236953,
+      "num_tokens": 385320.0,
+      "step": 40
+    },
+    {
+      "entropy": 3.6431967571377752,
+      "epoch": 0.16,
+      "grad_norm": 155.0,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 7.2582,
+      "mean_token_accuracy": 0.18825604091398418,
+      "num_tokens": 486218.0,
+      "step": 50
+    },
+    {
+      "entropy": 4.013543170690537,
+      "epoch": 0.192,
+      "grad_norm": 121.0,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 6.9069,
+      "mean_token_accuracy": 0.18985986979678274,
+      "num_tokens": 581717.0,
+      "step": 60
+    },
+    {
+      "entropy": 4.40448921173811,
+      "epoch": 0.224,
+      "grad_norm": 94.0,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 6.3322,
+      "mean_token_accuracy": 0.20462538227438926,
+      "num_tokens": 676500.0,
+      "step": 70
+    },
+    {
+      "entropy": 4.721661370992661,
+      "epoch": 0.256,
+      "grad_norm": 64.5,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 5.8578,
+      "mean_token_accuracy": 0.2137928050942719,
+      "num_tokens": 775542.0,
+      "step": 80
+    },
+    {
+      "entropy": 4.859676861763001,
+      "epoch": 0.288,
+      "grad_norm": 47.25,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 5.4304,
+      "mean_token_accuracy": 0.22805662611499428,
+      "num_tokens": 872932.0,
+      "step": 90
+    },
+    {
+      "entropy": 4.871997155249119,
+      "epoch": 0.32,
+      "grad_norm": 29.375,
+      "learning_rate": 1.98e-06,
+      "loss": 5.1109,
+      "mean_token_accuracy": 0.25303157502785323,
+      "num_tokens": 968340.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.32,
+      "eval_coding_entropy": 1.926878339767456,
+      "eval_coding_loss": 2.3084304332733154,
+      "eval_coding_mean_token_accuracy": 0.48411355090141295,
+      "eval_coding_num_tokens": 968340.0,
+      "eval_coding_runtime": 80.1556,
+      "eval_coding_samples_per_second": 6.238,
+      "eval_coding_steps_per_second": 3.119,
+      "step": 100
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 4.7239161949157715,
+      "eval_biology_loss": 4.968222618103027,
+      "eval_biology_mean_token_accuracy": 0.26778400188684465,
+      "eval_biology_num_tokens": 968340.0,
+      "eval_biology_runtime": 34.6244,
+      "eval_biology_samples_per_second": 14.441,
+      "eval_biology_steps_per_second": 7.22,
+      "step": 100
+    },
+    {
+      "entropy": 4.715248446166515,
+      "epoch": 0.352,
+      "grad_norm": 22.375,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 4.8208,
+      "mean_token_accuracy": 0.27833458138629796,
+      "num_tokens": 1062214.0,
+      "step": 110
+    },
+    {
+      "entropy": 4.553062991797924,
+      "epoch": 0.384,
+      "grad_norm": 24.0,
+      "learning_rate": 2.38e-06,
+      "loss": 4.6277,
+      "mean_token_accuracy": 0.29350204383954404,
+      "num_tokens": 1161800.0,
+      "step": 120
+    },
+    {
+      "entropy": 4.52610100209713,
+      "epoch": 0.416,
+      "grad_norm": 17.625,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 4.5457,
+      "mean_token_accuracy": 0.3010273904539645,
+      "num_tokens": 1259894.0,
+      "step": 130
+    },
+    {
+      "entropy": 4.3714508935809135,
+      "epoch": 0.448,
+      "grad_norm": 24.5,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 4.3595,
+      "mean_token_accuracy": 0.3121769287623465,
+      "num_tokens": 1358545.0,
+      "step": 140
+    },
+    {
+      "entropy": 4.206141373515129,
+      "epoch": 0.48,
+      "grad_norm": 20.0,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 4.2206,
+      "mean_token_accuracy": 0.32826427686959503,
+      "num_tokens": 1462493.0,
+      "step": 150
+    },
+    {
+      "entropy": 4.1862502753734585,
+      "epoch": 0.512,
+      "grad_norm": 15.6875,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 4.2016,
+      "mean_token_accuracy": 0.33233328983187677,
+      "num_tokens": 1557047.0,
+      "step": 160
+    },
+    {
+      "entropy": 4.118708249926567,
+      "epoch": 0.544,
+      "grad_norm": 16.375,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 4.0964,
+      "mean_token_accuracy": 0.3453528699465096,
+      "num_tokens": 1658711.0,
+      "step": 170
+    },
+    {
+      "entropy": 4.135542331635952,
+      "epoch": 0.576,
+      "grad_norm": 17.25,
+      "learning_rate": 3.58e-06,
+      "loss": 4.1201,
+      "mean_token_accuracy": 0.34580441238358617,
+      "num_tokens": 1756413.0,
+      "step": 180
+    },
+    {
+      "entropy": 4.003959603607655,
+      "epoch": 0.608,
+      "grad_norm": 18.375,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 4.0091,
+      "mean_token_accuracy": 0.3496086118742824,
+      "num_tokens": 1853054.0,
+      "step": 190
+    },
+    {
+      "entropy": 4.024305436015129,
+      "epoch": 0.64,
+      "grad_norm": 16.5,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 3.9643,
+      "mean_token_accuracy": 0.35361671894788743,
+      "num_tokens": 1953273.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_coding_entropy": 1.8568662638664246,
+      "eval_coding_loss": 2.1912217140197754,
+      "eval_coding_mean_token_accuracy": 0.5063816961050034,
+      "eval_coding_num_tokens": 1953273.0,
+      "eval_coding_runtime": 79.6784,
+      "eval_coding_samples_per_second": 6.275,
+      "eval_coding_steps_per_second": 3.138,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 3.9995334424972535,
+      "eval_biology_loss": 3.9905741214752197,
+      "eval_biology_mean_token_accuracy": 0.36172509467601777,
+      "eval_biology_num_tokens": 1953273.0,
+      "eval_biology_runtime": 35.4091,
+      "eval_biology_samples_per_second": 14.121,
+      "eval_biology_steps_per_second": 7.06,
+      "step": 200
+    },
+    {
+      "entropy": 3.998256941139698,
+      "epoch": 0.672,
+      "grad_norm": 22.25,
+      "learning_rate": 4.18e-06,
+      "loss": 3.9371,
+      "mean_token_accuracy": 0.3586596045643091,
+      "num_tokens": 2050255.0,
+      "step": 210
+    },
+    {
+      "entropy": 3.8787728905677796,
+      "epoch": 0.704,
+      "grad_norm": 18.5,
+      "learning_rate": 4.38e-06,
+      "loss": 3.8922,
+      "mean_token_accuracy": 0.363262693118304,
+      "num_tokens": 2150396.0,
+      "step": 220
+    },
+    {
+      "entropy": 3.881282126903534,
+      "epoch": 0.736,
+      "grad_norm": 15.375,
+      "learning_rate": 4.58e-06,
+      "loss": 3.8711,
+      "mean_token_accuracy": 0.36838123146444557,
+      "num_tokens": 2250726.0,
+      "step": 230
+    },
+    {
+      "entropy": 3.8907541409134865,
+      "epoch": 0.768,
+      "grad_norm": 19.0,
+      "learning_rate": 4.78e-06,
+      "loss": 3.8523,
+      "mean_token_accuracy": 0.36881062425673006,
+      "num_tokens": 2346168.0,
+      "step": 240
+    },
+    {
+      "entropy": 3.8764460012316704,
+      "epoch": 0.8,
+      "grad_norm": 16.75,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 3.833,
+      "mean_token_accuracy": 0.37063299771398306,
+      "num_tokens": 2446303.0,
+      "step": 250
+    },
+    {
+      "entropy": 3.8517253383994103,
+      "epoch": 0.832,
+      "grad_norm": 16.5,
+      "learning_rate": 5.18e-06,
+      "loss": 3.8114,
+      "mean_token_accuracy": 0.3745407974347472,
+      "num_tokens": 2540899.0,
+      "step": 260
+    },
+    {
+      "entropy": 3.778804798424244,
+      "epoch": 0.864,
+      "grad_norm": 15.9375,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 3.73,
+      "mean_token_accuracy": 0.37728464547544716,
+      "num_tokens": 2638613.0,
+      "step": 270
+    },
+    {
+      "entropy": 3.7176755860447885,
+      "epoch": 0.896,
+      "grad_norm": 19.875,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 3.6871,
+      "mean_token_accuracy": 0.3835112711414695,
+      "num_tokens": 2743826.0,
+      "step": 280
+    },
+    {
+      "entropy": 3.7258670911192895,
+      "epoch": 0.928,
+      "grad_norm": 21.75,
+      "learning_rate": 5.78e-06,
+      "loss": 3.6856,
+      "mean_token_accuracy": 0.38429632876068354,
+      "num_tokens": 2847721.0,
+      "step": 290
+    },
+    {
+      "entropy": 3.712230232357979,
+      "epoch": 0.96,
+      "grad_norm": 15.6875,
+      "learning_rate": 5.98e-06,
+      "loss": 3.6164,
+      "mean_token_accuracy": 0.3888826366513968,
+      "num_tokens": 2948735.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "eval_coding_entropy": 1.794304684638977,
+      "eval_coding_loss": 2.1738245487213135,
+      "eval_coding_mean_token_accuracy": 0.5117835735082626,
+      "eval_coding_num_tokens": 2948735.0,
+      "eval_coding_runtime": 78.9169,
+      "eval_coding_samples_per_second": 6.336,
+      "eval_coding_steps_per_second": 3.168,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "eval_biology_entropy": 3.593994511604309,
+      "eval_biology_loss": 3.689100503921509,
+      "eval_biology_mean_token_accuracy": 0.39202056431770327,
+      "eval_biology_num_tokens": 2948735.0,
+      "eval_biology_runtime": 34.5492,
+      "eval_biology_samples_per_second": 14.472,
+      "eval_biology_steps_per_second": 7.236,
+      "step": 300
+    },
+    {
+      "entropy": 3.6798228099942207,
+      "epoch": 0.992,
+      "grad_norm": 18.375,
+      "learning_rate": 6.18e-06,
+      "loss": 3.6273,
+      "mean_token_accuracy": 0.3867473939433694,
+      "num_tokens": 3049436.0,
+      "step": 310
+    },
+    {
+      "entropy": 3.6835175109537026,
+      "epoch": 1.0224,
+      "grad_norm": 15.875,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 3.626,
+      "mean_token_accuracy": 0.3879838499583696,
+      "num_tokens": 3140209.0,
+      "step": 320
+    },
+    {
+      "entropy": 3.61849829852581,
+      "epoch": 1.0544,
+      "grad_norm": 15.1875,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 3.5842,
+      "mean_token_accuracy": 0.3950715743005276,
+      "num_tokens": 3237743.0,
+      "step": 330
+    },
+    {
+      "entropy": 3.6082335516810415,
+      "epoch": 1.0864,
+      "grad_norm": 15.125,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 3.556,
+      "mean_token_accuracy": 0.3969528621062636,
+      "num_tokens": 3336343.0,
+      "step": 340
+    },
+    {
+      "entropy": 3.549923361837864,
+      "epoch": 1.1184,
+      "grad_norm": 18.75,
+      "learning_rate": 6.98e-06,
+      "loss": 3.4731,
+      "mean_token_accuracy": 0.4048408783972263,
+      "num_tokens": 3432312.0,
+      "step": 350
+    },
+    {
+      "entropy": 3.5383049845695496,
+      "epoch": 1.1504,
+      "grad_norm": 17.125,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 3.485,
+      "mean_token_accuracy": 0.4019928781315684,
+      "num_tokens": 3529743.0,
+      "step": 360
+    },
+    {
+      "entropy": 3.550269864499569,
+      "epoch": 1.1824,
+      "grad_norm": 18.625,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 3.4992,
+      "mean_token_accuracy": 0.40656839609146117,
+      "num_tokens": 3628421.0,
+      "step": 370
+    },
+    {
+      "entropy": 3.549488727748394,
+      "epoch": 1.2144,
+      "grad_norm": 15.6875,
+      "learning_rate": 7.58e-06,
+      "loss": 3.5189,
+      "mean_token_accuracy": 0.40171824395656586,
+      "num_tokens": 3731754.0,
+      "step": 380
+    },
+    {
+      "entropy": 3.5428646177053453,
+      "epoch": 1.2464,
+      "grad_norm": 16.875,
+      "learning_rate": 7.78e-06,
+      "loss": 3.4652,
+      "mean_token_accuracy": 0.40349574480205774,
+      "num_tokens": 3833669.0,
+      "step": 390
+    },
+    {
+      "entropy": 3.412770676612854,
+      "epoch": 1.2784,
+      "grad_norm": 19.375,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 3.3885,
+      "mean_token_accuracy": 0.4114306215196848,
+      "num_tokens": 3933489.0,
+      "step": 400
+    },
+    {
+      "epoch": 1.2784,
+      "eval_coding_entropy": 1.864494191646576,
+      "eval_coding_loss": 2.1695075035095215,
+      "eval_coding_mean_token_accuracy": 0.5126164103746415,
+      "eval_coding_num_tokens": 3933489.0,
+      "eval_coding_runtime": 79.6045,
+      "eval_coding_samples_per_second": 6.281,
+      "eval_coding_steps_per_second": 3.141,
+      "step": 400
+    },
+    {
+      "epoch": 1.2784,
+      "eval_biology_entropy": 3.4962758731842043,
+      "eval_biology_loss": 3.484936475753784,
+      "eval_biology_mean_token_accuracy": 0.4131394236087799,
+      "eval_biology_num_tokens": 3933489.0,
+      "eval_biology_runtime": 34.5802,
+      "eval_biology_samples_per_second": 14.459,
+      "eval_biology_steps_per_second": 7.23,
+      "step": 400
+    },
+    {
+      "entropy": 3.447491915524006,
+      "epoch": 1.3104,
+      "grad_norm": 17.375,
+      "learning_rate": 8.18e-06,
+      "loss": 3.397,
+      "mean_token_accuracy": 0.4184817912057042,
+      "num_tokens": 4028501.0,
+      "step": 410
+    },
+    {
+      "entropy": 3.491288626194,
+      "epoch": 1.3424,
+      "grad_norm": 22.0,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 3.4228,
+      "mean_token_accuracy": 0.4082869263365865,
+      "num_tokens": 4128412.0,
+      "step": 420
+    },
+    {
+      "entropy": 3.4260040298104286,
+      "epoch": 1.3744,
+      "grad_norm": 18.25,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 3.3652,
+      "mean_token_accuracy": 0.417889959923923,
+      "num_tokens": 4228703.0,
+      "step": 430
+    },
+    {
+      "entropy": 3.4077636525034904,
+      "epoch": 1.4064,
+      "grad_norm": 20.375,
+      "learning_rate": 8.78e-06,
+      "loss": 3.3644,
+      "mean_token_accuracy": 0.41263102628290654,
+      "num_tokens": 4329623.0,
+      "step": 440
+    },
+    {
+      "entropy": 3.4000454649329184,
+      "epoch": 1.4384000000000001,
+      "grad_norm": 17.25,
+      "learning_rate": 8.98e-06,
+      "loss": 3.3629,
+      "mean_token_accuracy": 0.41560379061847924,
+      "num_tokens": 4424877.0,
+      "step": 450
+    },
+    {
+      "entropy": 3.473749025166035,
+      "epoch": 1.4704,
+      "grad_norm": 18.0,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 3.3787,
+      "mean_token_accuracy": 0.4140065621584654,
+      "num_tokens": 4518810.0,
+      "step": 460
+    },
+    {
+      "entropy": 3.418153963983059,
+      "epoch": 1.5024,
+      "grad_norm": 21.875,
+      "learning_rate": 9.38e-06,
+      "loss": 3.3539,
+      "mean_token_accuracy": 0.41980997528880837,
+      "num_tokens": 4617323.0,
+      "step": 470
+    },
+    {
+      "entropy": 3.410584717988968,
+      "epoch": 1.5344,
+      "grad_norm": 20.875,
+      "learning_rate": 9.58e-06,
+      "loss": 3.3493,
+      "mean_token_accuracy": 0.4202044107019901,
+      "num_tokens": 4719316.0,
+      "step": 480
+    },
+    {
+      "entropy": 3.3172555193305016,
+      "epoch": 1.5664,
+      "grad_norm": 23.75,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 3.2781,
+      "mean_token_accuracy": 0.42654727064073084,
+      "num_tokens": 4818127.0,
+      "step": 490
+    },
+    {
+      "entropy": 3.3434231370687484,
+      "epoch": 1.5984,
+      "grad_norm": 17.75,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 3.2655,
+      "mean_token_accuracy": 0.42654075026512145,
+      "num_tokens": 4915672.0,
+      "step": 500
+    },
+    {
+      "epoch": 1.5984,
+      "eval_coding_entropy": 1.9044507808685303,
+      "eval_coding_loss": 2.163560152053833,
+      "eval_coding_mean_token_accuracy": 0.5137092983722686,
+      "eval_coding_num_tokens": 4915672.0,
+      "eval_coding_runtime": 78.8714,
+      "eval_coding_samples_per_second": 6.339,
+      "eval_coding_steps_per_second": 3.17,
+      "step": 500
+    },
+    {
+      "epoch": 1.5984,
+      "eval_biology_entropy": 3.2849253273010253,
+      "eval_biology_loss": 3.3220765590667725,
+      "eval_biology_mean_token_accuracy": 0.4304432601928711,
+      "eval_biology_num_tokens": 4915672.0,
+      "eval_biology_runtime": 34.5563,
+      "eval_biology_samples_per_second": 14.469,
+      "eval_biology_steps_per_second": 7.235,
+      "step": 500
+    },
+    {
+      "entropy": 3.276486141234636,
+      "epoch": 1.6303999999999998,
+      "grad_norm": 19.5,
+      "learning_rate": 1.018e-05,
+      "loss": 3.2382,
+      "mean_token_accuracy": 0.42907340787351134,
+      "num_tokens": 5015381.0,
+      "step": 510
+    },
+    {
+      "entropy": 3.3447541266679766,
+      "epoch": 1.6623999999999999,
+      "grad_norm": 23.25,
+      "learning_rate": 1.038e-05,
+      "loss": 3.2677,
+      "mean_token_accuracy": 0.42632252164185047,
+      "num_tokens": 5117126.0,
+      "step": 520
+    },
+    {
+      "entropy": 3.3336540818214417,
+      "epoch": 1.6944,
+      "grad_norm": 20.75,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 3.299,
+      "mean_token_accuracy": 0.4259482711553574,
+      "num_tokens": 5213073.0,
+      "step": 530
+    },
+    {
+      "entropy": 3.319507985562086,
+      "epoch": 1.7264,
+      "grad_norm": 20.0,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 3.2425,
+      "mean_token_accuracy": 0.4222921848297119,
+      "num_tokens": 5308786.0,
+      "step": 540
+    },
+    {
+      "entropy": 3.249939414858818,
+      "epoch": 1.7584,
+      "grad_norm": 23.625,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 3.2109,
+      "mean_token_accuracy": 0.4321354482322931,
+      "num_tokens": 5405141.0,
+      "step": 550
+    },
+    {
+      "entropy": 3.2699292853474615,
+      "epoch": 1.7904,
+      "grad_norm": 24.25,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 3.2126,
+      "mean_token_accuracy": 0.4329648604616523,
+      "num_tokens": 5505162.0,
+      "step": 560
+    },
+    {
+      "entropy": 3.2213672384619714,
+      "epoch": 1.8224,
+      "grad_norm": 19.375,
+      "learning_rate": 1.138e-05,
+      "loss": 3.1498,
+      "mean_token_accuracy": 0.4372351700440049,
+      "num_tokens": 5604409.0,
+      "step": 570
+    },
+    {
+      "entropy": 3.158064991235733,
+      "epoch": 1.8544,
+      "grad_norm": 25.625,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 3.0863,
+      "mean_token_accuracy": 0.44516869317740204,
+      "num_tokens": 5705344.0,
+      "step": 580
+    },
+    {
+      "entropy": 3.2133444771170616,
+      "epoch": 1.8864,
+      "grad_norm": 21.125,
+      "learning_rate": 1.178e-05,
+      "loss": 3.1518,
+      "mean_token_accuracy": 0.4377427864819765,
+      "num_tokens": 5794262.0,
+      "step": 590
+    },
+    {
+      "entropy": 3.2123986557126045,
+      "epoch": 1.9184,
+      "grad_norm": 21.0,
+      "learning_rate": 1.198e-05,
+      "loss": 3.1537,
+      "mean_token_accuracy": 0.439489153586328,
+      "num_tokens": 5895496.0,
+      "step": 600
+    },
+    {
+      "epoch": 1.9184,
+      "eval_coding_entropy": 1.892683578968048,
+      "eval_coding_loss": 2.1593310832977295,
+      "eval_coding_mean_token_accuracy": 0.514176591873169,
+      "eval_coding_num_tokens": 5895496.0,
+      "eval_coding_runtime": 78.9615,
+      "eval_coding_samples_per_second": 6.332,
+      "eval_coding_steps_per_second": 3.166,
+      "step": 600
+    },
+    {
+      "epoch": 1.9184,
+      "eval_biology_entropy": 3.14973046875,
+      "eval_biology_loss": 3.1956348419189453,
+      "eval_biology_mean_token_accuracy": 0.4422305516004562,
+      "eval_biology_num_tokens": 5895496.0,
+      "eval_biology_runtime": 34.5624,
+      "eval_biology_samples_per_second": 14.467,
+      "eval_biology_steps_per_second": 7.233,
+      "step": 600
+    },
+    {
+      "entropy": 3.2023044779896734,
+      "epoch": 1.9504000000000001,
+      "grad_norm": 20.375,
+      "learning_rate": 1.218e-05,
+      "loss": 3.145,
+      "mean_token_accuracy": 0.4402220517396927,
+      "num_tokens": 5992443.0,
+      "step": 610
+    },
+    {
+      "entropy": 3.173665773868561,
+      "epoch": 1.9824000000000002,
+      "grad_norm": 19.375,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 3.1235,
+      "mean_token_accuracy": 0.43916483521461486,
+      "num_tokens": 6092718.0,
+      "step": 620
+    },
+    {
+      "entropy": 3.130789705012974,
+      "epoch": 2.0128,
+      "grad_norm": 21.5,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 3.0485,
+      "mean_token_accuracy": 0.4457206469225256,
+      "num_tokens": 6184982.0,
+      "step": 630
+    },
+    {
+      "entropy": 3.024783915281296,
+      "epoch": 2.0448,
+      "grad_norm": 19.875,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 2.992,
+      "mean_token_accuracy": 0.45380019545555117,
+      "num_tokens": 6286901.0,
+      "step": 640
+    },
+    {
+      "entropy": 3.1106227308511736,
+      "epoch": 2.0768,
+      "grad_norm": 20.25,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 3.0474,
+      "mean_token_accuracy": 0.4507150810211897,
+      "num_tokens": 6381706.0,
+      "step": 650
+    },
+    {
+      "entropy": 3.070780225098133,
+      "epoch": 2.1088,
+      "grad_norm": 22.125,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 3.014,
+      "mean_token_accuracy": 0.45487339831888673,
+      "num_tokens": 6478891.0,
+      "step": 660
+    },
+    {
+      "entropy": 3.1098087787628175,
+      "epoch": 2.1408,
+      "grad_norm": 25.75,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 3.0755,
+      "mean_token_accuracy": 0.44592468421906234,
+      "num_tokens": 6576566.0,
+      "step": 670
+    },
+    {
+      "entropy": 3.086433382332325,
+      "epoch": 2.1728,
+      "grad_norm": 18.5,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 3.0116,
+      "mean_token_accuracy": 0.4536932479590178,
+      "num_tokens": 6678216.0,
+      "step": 680
+    },
+    {
+      "entropy": 3.0678569465875625,
+      "epoch": 2.2048,
+      "grad_norm": 19.5,
+      "learning_rate": 1.378e-05,
+      "loss": 2.9845,
+      "mean_token_accuracy": 0.4514392616227269,
+      "num_tokens": 6771313.0,
+      "step": 690
+    },
+    {
+      "entropy": 3.0470827482640743,
+      "epoch": 2.2368,
+      "grad_norm": 23.0,
+      "learning_rate": 1.398e-05,
+      "loss": 2.9636,
+      "mean_token_accuracy": 0.4556872248649597,
+      "num_tokens": 6872028.0,
+      "step": 700
+    },
+    {
+      "epoch": 2.2368,
+      "eval_coding_entropy": 1.8572090373039245,
+      "eval_coding_loss": 2.177124500274658,
+      "eval_coding_mean_token_accuracy": 0.513177493929863,
+      "eval_coding_num_tokens": 6872028.0,
+      "eval_coding_runtime": 78.9661,
+      "eval_coding_samples_per_second": 6.332,
+      "eval_coding_steps_per_second": 3.166,
+      "step": 700
+    },
+    {
+      "epoch": 2.2368,
+      "eval_biology_entropy": 2.983607707500458,
+      "eval_biology_loss": 3.0935444831848145,
+      "eval_biology_mean_token_accuracy": 0.4546413152217865,
+      "eval_biology_num_tokens": 6872028.0,
+      "eval_biology_runtime": 34.9245,
+      "eval_biology_samples_per_second": 14.317,
+      "eval_biology_steps_per_second": 7.158,
+      "step": 700
+    },
+    {
+      "entropy": 3.0512571170926095,
+      "epoch": 2.2688,
+      "grad_norm": 18.75,
+      "learning_rate": 1.418e-05,
+      "loss": 2.9764,
+      "mean_token_accuracy": 0.45544591508805754,
+      "num_tokens": 6971802.0,
+      "step": 710
+    },
+    {
+      "entropy": 3.079524612426758,
+      "epoch": 2.3008,
+      "grad_norm": 22.0,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 3.0142,
+      "mean_token_accuracy": 0.45264575611799956,
+      "num_tokens": 7067482.0,
+      "step": 720
+    },
+    {
+      "entropy": 3.0822106420993807,
+      "epoch": 2.3327999999999998,
+      "grad_norm": 25.75,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 3.0382,
+      "mean_token_accuracy": 0.45174078848212956,
+      "num_tokens": 7164629.0,
+      "step": 730
+    },
+    {
+      "entropy": 3.01283405572176,
+      "epoch": 2.3648,
+      "grad_norm": 20.25,
+      "learning_rate": 1.478e-05,
+      "loss": 2.9356,
+      "mean_token_accuracy": 0.46082064919173715,
+      "num_tokens": 7260811.0,
+      "step": 740
+    },
+    {
+      "entropy": 3.0809263423085214,
+      "epoch": 2.3968,
+      "grad_norm": 18.5,
+      "learning_rate": 1.498e-05,
+      "loss": 2.9877,
+      "mean_token_accuracy": 0.45245055090636016,
+      "num_tokens": 7353504.0,
+      "step": 750
+    },
+    {
+      "entropy": 3.0187013909220695,
+      "epoch": 2.4288,
+      "grad_norm": 22.75,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 2.9599,
+      "mean_token_accuracy": 0.4591849962249398,
+      "num_tokens": 7450239.0,
+      "step": 760
+    },
+    {
+      "entropy": 2.969215099513531,
+      "epoch": 2.4608,
+      "grad_norm": 22.875,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 2.8833,
+      "mean_token_accuracy": 0.4634178843349218,
+      "num_tokens": 7549458.0,
+      "step": 770
+    },
+    {
+      "entropy": 2.9438143245875836,
+      "epoch": 2.4928,
+      "grad_norm": 18.5,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 2.9044,
+      "mean_token_accuracy": 0.4644233400002122,
+      "num_tokens": 7646646.0,
+      "step": 780
+    },
+    {
+      "entropy": 2.9995977893471717,
+      "epoch": 2.5248,
+      "grad_norm": 19.875,
+      "learning_rate": 1.578e-05,
+      "loss": 2.9524,
+      "mean_token_accuracy": 0.46077679973095653,
+      "num_tokens": 7743967.0,
+      "step": 790
+    },
+    {
+      "entropy": 2.9542898386716843,
+      "epoch": 2.5568,
+      "grad_norm": 21.75,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 2.8718,
+      "mean_token_accuracy": 0.4678089875727892,
+      "num_tokens": 7844679.0,
+      "step": 800
+    },
+    {
+      "epoch": 2.5568,
+      "eval_coding_entropy": 1.8718931813240052,
+      "eval_coding_loss": 2.1724154949188232,
+      "eval_coding_mean_token_accuracy": 0.5149018170833588,
+      "eval_coding_num_tokens": 7844679.0,
+      "eval_coding_runtime": 78.9496,
+      "eval_coding_samples_per_second": 6.333,
+      "eval_coding_steps_per_second": 3.167,
+      "step": 800
+    },
+    {
+      "epoch": 2.5568,
+      "eval_biology_entropy": 2.85398051404953,
+      "eval_biology_loss": 2.9827656745910645,
+      "eval_biology_mean_token_accuracy": 0.465399906039238,
+      "eval_biology_num_tokens": 7844679.0,
+      "eval_biology_runtime": 34.5946,
+      "eval_biology_samples_per_second": 14.453,
+      "eval_biology_steps_per_second": 7.227,
+      "step": 800
+    },
+    {
+      "entropy": 2.905091355741024,
+      "epoch": 2.5888,
+      "grad_norm": 20.0,
+      "learning_rate": 1.618e-05,
+      "loss": 2.8797,
+      "mean_token_accuracy": 0.47355432007461784,
+      "num_tokens": 7944653.0,
+      "step": 810
+    },
+    {
+      "entropy": 2.941771225631237,
+      "epoch": 2.6208,
+      "grad_norm": 20.625,
+      "learning_rate": 1.638e-05,
+      "loss": 2.857,
+      "mean_token_accuracy": 0.4673197165131569,
+      "num_tokens": 8043783.0,
+      "step": 820
+    },
+    {
+      "entropy": 2.992217946052551,
+      "epoch": 2.6528,
+      "grad_norm": 22.25,
+      "learning_rate": 1.658e-05,
+      "loss": 2.9048,
+      "mean_token_accuracy": 0.46489576790481807,
+      "num_tokens": 8142126.0,
+      "step": 830
+    },
+    {
+      "entropy": 2.9804361656308176,
+      "epoch": 2.6848,
+      "grad_norm": 18.75,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 2.9216,
+      "mean_token_accuracy": 0.462474500015378,
+      "num_tokens": 8235244.0,
+      "step": 840
+    },
+    {
+      "entropy": 2.933942112326622,
+      "epoch": 2.7168,
+      "grad_norm": 18.125,
+      "learning_rate": 1.698e-05,
+      "loss": 2.8811,
+      "mean_token_accuracy": 0.4669422248378396,
+      "num_tokens": 8336300.0,
+      "step": 850
+    },
+    {
+      "entropy": 2.826403010636568,
+      "epoch": 2.7488,
+      "grad_norm": 20.375,
+      "learning_rate": 1.718e-05,
+      "loss": 2.7594,
+      "mean_token_accuracy": 0.47525308579206466,
+      "num_tokens": 8440110.0,
+      "step": 860
+    },
+    {
+      "entropy": 2.8751538641750813,
+      "epoch": 2.7808,
+      "grad_norm": 22.125,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 2.8221,
+      "mean_token_accuracy": 0.4738822041079402,
+      "num_tokens": 8533528.0,
+      "step": 870
+    },
+    {
+      "entropy": 2.885574182868004,
+      "epoch": 2.8128,
+      "grad_norm": 22.375,
+      "learning_rate": 1.758e-05,
+      "loss": 2.8067,
+      "mean_token_accuracy": 0.47314695566892623,
+      "num_tokens": 8632702.0,
+      "step": 880
+    },
+    {
+      "entropy": 2.9287849619984625,
+      "epoch": 2.8448,
+      "grad_norm": 20.25,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 2.8323,
+      "mean_token_accuracy": 0.4664028905332088,
+      "num_tokens": 8729727.0,
+      "step": 890
+    },
+    {
+      "entropy": 2.885370475798845,
+      "epoch": 2.8768000000000002,
+      "grad_norm": 20.75,
+      "learning_rate": 1.798e-05,
+      "loss": 2.8442,
+      "mean_token_accuracy": 0.46528117191046475,
+      "num_tokens": 8831917.0,
+      "step": 900
+    },
+    {
+      "epoch": 2.8768000000000002,
+      "eval_coding_entropy": 1.8902995553016662,
+      "eval_coding_loss": 2.1872222423553467,
+      "eval_coding_mean_token_accuracy": 0.5130699964761734,
+      "eval_coding_num_tokens": 8831917.0,
+      "eval_coding_runtime": 78.9145,
+      "eval_coding_samples_per_second": 6.336,
+      "eval_coding_steps_per_second": 3.168,
+      "step": 900
+    },
+    {
+      "epoch": 2.8768000000000002,
+      "eval_biology_entropy": 2.868830183506012,
+      "eval_biology_loss": 2.891908884048462,
+      "eval_biology_mean_token_accuracy": 0.4745277794599533,
+      "eval_biology_num_tokens": 8831917.0,
+      "eval_biology_runtime": 34.562,
+      "eval_biology_samples_per_second": 14.467,
+      "eval_biology_steps_per_second": 7.233,
+      "step": 900
+    },
+    {
+      "entropy": 2.884679964929819,
+      "epoch": 2.9088000000000003,
+      "grad_norm": 22.0,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 2.7851,
+      "mean_token_accuracy": 0.4735592044889927,
+      "num_tokens": 8931811.0,
+      "step": 910
+    },
+    {
+      "entropy": 2.8508158676326274,
+      "epoch": 2.9408,
+      "grad_norm": 23.25,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 2.7834,
+      "mean_token_accuracy": 0.47639975920319555,
+      "num_tokens": 9033255.0,
+      "step": 920
+    },
+    {
+      "entropy": 2.8332293681800365,
+      "epoch": 2.9728,
+      "grad_norm": 19.875,
+      "learning_rate": 1.858e-05,
+      "loss": 2.7458,
+      "mean_token_accuracy": 0.47748742904514074,
+      "num_tokens": 9129675.0,
+      "step": 930
+    },
+    {
+      "entropy": 2.8580648820651207,
+      "epoch": 3.0032,
+      "grad_norm": 20.375,
+      "learning_rate": 1.878e-05,
+      "loss": 2.7791,
+      "mean_token_accuracy": 0.4723091986226408,
+      "num_tokens": 9224886.0,
+      "step": 940
+    },
+    {
+      "entropy": 2.7373731434345245,
+      "epoch": 3.0352,
+      "grad_norm": 18.625,
+      "learning_rate": 1.898e-05,
+      "loss": 2.6356,
+      "mean_token_accuracy": 0.4919461887329817,
+      "num_tokens": 9325062.0,
+      "step": 950
+    },
+    {
+      "entropy": 2.727712345123291,
+      "epoch": 3.0672,
+      "grad_norm": 20.75,
+      "learning_rate": 1.918e-05,
+      "loss": 2.6412,
+      "mean_token_accuracy": 0.4901068951934576,
+      "num_tokens": 9424019.0,
+      "step": 960
+    },
+    {
+      "entropy": 2.707367330044508,
+      "epoch": 3.0992,
+      "grad_norm": 20.625,
+      "learning_rate": 1.938e-05,
+      "loss": 2.6608,
+      "mean_token_accuracy": 0.4915515352040529,
+      "num_tokens": 9526246.0,
+      "step": 970
+    },
+    {
+      "entropy": 2.718513052165508,
+      "epoch": 3.1312,
+      "grad_norm": 18.25,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 2.6232,
+      "mean_token_accuracy": 0.48776335902512075,
+      "num_tokens": 9624382.0,
+      "step": 980
+    },
+    {
+      "entropy": 2.7374294199049474,
+      "epoch": 3.1632,
+      "grad_norm": 21.375,
+      "learning_rate": 1.978e-05,
+      "loss": 2.6429,
+      "mean_token_accuracy": 0.48988417275249957,
+      "num_tokens": 9719411.0,
+      "step": 990
+    },
+    {
+      "entropy": 2.7222474955022333,
+      "epoch": 3.1952,
+      "grad_norm": 20.375,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 2.6263,
+      "mean_token_accuracy": 0.4935696581378579,
+      "num_tokens": 9819220.0,
+      "step": 1000
+    },
+    {
+      "epoch": 3.1952,
+      "eval_coding_entropy": 1.7593334603309632,
+      "eval_coding_loss": 2.205096960067749,
+      "eval_coding_mean_token_accuracy": 0.5136193619966507,
+      "eval_coding_num_tokens": 9819220.0,
+      "eval_coding_runtime": 78.8943,
+      "eval_coding_samples_per_second": 6.338,
+      "eval_coding_steps_per_second": 3.169,
+      "step": 1000
+    },
+    {
+      "epoch": 3.1952,
+      "eval_biology_entropy": 2.6804689655303955,
+      "eval_biology_loss": 2.818235397338867,
+      "eval_biology_mean_token_accuracy": 0.48389568710327147,
+      "eval_biology_num_tokens": 9819220.0,
+      "eval_biology_runtime": 34.6726,
+      "eval_biology_samples_per_second": 14.421,
+      "eval_biology_steps_per_second": 7.21,
+      "step": 1000
+    },
+    {
+      "entropy": 2.6821524411439897,
+      "epoch": 3.2272,
+      "grad_norm": 25.0,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 2.5881,
+      "mean_token_accuracy": 0.4957658048719168,
+      "num_tokens": 9917642.0,
+      "step": 1010
+    },
+    {
+      "entropy": 2.643439619243145,
+      "epoch": 3.2592,
+      "grad_norm": 17.25,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 2.5861,
+      "mean_token_accuracy": 0.4964464308694005,
+      "num_tokens": 10016171.0,
+      "step": 1020
+    },
+    {
+      "entropy": 2.69493540674448,
+      "epoch": 3.2912,
+      "grad_norm": 21.25,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 2.6244,
+      "mean_token_accuracy": 0.49275039993226527,
+      "num_tokens": 10113317.0,
+      "step": 1030
+    },
+    {
+      "entropy": 2.6980527356266975,
+      "epoch": 3.3232,
+      "grad_norm": 21.0,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 2.6358,
+      "mean_token_accuracy": 0.4931097852066159,
+      "num_tokens": 10209948.0,
+      "step": 1040
+    },
+    {
+      "entropy": 2.704365950077772,
+      "epoch": 3.3552,
+      "grad_norm": 17.375,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 2.6278,
+      "mean_token_accuracy": 0.4901678072288632,
+      "num_tokens": 10304993.0,
+      "step": 1050
+    },
+    {
+      "entropy": 2.6926429100334643,
+      "epoch": 3.3872,
+      "grad_norm": 23.375,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 2.6281,
+      "mean_token_accuracy": 0.49370026849210263,
+      "num_tokens": 10403340.0,
+      "step": 1060
+    },
+    {
+      "entropy": 2.7046832494437694,
+      "epoch": 3.4192,
+      "grad_norm": 23.625,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 2.6136,
+      "mean_token_accuracy": 0.49537604544311764,
+      "num_tokens": 10499289.0,
+      "step": 1070
+    },
+    {
+      "entropy": 2.6765974409878255,
+      "epoch": 3.4512,
+      "grad_norm": 22.75,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 2.6201,
+      "mean_token_accuracy": 0.49774726927280427,
+      "num_tokens": 10595192.0,
+      "step": 1080
+    },
+    {
+      "entropy": 2.661435030400753,
+      "epoch": 3.4832,
+      "grad_norm": 20.375,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 2.6017,
+      "mean_token_accuracy": 0.49518649335950615,
+      "num_tokens": 10692762.0,
+      "step": 1090
+    },
+    {
+      "entropy": 2.6369900465011598,
+      "epoch": 3.5152,
+      "grad_norm": 20.25,
+      "learning_rate": 1.978e-05,
+      "loss": 2.5617,
+      "mean_token_accuracy": 0.4994138855487108,
+      "num_tokens": 10795865.0,
+      "step": 1100
+    },
+    {
+      "epoch": 3.5152,
+      "eval_coding_entropy": 1.8151787004470825,
+      "eval_coding_loss": 2.212179183959961,
+      "eval_coding_mean_token_accuracy": 0.5126535563468934,
+      "eval_coding_num_tokens": 10795865.0,
+      "eval_coding_runtime": 79.0002,
+      "eval_coding_samples_per_second": 6.329,
+      "eval_coding_steps_per_second": 3.165,
+      "step": 1100
+    },
+    {
+      "epoch": 3.5152,
+      "eval_biology_entropy": 2.6696902551651003,
+      "eval_biology_loss": 2.7315852642059326,
+      "eval_biology_mean_token_accuracy": 0.4940973354578018,
+      "eval_biology_num_tokens": 10795865.0,
+      "eval_biology_runtime": 34.6872,
+      "eval_biology_samples_per_second": 14.415,
+      "eval_biology_steps_per_second": 7.207,
+      "step": 1100
+    },
+    {
+      "entropy": 2.64646837413311,
+      "epoch": 3.5472,
+      "grad_norm": 19.75,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 2.5694,
+      "mean_token_accuracy": 0.5007585499435663,
+      "num_tokens": 10896150.0,
+      "step": 1110
+    },
+    {
+      "entropy": 2.5886894077062608,
+      "epoch": 3.5792,
+      "grad_norm": 20.125,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 2.5421,
+      "mean_token_accuracy": 0.503319687396288,
+      "num_tokens": 10994009.0,
+      "step": 1120
+    },
+    {
+      "entropy": 2.664226660132408,
+      "epoch": 3.6112,
+      "grad_norm": 21.0,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 2.5748,
+      "mean_token_accuracy": 0.4976232260465622,
+      "num_tokens": 11088408.0,
+      "step": 1130
+    },
+    {
+      "entropy": 2.591143436729908,
+      "epoch": 3.6432,
+      "grad_norm": 19.0,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 2.5241,
+      "mean_token_accuracy": 0.5072892276570201,
+      "num_tokens": 11189792.0,
+      "step": 1140
+    },
+    {
+      "entropy": 2.607444966584444,
+      "epoch": 3.6752000000000002,
+      "grad_norm": 18.375,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 2.5651,
+      "mean_token_accuracy": 0.5035420592874289,
+      "num_tokens": 11287885.0,
+      "step": 1150
+    },
+    {
+      "entropy": 2.5825724259018896,
+      "epoch": 3.7072000000000003,
+      "grad_norm": 18.875,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 2.519,
+      "mean_token_accuracy": 0.5080195646733046,
+      "num_tokens": 11386948.0,
+      "step": 1160
+    },
+    {
+      "entropy": 2.638581147044897,
+      "epoch": 3.7392,
+      "grad_norm": 15.8125,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 2.5488,
+      "mean_token_accuracy": 0.4984665771946311,
+      "num_tokens": 11486738.0,
+      "step": 1170
+    },
+    {
+      "entropy": 2.623228520154953,
+      "epoch": 3.7712,
+      "grad_norm": 18.5,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 2.5644,
+      "mean_token_accuracy": 0.4989262154325843,
+      "num_tokens": 11587111.0,
+      "step": 1180
+    },
+    {
+      "entropy": 2.595949812233448,
+      "epoch": 3.8032,
+      "grad_norm": 20.25,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 2.5348,
+      "mean_token_accuracy": 0.5062878143042326,
+      "num_tokens": 11683518.0,
+      "step": 1190
+    },
+    {
+      "entropy": 2.552212902158499,
+      "epoch": 3.8352,
+      "grad_norm": 18.5,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 2.4693,
+      "mean_token_accuracy": 0.5114001492038369,
+      "num_tokens": 11787112.0,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8352,
+      "eval_coding_entropy": 1.821248462677002,
+      "eval_coding_loss": 2.2172763347625732,
+      "eval_coding_mean_token_accuracy": 0.5105675404071808,
+      "eval_coding_num_tokens": 11787112.0,
+      "eval_coding_runtime": 79.0098,
+      "eval_coding_samples_per_second": 6.328,
+      "eval_coding_steps_per_second": 3.164,
+      "step": 1200
+    },
+    {
+      "epoch": 3.8352,
+      "eval_biology_entropy": 2.618246515274048,
+      "eval_biology_loss": 2.6723146438598633,
+      "eval_biology_mean_token_accuracy": 0.499475853562355,
+      "eval_biology_num_tokens": 11787112.0,
+      "eval_biology_runtime": 34.6592,
+      "eval_biology_samples_per_second": 14.426,
+      "eval_biology_steps_per_second": 7.213,
+      "step": 1200
+    },
+    {
+      "entropy": 2.606553990393877,
+      "epoch": 3.8672,
+      "grad_norm": 19.5,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 2.5365,
+      "mean_token_accuracy": 0.5034115230664611,
+      "num_tokens": 11882695.0,
+      "step": 1210
+    },
+    {
+      "entropy": 2.623321808129549,
+      "epoch": 3.8992,
+      "grad_norm": 17.5,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 2.5545,
+      "mean_token_accuracy": 0.5018050175160169,
+      "num_tokens": 11983604.0,
+      "step": 1220
+    },
+    {
+      "entropy": 2.588004069030285,
+      "epoch": 3.9312,
+      "grad_norm": 17.375,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 2.5174,
+      "mean_token_accuracy": 0.5051012465730309,
+      "num_tokens": 12076708.0,
+      "step": 1230
+    },
+    {
+      "entropy": 2.6090625591576098,
+      "epoch": 3.9632,
+      "grad_norm": 16.875,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 2.5337,
+      "mean_token_accuracy": 0.5051190486177802,
+      "num_tokens": 12176747.0,
+      "step": 1240
+    },
+    {
+      "entropy": 2.584036312997341,
+      "epoch": 3.9952,
+      "grad_norm": 19.75,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 2.4902,
+      "mean_token_accuracy": 0.5087237313389779,
+      "num_tokens": 12272241.0,
+      "step": 1250
+    },
+    {
+      "entropy": 2.472858669726472,
+      "epoch": 4.0256,
+      "grad_norm": 18.125,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 2.3856,
+      "mean_token_accuracy": 0.521043097110171,
+      "num_tokens": 12359739.0,
+      "step": 1260
+    },
+    {
+      "entropy": 2.34643474444747,
+      "epoch": 4.0576,
+      "grad_norm": 21.0,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 2.2626,
+      "mean_token_accuracy": 0.5364229042083025,
+      "num_tokens": 12460555.0,
+      "step": 1270
+    },
+    {
+      "entropy": 2.4080297112464906,
+      "epoch": 4.0896,
+      "grad_norm": 17.25,
+      "learning_rate": 1.938e-05,
+      "loss": 2.355,
+      "mean_token_accuracy": 0.5255401773378253,
+      "num_tokens": 12554461.0,
+      "step": 1280
+    },
+    {
+      "entropy": 2.394774776697159,
+      "epoch": 4.1216,
+      "grad_norm": 16.5,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 2.3188,
+      "mean_token_accuracy": 0.5288082377985119,
+      "num_tokens": 12653465.0,
+      "step": 1290
+    },
+    {
+      "entropy": 2.391275504231453,
+      "epoch": 4.1536,
+      "grad_norm": 21.625,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 2.2848,
+      "mean_token_accuracy": 0.5329740956425667,
+      "num_tokens": 12748340.0,
+      "step": 1300
+    },
+    {
+      "epoch": 4.1536,
+      "eval_coding_entropy": 1.6686400065422058,
+      "eval_coding_loss": 2.2645514011383057,
+      "eval_coding_mean_token_accuracy": 0.5102254449129104,
+      "eval_coding_num_tokens": 12748340.0,
+      "eval_coding_runtime": 79.0388,
+      "eval_coding_samples_per_second": 6.326,
+      "eval_coding_steps_per_second": 3.163,
+      "step": 1300
+    },
+    {
+      "epoch": 4.1536,
+      "eval_biology_entropy": 2.3491821846961973,
+      "eval_biology_loss": 2.634176015853882,
+      "eval_biology_mean_token_accuracy": 0.5061246186494828,
+      "eval_biology_num_tokens": 12748340.0,
+      "eval_biology_runtime": 34.6148,
+      "eval_biology_samples_per_second": 14.445,
+      "eval_biology_steps_per_second": 7.222,
+      "step": 1300
+    },
+    {
+      "entropy": 2.3668478064239027,
+      "epoch": 4.1856,
+      "grad_norm": 20.125,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 2.2853,
+      "mean_token_accuracy": 0.5337071912363172,
+      "num_tokens": 12847096.0,
+      "step": 1310
+    },
+    {
+      "entropy": 2.311310113221407,
+      "epoch": 4.2176,
+      "grad_norm": 15.0625,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 2.2384,
+      "mean_token_accuracy": 0.5418357422575355,
+      "num_tokens": 12946388.0,
+      "step": 1320
+    },
+    {
+      "entropy": 2.352651536464691,
+      "epoch": 4.2496,
+      "grad_norm": 16.375,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 2.2586,
+      "mean_token_accuracy": 0.5339978907257319,
+      "num_tokens": 13042597.0,
+      "step": 1330
+    },
+    {
+      "entropy": 2.3916583247482777,
+      "epoch": 4.2816,
+      "grad_norm": 20.875,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 2.3214,
+      "mean_token_accuracy": 0.5277318973094225,
+      "num_tokens": 13137019.0,
+      "step": 1340
+    },
+    {
+      "entropy": 2.407442457228899,
+      "epoch": 4.3136,
+      "grad_norm": 22.375,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 2.3233,
+      "mean_token_accuracy": 0.529756323620677,
+      "num_tokens": 13234780.0,
+      "step": 1350
+    },
+    {
+      "entropy": 2.370780614763498,
+      "epoch": 4.3456,
+      "grad_norm": 16.75,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 2.2856,
+      "mean_token_accuracy": 0.5306474581360817,
+      "num_tokens": 13334469.0,
+      "step": 1360
+    },
+    {
+      "entropy": 2.375711131095886,
+      "epoch": 4.3776,
+      "grad_norm": 17.125,
+      "learning_rate": 1.918e-05,
+      "loss": 2.3092,
+      "mean_token_accuracy": 0.5324419310316444,
+      "num_tokens": 13432836.0,
+      "step": 1370
+    },
+    {
+      "entropy": 2.4337742544710634,
+      "epoch": 4.4096,
+      "grad_norm": 15.3125,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 2.3333,
+      "mean_token_accuracy": 0.5236062532290816,
+      "num_tokens": 13530508.0,
+      "step": 1380
+    },
+    {
+      "entropy": 2.330331479758024,
+      "epoch": 4.4416,
+      "grad_norm": 18.125,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 2.2633,
+      "mean_token_accuracy": 0.5389384172856808,
+      "num_tokens": 13633161.0,
+      "step": 1390
+    },
+    {
+      "entropy": 2.3866625733673574,
+      "epoch": 4.4736,
+      "grad_norm": 15.375,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 2.2892,
+      "mean_token_accuracy": 0.5333944335579872,
+      "num_tokens": 13731330.0,
+      "step": 1400
+    },
+    {
+      "epoch": 4.4736,
+      "eval_coding_entropy": 1.7261567716598512,
+      "eval_coding_loss": 2.27376389503479,
+      "eval_coding_mean_token_accuracy": 0.5079633537530899,
+      "eval_coding_num_tokens": 13731330.0,
+      "eval_coding_runtime": 78.9748,
+      "eval_coding_samples_per_second": 6.331,
+      "eval_coding_steps_per_second": 3.166,
+      "step": 1400
+    },
+    {
+      "epoch": 4.4736,
+      "eval_biology_entropy": 2.3869504251480103,
+      "eval_biology_loss": 2.5860044956207275,
+      "eval_biology_mean_token_accuracy": 0.5117661204338074,
+      "eval_biology_num_tokens": 13731330.0,
+      "eval_biology_runtime": 34.6224,
+      "eval_biology_samples_per_second": 14.442,
+      "eval_biology_steps_per_second": 7.221,
+      "step": 1400
+    },
+    {
+      "entropy": 2.3977055683732034,
+      "epoch": 4.5056,
+      "grad_norm": 17.0,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 2.3212,
+      "mean_token_accuracy": 0.526662690937519,
+      "num_tokens": 13822911.0,
+      "step": 1410
+    },
+    {
+      "entropy": 2.313427411019802,
+      "epoch": 4.5376,
+      "grad_norm": 17.375,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 2.2541,
+      "mean_token_accuracy": 0.5411791836842894,
+      "num_tokens": 13926058.0,
+      "step": 1420
+    },
+    {
+      "entropy": 2.2551471143960953,
+      "epoch": 4.5696,
+      "grad_norm": 17.0,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 2.1854,
+      "mean_token_accuracy": 0.5493083782494068,
+      "num_tokens": 14035552.0,
+      "step": 1430
+    },
+    {
+      "entropy": 2.3578838713467123,
+      "epoch": 4.6016,
+      "grad_norm": 16.625,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 2.2836,
+      "mean_token_accuracy": 0.5363319680094719,
+      "num_tokens": 14134769.0,
+      "step": 1440
+    },
+    {
+      "entropy": 2.3566885352134705,
+      "epoch": 4.6336,
+      "grad_norm": 21.625,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 2.2625,
+      "mean_token_accuracy": 0.5347570797428489,
+      "num_tokens": 14232004.0,
+      "step": 1450
+    },
+    {
+      "entropy": 2.3043198555707933,
+      "epoch": 4.6655999999999995,
+      "grad_norm": 16.875,
+      "learning_rate": 1.898e-05,
+      "loss": 2.2346,
+      "mean_token_accuracy": 0.5425456454977393,
+      "num_tokens": 14333953.0,
+      "step": 1460
+    },
+    {
+      "entropy": 2.310309498012066,
+      "epoch": 4.6975999999999996,
+      "grad_norm": 16.625,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 2.2344,
+      "mean_token_accuracy": 0.5393823731690646,
+      "num_tokens": 14437164.0,
+      "step": 1470
+    },
+    {
+      "entropy": 2.373570270091295,
+      "epoch": 4.7296,
+      "grad_norm": 19.25,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 2.2902,
+      "mean_token_accuracy": 0.5355667784810066,
+      "num_tokens": 14532641.0,
+      "step": 1480
+    },
+    {
+      "entropy": 2.345385494083166,
+      "epoch": 4.7616,
+      "grad_norm": 18.25,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 2.2528,
+      "mean_token_accuracy": 0.5387666182592511,
+      "num_tokens": 14633165.0,
+      "step": 1490
+    },
+    {
+      "entropy": 2.3080789014697074,
+      "epoch": 4.7936,
+      "grad_norm": 16.625,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 2.2308,
+      "mean_token_accuracy": 0.5400335062295198,
+      "num_tokens": 14732212.0,
+      "step": 1500
+    },
+    {
+      "epoch": 4.7936,
+      "eval_coding_entropy": 1.73171142911911,
+      "eval_coding_loss": 2.283193826675415,
+      "eval_coding_mean_token_accuracy": 0.508676981806755,
+      "eval_coding_num_tokens": 14732212.0,
+      "eval_coding_runtime": 78.9397,
+      "eval_coding_samples_per_second": 6.334,
+      "eval_coding_steps_per_second": 3.167,
+      "step": 1500
+    },
+    {
+      "epoch": 4.7936,
+      "eval_biology_entropy": 2.392736424446106,
+      "eval_biology_loss": 2.536526679992676,
+      "eval_biology_mean_token_accuracy": 0.5183096739053726,
+      "eval_biology_num_tokens": 14732212.0,
+      "eval_biology_runtime": 34.5523,
+      "eval_biology_samples_per_second": 14.471,
+      "eval_biology_steps_per_second": 7.235,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 32,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.813670757145723e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79db69065fa1ad5ecafbaff0d1022e56d5ce51ba2b1b123bcd24bb6455fb147e
+size 6353