Upload folder using huggingface_hub

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-1500/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-1500/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-1500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-1500/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-1500/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-1500/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50ecefd5d553711554c622b14d60a405d47e9b36d701e58153ca01e6bee12161
+size 4962221464

checkpoint-1500/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03a5762367f0fc6b55634cfbd048c217d9dba1f21be225a06f62eb4449f6d32b
+size 3670322200

checkpoint-1500/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-1500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb04253b8087f37bdc4e412f52d99e97e7f047462692095a522970a851161188
+size 7927918951

checkpoint-1500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7905839719e409e7a3dc16880442546c5525350d5767488905f5a02ad96619e2
+size 14645

checkpoint-1500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb8afd6405fc2f72bcdc3e89147a10597d475bf4457b8f98d9c27aa5442f3ef9
+size 1465

checkpoint-1500/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-1500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-1500/trainer_state.json

@@ -0,0 +1,1864 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.4,
+  "eval_steps": 100,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.5915625745430588,
+      "epoch": 0.016,
+      "grad_norm": 5.71875,
+      "learning_rate": 1.8e-07,
+      "loss": 0.8466,
+      "mean_token_accuracy": 0.8017937403172255,
+      "num_tokens": 340696.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.5926362803205848,
+      "epoch": 0.032,
+      "grad_norm": 5.59375,
+      "learning_rate": 3.8e-07,
+      "loss": 0.8487,
+      "mean_token_accuracy": 0.8023959375917912,
+      "num_tokens": 675930.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.591606929153204,
+      "epoch": 0.048,
+      "grad_norm": 4.96875,
+      "learning_rate": 5.800000000000001e-07,
+      "loss": 0.8302,
+      "mean_token_accuracy": 0.8044086921960115,
+      "num_tokens": 1007956.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.6204556789249182,
+      "epoch": 0.064,
+      "grad_norm": 4.9375,
+      "learning_rate": 7.8e-07,
+      "loss": 0.8457,
+      "mean_token_accuracy": 0.7998181059956551,
+      "num_tokens": 1340560.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.6393004363402724,
+      "epoch": 0.08,
+      "grad_norm": 4.3125,
+      "learning_rate": 9.800000000000001e-07,
+      "loss": 0.8343,
+      "mean_token_accuracy": 0.8025688324123621,
+      "num_tokens": 1679432.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.6593567343428731,
+      "epoch": 0.096,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 0.8046,
+      "mean_token_accuracy": 0.8078926544636488,
+      "num_tokens": 2021570.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.7051705561578274,
+      "epoch": 0.112,
+      "grad_norm": 2.8125,
+      "learning_rate": 1.3800000000000001e-06,
+      "loss": 0.8246,
+      "mean_token_accuracy": 0.8029463469982148,
+      "num_tokens": 2359297.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.754444376192987,
+      "epoch": 0.128,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.5800000000000001e-06,
+      "loss": 0.8435,
+      "mean_token_accuracy": 0.79781529083848,
+      "num_tokens": 2683168.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.7250820267945528,
+      "epoch": 0.144,
+      "grad_norm": 1.90625,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 0.8195,
+      "mean_token_accuracy": 0.8042717296630144,
+      "num_tokens": 3019426.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.7252413807436824,
+      "epoch": 0.16,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.98e-06,
+      "loss": 0.8097,
+      "mean_token_accuracy": 0.8047077178955078,
+      "num_tokens": 3354390.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_chemistry_entropy": 0.7109885907173157,
+      "eval_chemistry_loss": 0.804060161113739,
+      "eval_chemistry_mean_token_accuracy": 0.8113441648483276,
+      "eval_chemistry_num_tokens": 3354390.0,
+      "eval_chemistry_runtime": 34.8657,
+      "eval_chemistry_samples_per_second": 14.341,
+      "eval_chemistry_steps_per_second": 3.585,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_physics_entropy": 0.7321163377761841,
+      "eval_physics_loss": 0.8287621140480042,
+      "eval_physics_mean_token_accuracy": 0.8033607420921326,
+      "eval_physics_num_tokens": 3354390.0,
+      "eval_physics_runtime": 41.3727,
+      "eval_physics_samples_per_second": 12.085,
+      "eval_physics_steps_per_second": 3.021,
+      "step": 100
+    },
+    {
+      "entropy": 0.7267085557803512,
+      "epoch": 0.176,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 0.8058,
+      "mean_token_accuracy": 0.8033798363059759,
+      "num_tokens": 3684374.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.7212623069062829,
+      "epoch": 0.192,
+      "grad_norm": 1.828125,
+      "learning_rate": 2.38e-06,
+      "loss": 0.7989,
+      "mean_token_accuracy": 0.8043727930635214,
+      "num_tokens": 4023915.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.7022871913388371,
+      "epoch": 0.208,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.5800000000000003e-06,
+      "loss": 0.7758,
+      "mean_token_accuracy": 0.8102051325142383,
+      "num_tokens": 4352045.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.6965231081470847,
+      "epoch": 0.224,
+      "grad_norm": 1.8203125,
+      "learning_rate": 2.7800000000000005e-06,
+      "loss": 0.7726,
+      "mean_token_accuracy": 0.8089764468371868,
+      "num_tokens": 4681371.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.6956626677885651,
+      "epoch": 0.24,
+      "grad_norm": 1.6640625,
+      "learning_rate": 2.9800000000000003e-06,
+      "loss": 0.7608,
+      "mean_token_accuracy": 0.8104452796280384,
+      "num_tokens": 5021784.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.708347842283547,
+      "epoch": 0.256,
+      "grad_norm": 1.9453125,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 0.7636,
+      "mean_token_accuracy": 0.809263763204217,
+      "num_tokens": 5367308.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.7138571232557297,
+      "epoch": 0.272,
+      "grad_norm": 2.03125,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": 0.7721,
+      "mean_token_accuracy": 0.8050564888864755,
+      "num_tokens": 5699101.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.7222439678385854,
+      "epoch": 0.288,
+      "grad_norm": 1.7109375,
+      "learning_rate": 3.58e-06,
+      "loss": 0.7766,
+      "mean_token_accuracy": 0.8063390862196684,
+      "num_tokens": 6022837.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.7117253359407186,
+      "epoch": 0.304,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 0.7535,
+      "mean_token_accuracy": 0.809041840955615,
+      "num_tokens": 6343826.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.7200546801090241,
+      "epoch": 0.32,
+      "grad_norm": 1.7265625,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 0.7524,
+      "mean_token_accuracy": 0.8073045868426562,
+      "num_tokens": 6670164.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_chemistry_entropy": 0.711975371837616,
+      "eval_chemistry_loss": 0.7488377690315247,
+      "eval_chemistry_mean_token_accuracy": 0.8110983457565307,
+      "eval_chemistry_num_tokens": 6670164.0,
+      "eval_chemistry_runtime": 34.8588,
+      "eval_chemistry_samples_per_second": 14.344,
+      "eval_chemistry_steps_per_second": 3.586,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_physics_entropy": 0.7178810501098633,
+      "eval_physics_loss": 0.7585415244102478,
+      "eval_physics_mean_token_accuracy": 0.8081966361999512,
+      "eval_physics_num_tokens": 6670164.0,
+      "eval_physics_runtime": 41.7087,
+      "eval_physics_samples_per_second": 11.988,
+      "eval_physics_steps_per_second": 2.997,
+      "step": 200
+    },
+    {
+      "entropy": 0.7111945543438196,
+      "epoch": 0.336,
+      "grad_norm": 1.578125,
+      "learning_rate": 4.18e-06,
+      "loss": 0.7313,
+      "mean_token_accuracy": 0.8092666126787662,
+      "num_tokens": 7003404.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.6962820990011096,
+      "epoch": 0.352,
+      "grad_norm": 1.8828125,
+      "learning_rate": 4.38e-06,
+      "loss": 0.712,
+      "mean_token_accuracy": 0.8133333619683981,
+      "num_tokens": 7342593.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.6933605512604117,
+      "epoch": 0.368,
+      "grad_norm": 1.6484375,
+      "learning_rate": 4.58e-06,
+      "loss": 0.6986,
+      "mean_token_accuracy": 0.8163042381405831,
+      "num_tokens": 7675569.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.7312716225162148,
+      "epoch": 0.384,
+      "grad_norm": 1.9296875,
+      "learning_rate": 4.78e-06,
+      "loss": 0.7319,
+      "mean_token_accuracy": 0.8038436233997345,
+      "num_tokens": 8012218.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.6996616214513779,
+      "epoch": 0.4,
+      "grad_norm": 1.796875,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 0.6917,
+      "mean_token_accuracy": 0.8134940605610609,
+      "num_tokens": 8350120.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.6945684082806111,
+      "epoch": 0.416,
+      "grad_norm": 1.6953125,
+      "learning_rate": 5.18e-06,
+      "loss": 0.678,
+      "mean_token_accuracy": 0.8151624854654074,
+      "num_tokens": 8688773.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.7035746525973081,
+      "epoch": 0.432,
+      "grad_norm": 1.8359375,
+      "learning_rate": 5.380000000000001e-06,
+      "loss": 0.6888,
+      "mean_token_accuracy": 0.8118865180760622,
+      "num_tokens": 9022094.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.6850203935056924,
+      "epoch": 0.448,
+      "grad_norm": 1.6640625,
+      "learning_rate": 5.580000000000001e-06,
+      "loss": 0.6727,
+      "mean_token_accuracy": 0.8186012674123049,
+      "num_tokens": 9360608.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.6806540634483099,
+      "epoch": 0.464,
+      "grad_norm": 1.875,
+      "learning_rate": 5.78e-06,
+      "loss": 0.6592,
+      "mean_token_accuracy": 0.8200464811176061,
+      "num_tokens": 9688896.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.6852828785777092,
+      "epoch": 0.48,
+      "grad_norm": 1.828125,
+      "learning_rate": 5.98e-06,
+      "loss": 0.6672,
+      "mean_token_accuracy": 0.8180346965789795,
+      "num_tokens": 10025864.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_chemistry_entropy": 0.7274510788917542,
+      "eval_chemistry_loss": 0.7009354829788208,
+      "eval_chemistry_mean_token_accuracy": 0.8133535690307617,
+      "eval_chemistry_num_tokens": 10025864.0,
+      "eval_chemistry_runtime": 34.811,
+      "eval_chemistry_samples_per_second": 14.363,
+      "eval_chemistry_steps_per_second": 3.591,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_physics_entropy": 0.7100022671222687,
+      "eval_physics_loss": 0.7013525366783142,
+      "eval_physics_mean_token_accuracy": 0.813646514415741,
+      "eval_physics_num_tokens": 10025864.0,
+      "eval_physics_runtime": 41.4407,
+      "eval_physics_samples_per_second": 12.065,
+      "eval_physics_steps_per_second": 3.016,
+      "step": 300
+    },
+    {
+      "entropy": 0.7103753052651882,
+      "epoch": 0.496,
+      "grad_norm": 1.6171875,
+      "learning_rate": 6.18e-06,
+      "loss": 0.6847,
+      "mean_token_accuracy": 0.812765546515584,
+      "num_tokens": 10362573.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.6982777336612344,
+      "epoch": 0.512,
+      "grad_norm": 1.8125,
+      "learning_rate": 6.380000000000001e-06,
+      "loss": 0.6716,
+      "mean_token_accuracy": 0.8169939771294594,
+      "num_tokens": 10694440.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.708159969188273,
+      "epoch": 0.528,
+      "grad_norm": 1.7421875,
+      "learning_rate": 6.5800000000000005e-06,
+      "loss": 0.693,
+      "mean_token_accuracy": 0.8125004164874554,
+      "num_tokens": 11022357.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.699538960121572,
+      "epoch": 0.544,
+      "grad_norm": 1.828125,
+      "learning_rate": 6.780000000000001e-06,
+      "loss": 0.6711,
+      "mean_token_accuracy": 0.8153361644595861,
+      "num_tokens": 11345911.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.6910958087071777,
+      "epoch": 0.56,
+      "grad_norm": 1.5078125,
+      "learning_rate": 6.98e-06,
+      "loss": 0.6659,
+      "mean_token_accuracy": 0.8177970543503761,
+      "num_tokens": 11687225.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.6805725138634443,
+      "epoch": 0.576,
+      "grad_norm": 1.6484375,
+      "learning_rate": 7.180000000000001e-06,
+      "loss": 0.6565,
+      "mean_token_accuracy": 0.8207161612808704,
+      "num_tokens": 12015720.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.6865102933719754,
+      "epoch": 0.592,
+      "grad_norm": 1.84375,
+      "learning_rate": 7.3800000000000005e-06,
+      "loss": 0.6638,
+      "mean_token_accuracy": 0.8175321508198976,
+      "num_tokens": 12343393.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.6841516578570008,
+      "epoch": 0.608,
+      "grad_norm": 1.6328125,
+      "learning_rate": 7.58e-06,
+      "loss": 0.6582,
+      "mean_token_accuracy": 0.8192304316908121,
+      "num_tokens": 12671474.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.7005494212731719,
+      "epoch": 0.624,
+      "grad_norm": 1.6015625,
+      "learning_rate": 7.78e-06,
+      "loss": 0.6674,
+      "mean_token_accuracy": 0.8156571265310049,
+      "num_tokens": 12996077.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.6937378702685237,
+      "epoch": 0.64,
+      "grad_norm": 1.6171875,
+      "learning_rate": 7.980000000000002e-06,
+      "loss": 0.6701,
+      "mean_token_accuracy": 0.8157013915479183,
+      "num_tokens": 13325291.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_chemistry_entropy": 0.7147737612724304,
+      "eval_chemistry_loss": 0.692807137966156,
+      "eval_chemistry_mean_token_accuracy": 0.8137030324935913,
+      "eval_chemistry_num_tokens": 13325291.0,
+      "eval_chemistry_runtime": 34.8141,
+      "eval_chemistry_samples_per_second": 14.362,
+      "eval_chemistry_steps_per_second": 3.591,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_physics_entropy": 0.684398841381073,
+      "eval_physics_loss": 0.6776748895645142,
+      "eval_physics_mean_token_accuracy": 0.8169920797348023,
+      "eval_physics_num_tokens": 13325291.0,
+      "eval_physics_runtime": 41.4558,
+      "eval_physics_samples_per_second": 12.061,
+      "eval_physics_steps_per_second": 3.015,
+      "step": 400
+    },
+    {
+      "entropy": 0.6792764134705067,
+      "epoch": 0.656,
+      "grad_norm": 1.625,
+      "learning_rate": 8.18e-06,
+      "loss": 0.6538,
+      "mean_token_accuracy": 0.8188512332737445,
+      "num_tokens": 13671434.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.6741176716983318,
+      "epoch": 0.672,
+      "grad_norm": 1.703125,
+      "learning_rate": 8.380000000000001e-06,
+      "loss": 0.6479,
+      "mean_token_accuracy": 0.8206427529454231,
+      "num_tokens": 13993668.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.6748962674289942,
+      "epoch": 0.688,
+      "grad_norm": 1.5625,
+      "learning_rate": 8.580000000000001e-06,
+      "loss": 0.6489,
+      "mean_token_accuracy": 0.821071095019579,
+      "num_tokens": 14334907.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.6882562624290586,
+      "epoch": 0.704,
+      "grad_norm": 1.765625,
+      "learning_rate": 8.78e-06,
+      "loss": 0.6631,
+      "mean_token_accuracy": 0.8166982632130384,
+      "num_tokens": 14670639.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.6710160901769996,
+      "epoch": 0.72,
+      "grad_norm": 1.828125,
+      "learning_rate": 8.98e-06,
+      "loss": 0.6451,
+      "mean_token_accuracy": 0.8207047082483768,
+      "num_tokens": 14987995.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.6616904048249126,
+      "epoch": 0.736,
+      "grad_norm": 1.65625,
+      "learning_rate": 9.180000000000002e-06,
+      "loss": 0.6374,
+      "mean_token_accuracy": 0.8214268710464239,
+      "num_tokens": 15324659.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.6631792033091187,
+      "epoch": 0.752,
+      "grad_norm": 1.6171875,
+      "learning_rate": 9.38e-06,
+      "loss": 0.6379,
+      "mean_token_accuracy": 0.8226557578891516,
+      "num_tokens": 15657342.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.6816092301160097,
+      "epoch": 0.768,
+      "grad_norm": 1.640625,
+      "learning_rate": 9.58e-06,
+      "loss": 0.6582,
+      "mean_token_accuracy": 0.8162681456655264,
+      "num_tokens": 15988993.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.6584033869206906,
+      "epoch": 0.784,
+      "grad_norm": 1.6484375,
+      "learning_rate": 9.780000000000001e-06,
+      "loss": 0.6307,
+      "mean_token_accuracy": 0.8244805630296469,
+      "num_tokens": 16333857.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.6675900464877487,
+      "epoch": 0.8,
+      "grad_norm": 1.625,
+      "learning_rate": 9.980000000000001e-06,
+      "loss": 0.6444,
+      "mean_token_accuracy": 0.8208535581827163,
+      "num_tokens": 16664690.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_chemistry_entropy": 0.7186227707862854,
+      "eval_chemistry_loss": 0.6891804337501526,
+      "eval_chemistry_mean_token_accuracy": 0.812873167514801,
+      "eval_chemistry_num_tokens": 16664690.0,
+      "eval_chemistry_runtime": 34.7308,
+      "eval_chemistry_samples_per_second": 14.396,
+      "eval_chemistry_steps_per_second": 3.599,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_physics_entropy": 0.6725036988258362,
+      "eval_physics_loss": 0.6597840785980225,
+      "eval_physics_mean_token_accuracy": 0.8193980932235718,
+      "eval_physics_num_tokens": 16664690.0,
+      "eval_physics_runtime": 41.0989,
+      "eval_physics_samples_per_second": 12.166,
+      "eval_physics_steps_per_second": 3.041,
+      "step": 500
+    },
+    {
+      "entropy": 0.6641527567058801,
+      "epoch": 0.816,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.018e-05,
+      "loss": 0.6389,
+      "mean_token_accuracy": 0.8205956391990185,
+      "num_tokens": 16989307.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.6729596307501197,
+      "epoch": 0.832,
+      "grad_norm": 1.5859375,
+      "learning_rate": 1.038e-05,
+      "loss": 0.6418,
+      "mean_token_accuracy": 0.8202375840395689,
+      "num_tokens": 17317524.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.6599426086992025,
+      "epoch": 0.848,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.0580000000000002e-05,
+      "loss": 0.6326,
+      "mean_token_accuracy": 0.8230770550668239,
+      "num_tokens": 17650424.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.66338025983423,
+      "epoch": 0.864,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.0780000000000002e-05,
+      "loss": 0.6412,
+      "mean_token_accuracy": 0.8213306616991758,
+      "num_tokens": 17985547.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.6687672117725014,
+      "epoch": 0.88,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.0980000000000002e-05,
+      "loss": 0.6426,
+      "mean_token_accuracy": 0.8197343621402979,
+      "num_tokens": 18315392.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.6246686290949583,
+      "epoch": 0.896,
+      "grad_norm": 1.375,
+      "learning_rate": 1.1180000000000001e-05,
+      "loss": 0.6014,
+      "mean_token_accuracy": 0.8294213119894266,
+      "num_tokens": 18667124.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.6640157140791416,
+      "epoch": 0.912,
+      "grad_norm": 1.5,
+      "learning_rate": 1.138e-05,
+      "loss": 0.6371,
+      "mean_token_accuracy": 0.820187458768487,
+      "num_tokens": 19002186.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.6525983370840549,
+      "epoch": 0.928,
+      "grad_norm": 1.6640625,
+      "learning_rate": 1.1580000000000001e-05,
+      "loss": 0.6304,
+      "mean_token_accuracy": 0.8224839419126511,
+      "num_tokens": 19335013.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.6459019472822547,
+      "epoch": 0.944,
+      "grad_norm": 1.6171875,
+      "learning_rate": 1.178e-05,
+      "loss": 0.6208,
+      "mean_token_accuracy": 0.8251514233648777,
+      "num_tokens": 19662733.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.6452493501827121,
+      "epoch": 0.96,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.198e-05,
+      "loss": 0.6276,
+      "mean_token_accuracy": 0.8237052172422409,
+      "num_tokens": 19996789.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_chemistry_entropy": 0.7299397034645081,
+      "eval_chemistry_loss": 0.6870628595352173,
+      "eval_chemistry_mean_token_accuracy": 0.8117328572273255,
+      "eval_chemistry_num_tokens": 19996789.0,
+      "eval_chemistry_runtime": 34.7851,
+      "eval_chemistry_samples_per_second": 14.374,
+      "eval_chemistry_steps_per_second": 3.593,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_physics_entropy": 0.6775157516002654,
+      "eval_physics_loss": 0.646020770072937,
+      "eval_physics_mean_token_accuracy": 0.8218063368797303,
+      "eval_physics_num_tokens": 19996789.0,
+      "eval_physics_runtime": 41.4104,
+      "eval_physics_samples_per_second": 12.074,
+      "eval_physics_steps_per_second": 3.019,
+      "step": 600
+    },
+    {
+      "entropy": 0.6451076414436102,
+      "epoch": 0.976,
+      "grad_norm": 1.5859375,
+      "learning_rate": 1.218e-05,
+      "loss": 0.6221,
+      "mean_token_accuracy": 0.825892923027277,
+      "num_tokens": 20332636.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.6509507959708571,
+      "epoch": 0.992,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.2380000000000002e-05,
+      "loss": 0.6214,
+      "mean_token_accuracy": 0.8233620677143335,
+      "num_tokens": 20662209.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.6204982601106167,
+      "epoch": 1.008,
+      "grad_norm": 1.46875,
+      "learning_rate": 1.2580000000000002e-05,
+      "loss": 0.5988,
+      "mean_token_accuracy": 0.8286421094089746,
+      "num_tokens": 21005731.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.6242268454283476,
+      "epoch": 1.024,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.2780000000000001e-05,
+      "loss": 0.6046,
+      "mean_token_accuracy": 0.82848214879632,
+      "num_tokens": 21335407.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.6163413025438785,
+      "epoch": 1.04,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.2980000000000001e-05,
+      "loss": 0.598,
+      "mean_token_accuracy": 0.8299466107040644,
+      "num_tokens": 21668490.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.6118087394163012,
+      "epoch": 1.056,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.3180000000000001e-05,
+      "loss": 0.5883,
+      "mean_token_accuracy": 0.8311309419572354,
+      "num_tokens": 22006039.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.6269845755770802,
+      "epoch": 1.072,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.3380000000000002e-05,
+      "loss": 0.6043,
+      "mean_token_accuracy": 0.8276471871882677,
+      "num_tokens": 22348206.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.6239592241123318,
+      "epoch": 1.088,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.3580000000000002e-05,
+      "loss": 0.5947,
+      "mean_token_accuracy": 0.8277118816971779,
+      "num_tokens": 22676163.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.6154211649671197,
+      "epoch": 1.104,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.378e-05,
+      "loss": 0.5951,
+      "mean_token_accuracy": 0.8299165233969689,
+      "num_tokens": 23009279.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.6186880182474852,
+      "epoch": 1.12,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.398e-05,
+      "loss": 0.5987,
+      "mean_token_accuracy": 0.827697791159153,
+      "num_tokens": 23346828.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_chemistry_entropy": 0.6806538386344909,
+      "eval_chemistry_loss": 0.6874776482582092,
+      "eval_chemistry_mean_token_accuracy": 0.8116622562408448,
+      "eval_chemistry_num_tokens": 23346828.0,
+      "eval_chemistry_runtime": 35.1659,
+      "eval_chemistry_samples_per_second": 14.218,
+      "eval_chemistry_steps_per_second": 3.555,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_physics_entropy": 0.6262999680042267,
+      "eval_physics_loss": 0.6376790404319763,
+      "eval_physics_mean_token_accuracy": 0.8227445411682129,
+      "eval_physics_num_tokens": 23346828.0,
+      "eval_physics_runtime": 41.445,
+      "eval_physics_samples_per_second": 12.064,
+      "eval_physics_steps_per_second": 3.016,
+      "step": 700
+    },
+    {
+      "entropy": 0.5965470356866718,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 1.359375,
+      "learning_rate": 1.418e-05,
+      "loss": 0.5793,
+      "mean_token_accuracy": 0.8342778768390418,
+      "num_tokens": 23689161.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.6170177994295954,
+      "epoch": 1.152,
+      "grad_norm": 1.6015625,
+      "learning_rate": 1.4380000000000001e-05,
+      "loss": 0.5984,
+      "mean_token_accuracy": 0.8301392119377852,
+      "num_tokens": 24016439.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.6231384797021746,
+      "epoch": 1.168,
+      "grad_norm": 1.640625,
+      "learning_rate": 1.4580000000000001e-05,
+      "loss": 0.6024,
+      "mean_token_accuracy": 0.8265216089785099,
+      "num_tokens": 24344431.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.6168253228068352,
+      "epoch": 1.184,
+      "grad_norm": 1.734375,
+      "learning_rate": 1.478e-05,
+      "loss": 0.5912,
+      "mean_token_accuracy": 0.8295746680349112,
+      "num_tokens": 24668087.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.6063494171947241,
+      "epoch": 1.2,
+      "grad_norm": 1.3828125,
+      "learning_rate": 1.498e-05,
+      "loss": 0.5805,
+      "mean_token_accuracy": 0.8319725230336189,
+      "num_tokens": 25000629.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.6178384765982627,
+      "epoch": 1.216,
+      "grad_norm": 1.6796875,
+      "learning_rate": 1.5180000000000002e-05,
+      "loss": 0.5998,
+      "mean_token_accuracy": 0.8278374817222357,
+      "num_tokens": 25324324.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.6116716200485826,
+      "epoch": 1.232,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.5380000000000002e-05,
+      "loss": 0.5838,
+      "mean_token_accuracy": 0.8310758478939533,
+      "num_tokens": 25658795.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.6036085536703467,
+      "epoch": 1.248,
+      "grad_norm": 1.4609375,
+      "learning_rate": 1.5580000000000003e-05,
+      "loss": 0.5835,
+      "mean_token_accuracy": 0.8323761031031609,
+      "num_tokens": 25993177.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.6078255159780384,
+      "epoch": 1.264,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.578e-05,
+      "loss": 0.5819,
+      "mean_token_accuracy": 0.8314513519406319,
+      "num_tokens": 26329726.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.6271371256560088,
+      "epoch": 1.28,
+      "grad_norm": 1.4609375,
+      "learning_rate": 1.5980000000000003e-05,
+      "loss": 0.6049,
+      "mean_token_accuracy": 0.8257384181022644,
+      "num_tokens": 26664093.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_chemistry_entropy": 0.6834847524166107,
+      "eval_chemistry_loss": 0.6882474422454834,
+      "eval_chemistry_mean_token_accuracy": 0.8116786751747131,
+      "eval_chemistry_num_tokens": 26664093.0,
+      "eval_chemistry_runtime": 34.7916,
+      "eval_chemistry_samples_per_second": 14.371,
+      "eval_chemistry_steps_per_second": 3.593,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_physics_entropy": 0.6253828897476196,
+      "eval_physics_loss": 0.6271702647209167,
+      "eval_physics_mean_token_accuracy": 0.8248588938713074,
+      "eval_physics_num_tokens": 26664093.0,
+      "eval_physics_runtime": 41.5612,
+      "eval_physics_samples_per_second": 12.03,
+      "eval_physics_steps_per_second": 3.008,
+      "step": 800
+    },
+    {
+      "entropy": 0.5998634301126003,
+      "epoch": 1.296,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.618e-05,
+      "loss": 0.5787,
+      "mean_token_accuracy": 0.8333883356302977,
+      "num_tokens": 26994877.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.6259857648983598,
+      "epoch": 1.312,
+      "grad_norm": 1.5703125,
+      "learning_rate": 1.638e-05,
+      "loss": 0.6016,
+      "mean_token_accuracy": 0.8277184132486581,
+      "num_tokens": 27328198.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.605760240368545,
+      "epoch": 1.328,
+      "grad_norm": 1.5546875,
+      "learning_rate": 1.658e-05,
+      "loss": 0.5891,
+      "mean_token_accuracy": 0.830071647465229,
+      "num_tokens": 27658818.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.6093621645122766,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 1.546875,
+      "learning_rate": 1.6780000000000002e-05,
+      "loss": 0.5882,
+      "mean_token_accuracy": 0.8291759472340345,
+      "num_tokens": 27991108.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.6125000966712832,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 1.3125,
+      "learning_rate": 1.698e-05,
+      "loss": 0.582,
+      "mean_token_accuracy": 0.8292897183448076,
+      "num_tokens": 28327775.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.6071364052593708,
+      "epoch": 1.376,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.718e-05,
+      "loss": 0.5823,
+      "mean_token_accuracy": 0.8310487173497677,
+      "num_tokens": 28666116.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.5939128069207072,
+      "epoch": 1.392,
+      "grad_norm": 1.4375,
+      "learning_rate": 1.7380000000000003e-05,
+      "loss": 0.5723,
+      "mean_token_accuracy": 0.8350477192550898,
+      "num_tokens": 29004293.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.6024157986044884,
+      "epoch": 1.408,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.758e-05,
+      "loss": 0.5827,
+      "mean_token_accuracy": 0.8312420520931483,
+      "num_tokens": 29330131.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.6242644298821688,
+      "epoch": 1.424,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.7780000000000003e-05,
+      "loss": 0.6039,
+      "mean_token_accuracy": 0.8263261545449495,
+      "num_tokens": 29661850.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.5897390013560653,
+      "epoch": 1.44,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.798e-05,
+      "loss": 0.5687,
+      "mean_token_accuracy": 0.8350921101868153,
+      "num_tokens": 29999392.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_chemistry_entropy": 0.6641502072811126,
+      "eval_chemistry_loss": 0.6906805634498596,
+      "eval_chemistry_mean_token_accuracy": 0.8092845277786255,
+      "eval_chemistry_num_tokens": 29999392.0,
+      "eval_chemistry_runtime": 34.7482,
+      "eval_chemistry_samples_per_second": 14.389,
+      "eval_chemistry_steps_per_second": 3.597,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_physics_entropy": 0.6082331819534301,
+      "eval_physics_loss": 0.6185559034347534,
+      "eval_physics_mean_token_accuracy": 0.8265736656188964,
+      "eval_physics_num_tokens": 29999392.0,
+      "eval_physics_runtime": 41.4301,
+      "eval_physics_samples_per_second": 12.069,
+      "eval_physics_steps_per_second": 3.017,
+      "step": 900
+    },
+    {
+      "entropy": 0.5889361893758178,
+      "epoch": 1.456,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.8180000000000002e-05,
+      "loss": 0.5722,
+      "mean_token_accuracy": 0.8360798362642526,
+      "num_tokens": 30341004.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.6223346112295985,
+      "epoch": 1.472,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.8380000000000004e-05,
+      "loss": 0.599,
+      "mean_token_accuracy": 0.8281814876943827,
+      "num_tokens": 30667689.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.6086510250344872,
+      "epoch": 1.488,
+      "grad_norm": 1.25,
+      "learning_rate": 1.858e-05,
+      "loss": 0.5865,
+      "mean_token_accuracy": 0.8305441569536924,
+      "num_tokens": 30999697.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.5955386120826006,
+      "epoch": 1.504,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.878e-05,
+      "loss": 0.5699,
+      "mean_token_accuracy": 0.8333926767110824,
+      "num_tokens": 31332775.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.6133549332618713,
+      "epoch": 1.52,
+      "grad_norm": 1.4609375,
+      "learning_rate": 1.898e-05,
+      "loss": 0.5915,
+      "mean_token_accuracy": 0.8288876831531524,
+      "num_tokens": 31666521.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.6097657412290574,
+      "epoch": 1.536,
+      "grad_norm": 1.4296875,
+      "learning_rate": 1.918e-05,
+      "loss": 0.5866,
+      "mean_token_accuracy": 0.8288511786609888,
+      "num_tokens": 31998232.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.5852528993040323,
+      "epoch": 1.552,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.938e-05,
+      "loss": 0.5684,
+      "mean_token_accuracy": 0.8353338003158569,
+      "num_tokens": 32328085.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.5982964089140296,
+      "epoch": 1.568,
+      "grad_norm": 1.375,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 0.5753,
+      "mean_token_accuracy": 0.832823446765542,
+      "num_tokens": 32656603.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.6145529938861728,
+      "epoch": 1.584,
+      "grad_norm": 1.3828125,
+      "learning_rate": 1.978e-05,
+      "loss": 0.5882,
+      "mean_token_accuracy": 0.8275394171476365,
+      "num_tokens": 32983769.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.6043317306786775,
+      "epoch": 1.6,
+      "grad_norm": 1.3125,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 0.5823,
+      "mean_token_accuracy": 0.8310152795165777,
+      "num_tokens": 33309282.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_chemistry_entropy": 0.6794673941135406,
+      "eval_chemistry_loss": 0.6896947026252747,
+      "eval_chemistry_mean_token_accuracy": 0.8101202611923217,
+      "eval_chemistry_num_tokens": 33309282.0,
+      "eval_chemistry_runtime": 34.8102,
+      "eval_chemistry_samples_per_second": 14.364,
+      "eval_chemistry_steps_per_second": 3.591,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_physics_entropy": 0.607559362411499,
+      "eval_physics_loss": 0.6121883392333984,
+      "eval_physics_mean_token_accuracy": 0.8272243642807007,
+      "eval_physics_num_tokens": 33309282.0,
+      "eval_physics_runtime": 41.0884,
+      "eval_physics_samples_per_second": 12.169,
+      "eval_physics_steps_per_second": 3.042,
+      "step": 1000
+    },
+    {
+      "entropy": 0.5851200748234987,
+      "epoch": 1.616,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.9980000000000002e-05,
+      "loss": 0.5675,
+      "mean_token_accuracy": 0.8361915960907936,
+      "num_tokens": 33647616.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.5935754748061299,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 1.3125,
+      "learning_rate": 1.995777777777778e-05,
+      "loss": 0.5753,
+      "mean_token_accuracy": 0.8332207947969437,
+      "num_tokens": 33977586.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.5731598405167461,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.9935555555555557e-05,
+      "loss": 0.5536,
+      "mean_token_accuracy": 0.8382600821554661,
+      "num_tokens": 34312436.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.5990395007655025,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 1.5390625,
+      "learning_rate": 1.9913333333333335e-05,
+      "loss": 0.5747,
+      "mean_token_accuracy": 0.8332929655909538,
+      "num_tokens": 34641549.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.589907999895513,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.9891111111111112e-05,
+      "loss": 0.5685,
+      "mean_token_accuracy": 0.8342956956475973,
+      "num_tokens": 34977031.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.5947070028632879,
+      "epoch": 1.696,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.986888888888889e-05,
+      "loss": 0.5736,
+      "mean_token_accuracy": 0.8329529393464327,
+      "num_tokens": 35300962.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.595324145630002,
+      "epoch": 1.712,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.9846666666666668e-05,
+      "loss": 0.574,
+      "mean_token_accuracy": 0.831612591072917,
+      "num_tokens": 35642662.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.581332896463573,
+      "epoch": 1.728,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.9824444444444445e-05,
+      "loss": 0.5648,
+      "mean_token_accuracy": 0.8354741148650646,
+      "num_tokens": 35980993.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.5898414205759763,
+      "epoch": 1.744,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.9802222222222226e-05,
+      "loss": 0.568,
+      "mean_token_accuracy": 0.834344669431448,
+      "num_tokens": 36326797.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.5798839410766959,
+      "epoch": 1.76,
+      "grad_norm": 1.3828125,
+      "learning_rate": 1.978e-05,
+      "loss": 0.5617,
+      "mean_token_accuracy": 0.8352703254669904,
+      "num_tokens": 36654797.0,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_chemistry_entropy": 0.662034518957138,
+      "eval_chemistry_loss": 0.6911905407905579,
+      "eval_chemistry_mean_token_accuracy": 0.8086481132507324,
+      "eval_chemistry_num_tokens": 36654797.0,
+      "eval_chemistry_runtime": 34.786,
+      "eval_chemistry_samples_per_second": 14.374,
+      "eval_chemistry_steps_per_second": 3.593,
+      "step": 1100
+    },
+    {
+      "epoch": 1.76,
+      "eval_physics_entropy": 0.5805578632354736,
+      "eval_physics_loss": 0.6039645671844482,
+      "eval_physics_mean_token_accuracy": 0.8294073853492737,
+      "eval_physics_num_tokens": 36654797.0,
+      "eval_physics_runtime": 41.4109,
+      "eval_physics_samples_per_second": 12.074,
+      "eval_physics_steps_per_second": 3.019,
+      "step": 1100
+    },
+    {
+      "entropy": 0.6007018189877271,
+      "epoch": 1.776,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.975777777777778e-05,
+      "loss": 0.5839,
+      "mean_token_accuracy": 0.8299636762589216,
+      "num_tokens": 36989310.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.5859053635969758,
+      "epoch": 1.792,
+      "grad_norm": 1.3828125,
+      "learning_rate": 1.9735555555555556e-05,
+      "loss": 0.5609,
+      "mean_token_accuracy": 0.8349329460412264,
+      "num_tokens": 37318535.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.5717526819556952,
+      "epoch": 1.808,
+      "grad_norm": 1.2109375,
+      "learning_rate": 1.9713333333333337e-05,
+      "loss": 0.5516,
+      "mean_token_accuracy": 0.837704149633646,
+      "num_tokens": 37658889.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.5855952553451061,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 1.25,
+      "learning_rate": 1.969111111111111e-05,
+      "loss": 0.5609,
+      "mean_token_accuracy": 0.8358533851802349,
+      "num_tokens": 37995198.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.576733535528183,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 1.2421875,
+      "learning_rate": 1.9668888888888892e-05,
+      "loss": 0.5602,
+      "mean_token_accuracy": 0.8364770855754614,
+      "num_tokens": 38325542.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.5805189371109009,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.9646666666666666e-05,
+      "loss": 0.5562,
+      "mean_token_accuracy": 0.8367651741951704,
+      "num_tokens": 38668148.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.5755893625319004,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.9624444444444447e-05,
+      "loss": 0.5567,
+      "mean_token_accuracy": 0.8351109046489,
+      "num_tokens": 39001652.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.593781272508204,
+      "epoch": 1.888,
+      "grad_norm": 1.375,
+      "learning_rate": 1.9602222222222225e-05,
+      "loss": 0.5698,
+      "mean_token_accuracy": 0.8326846692711115,
+      "num_tokens": 39333474.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.5997770514339209,
+      "epoch": 1.904,
+      "grad_norm": 1.25,
+      "learning_rate": 1.9580000000000002e-05,
+      "loss": 0.5793,
+      "mean_token_accuracy": 0.8314935185015202,
+      "num_tokens": 39663055.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.5944939412176609,
+      "epoch": 1.92,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.955777777777778e-05,
+      "loss": 0.5733,
+      "mean_token_accuracy": 0.8316057369112968,
+      "num_tokens": 39989824.0,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_chemistry_entropy": 0.6759118571281433,
+      "eval_chemistry_loss": 0.6883504986763,
+      "eval_chemistry_mean_token_accuracy": 0.8099100780487061,
+      "eval_chemistry_num_tokens": 39989824.0,
+      "eval_chemistry_runtime": 35.0777,
+      "eval_chemistry_samples_per_second": 14.254,
+      "eval_chemistry_steps_per_second": 3.564,
+      "step": 1200
+    },
+    {
+      "epoch": 1.92,
+      "eval_physics_entropy": 0.5913804955482483,
+      "eval_physics_loss": 0.5959852337837219,
+      "eval_physics_mean_token_accuracy": 0.8307065281867981,
+      "eval_physics_num_tokens": 39989824.0,
+      "eval_physics_runtime": 41.4833,
+      "eval_physics_samples_per_second": 12.053,
+      "eval_physics_steps_per_second": 3.013,
+      "step": 1200
+    },
+    {
+      "entropy": 0.5832644954323769,
+      "epoch": 1.936,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.9535555555555557e-05,
+      "loss": 0.5641,
+      "mean_token_accuracy": 0.8353390946984292,
+      "num_tokens": 40325070.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.5566705971956253,
+      "epoch": 1.952,
+      "grad_norm": 1.28125,
+      "learning_rate": 1.9513333333333335e-05,
+      "loss": 0.5374,
+      "mean_token_accuracy": 0.8413328900933266,
+      "num_tokens": 40659614.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.5830600494518876,
+      "epoch": 1.968,
+      "grad_norm": 1.15625,
+      "learning_rate": 1.9491111111111113e-05,
+      "loss": 0.5625,
+      "mean_token_accuracy": 0.835172512754798,
+      "num_tokens": 40991508.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.581772717833519,
+      "epoch": 1.984,
+      "grad_norm": 1.25,
+      "learning_rate": 1.946888888888889e-05,
+      "loss": 0.5629,
+      "mean_token_accuracy": 0.8342908933758736,
+      "num_tokens": 41328779.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.5848113460466265,
+      "epoch": 2.0,
+      "grad_norm": 1.171875,
+      "learning_rate": 1.9446666666666668e-05,
+      "loss": 0.5611,
+      "mean_token_accuracy": 0.8350322283804417,
+      "num_tokens": 41664296.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.48217232096940277,
+      "epoch": 2.016,
+      "grad_norm": 1.375,
+      "learning_rate": 1.9424444444444446e-05,
+      "loss": 0.4598,
+      "mean_token_accuracy": 0.8575068596750498,
+      "num_tokens": 42001193.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.47964414674788713,
+      "epoch": 2.032,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.9402222222222223e-05,
+      "loss": 0.4587,
+      "mean_token_accuracy": 0.8569075904786587,
+      "num_tokens": 42340757.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.49382132217288016,
+      "epoch": 2.048,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.938e-05,
+      "loss": 0.4713,
+      "mean_token_accuracy": 0.8529856160283089,
+      "num_tokens": 42670324.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.47823069952428343,
+      "epoch": 2.064,
+      "grad_norm": 1.3984375,
+      "learning_rate": 1.935777777777778e-05,
+      "loss": 0.4566,
+      "mean_token_accuracy": 0.8581176064908504,
+      "num_tokens": 43010231.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.48923970870673655,
+      "epoch": 2.08,
+      "grad_norm": 1.3828125,
+      "learning_rate": 1.9335555555555556e-05,
+      "loss": 0.4667,
+      "mean_token_accuracy": 0.8556209966540337,
+      "num_tokens": 43340227.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_chemistry_entropy": 0.546965966463089,
+      "eval_chemistry_loss": 0.7304518818855286,
+      "eval_chemistry_mean_token_accuracy": 0.8063767719268798,
+      "eval_chemistry_num_tokens": 43340227.0,
+      "eval_chemistry_runtime": 34.7458,
+      "eval_chemistry_samples_per_second": 14.39,
+      "eval_chemistry_steps_per_second": 3.598,
+      "step": 1300
+    },
+    {
+      "epoch": 2.08,
+      "eval_physics_entropy": 0.49418199634552,
+      "eval_physics_loss": 0.6146922707557678,
+      "eval_physics_mean_token_accuracy": 0.8299191513061523,
+      "eval_physics_num_tokens": 43340227.0,
+      "eval_physics_runtime": 41.365,
+      "eval_physics_samples_per_second": 12.088,
+      "eval_physics_steps_per_second": 3.022,
+      "step": 1300
+    },
+    {
+      "entropy": 0.4770382607355714,
+      "epoch": 2.096,
+      "grad_norm": 1.3984375,
+      "learning_rate": 1.9313333333333334e-05,
+      "loss": 0.461,
+      "mean_token_accuracy": 0.8564476538449526,
+      "num_tokens": 43673099.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.4707287013530731,
+      "epoch": 2.112,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.9291111111111115e-05,
+      "loss": 0.4509,
+      "mean_token_accuracy": 0.8583695068955421,
+      "num_tokens": 44005986.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.4803559150546789,
+      "epoch": 2.128,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.926888888888889e-05,
+      "loss": 0.4643,
+      "mean_token_accuracy": 0.8548851139843464,
+      "num_tokens": 44321465.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.48221837151795627,
+      "epoch": 2.144,
+      "grad_norm": 1.375,
+      "learning_rate": 1.924666666666667e-05,
+      "loss": 0.4619,
+      "mean_token_accuracy": 0.8559276178479195,
+      "num_tokens": 44661027.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.5006665755994618,
+      "epoch": 2.16,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.9224444444444444e-05,
+      "loss": 0.4817,
+      "mean_token_accuracy": 0.8508365921676159,
+      "num_tokens": 44994236.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.4735172586515546,
+      "epoch": 2.176,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.9202222222222225e-05,
+      "loss": 0.4583,
+      "mean_token_accuracy": 0.856869063898921,
+      "num_tokens": 45334280.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.47296210322529075,
+      "epoch": 2.192,
+      "grad_norm": 1.3984375,
+      "learning_rate": 1.918e-05,
+      "loss": 0.4593,
+      "mean_token_accuracy": 0.857900059223175,
+      "num_tokens": 45666954.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.47185815051198005,
+      "epoch": 2.208,
+      "grad_norm": 1.4609375,
+      "learning_rate": 1.915777777777778e-05,
+      "loss": 0.452,
+      "mean_token_accuracy": 0.858792957663536,
+      "num_tokens": 45999546.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.47457288652658464,
+      "epoch": 2.224,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.9135555555555555e-05,
+      "loss": 0.4591,
+      "mean_token_accuracy": 0.8570701099932194,
+      "num_tokens": 46333809.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.4764716824516654,
+      "epoch": 2.24,
+      "grad_norm": 1.3671875,
+      "learning_rate": 1.9113333333333336e-05,
+      "loss": 0.4596,
+      "mean_token_accuracy": 0.856460390239954,
+      "num_tokens": 46673328.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_chemistry_entropy": 0.5586697208881378,
+      "eval_chemistry_loss": 0.7306581735610962,
+      "eval_chemistry_mean_token_accuracy": 0.8062993106842041,
+      "eval_chemistry_num_tokens": 46673328.0,
+      "eval_chemistry_runtime": 34.6938,
+      "eval_chemistry_samples_per_second": 14.412,
+      "eval_chemistry_steps_per_second": 3.603,
+      "step": 1400
+    },
+    {
+      "epoch": 2.24,
+      "eval_physics_entropy": 0.5080966579914094,
+      "eval_physics_loss": 0.611297607421875,
+      "eval_physics_mean_token_accuracy": 0.830946072101593,
+      "eval_physics_num_tokens": 46673328.0,
+      "eval_physics_runtime": 41.3319,
+      "eval_physics_samples_per_second": 12.097,
+      "eval_physics_steps_per_second": 3.024,
+      "step": 1400
+    },
+    {
+      "entropy": 0.4922581292688847,
+      "epoch": 2.2560000000000002,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.9091111111111113e-05,
+      "loss": 0.4743,
+      "mean_token_accuracy": 0.8529286984354257,
+      "num_tokens": 47001509.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.47681235875934364,
+      "epoch": 2.2720000000000002,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.906888888888889e-05,
+      "loss": 0.4583,
+      "mean_token_accuracy": 0.8574760638177394,
+      "num_tokens": 47340142.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.4867669759318233,
+      "epoch": 2.288,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.904666666666667e-05,
+      "loss": 0.4711,
+      "mean_token_accuracy": 0.853106002509594,
+      "num_tokens": 47677550.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.4855895210057497,
+      "epoch": 2.304,
+      "grad_norm": 1.5703125,
+      "learning_rate": 1.9024444444444446e-05,
+      "loss": 0.4641,
+      "mean_token_accuracy": 0.855696403235197,
+      "num_tokens": 47993851.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.48173528388142584,
+      "epoch": 2.32,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.9002222222222224e-05,
+      "loss": 0.4624,
+      "mean_token_accuracy": 0.8548369336873293,
+      "num_tokens": 48332577.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.46310502383857965,
+      "epoch": 2.336,
+      "grad_norm": 1.3984375,
+      "learning_rate": 1.898e-05,
+      "loss": 0.4466,
+      "mean_token_accuracy": 0.8590828448534011,
+      "num_tokens": 48676623.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.46570080444216727,
+      "epoch": 2.352,
+      "grad_norm": 1.3828125,
+      "learning_rate": 1.895777777777778e-05,
+      "loss": 0.4477,
+      "mean_token_accuracy": 0.8588027618825436,
+      "num_tokens": 49007770.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.47828576155006886,
+      "epoch": 2.368,
+      "grad_norm": 1.4296875,
+      "learning_rate": 1.8935555555555556e-05,
+      "loss": 0.463,
+      "mean_token_accuracy": 0.8564012508839369,
+      "num_tokens": 49350016.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.48590457029640677,
+      "epoch": 2.384,
+      "grad_norm": 1.609375,
+      "learning_rate": 1.8913333333333334e-05,
+      "loss": 0.4677,
+      "mean_token_accuracy": 0.8538792595267296,
+      "num_tokens": 49679074.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.4809177804738283,
+      "epoch": 2.4,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.8891111111111115e-05,
+      "loss": 0.4631,
+      "mean_token_accuracy": 0.8553439110517502,
+      "num_tokens": 50020648.0,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_chemistry_entropy": 0.5487088859081268,
+      "eval_chemistry_loss": 0.7314473986625671,
+      "eval_chemistry_mean_token_accuracy": 0.8061141548156738,
+      "eval_chemistry_num_tokens": 50020648.0,
+      "eval_chemistry_runtime": 34.6612,
+      "eval_chemistry_samples_per_second": 14.425,
+      "eval_chemistry_steps_per_second": 3.606,
+      "step": 1500
+    },
+    {
+      "epoch": 2.4,
+      "eval_physics_entropy": 0.4990242130756378,
+      "eval_physics_loss": 0.6052322387695312,
+      "eval_physics_mean_token_accuracy": 0.8326480937004089,
+      "eval_physics_num_tokens": 50020648.0,
+      "eval_physics_runtime": 40.9615,
+      "eval_physics_samples_per_second": 12.207,
+      "eval_physics_steps_per_second": 3.052,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 16,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.718949646375592e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4e4123e9a29ef13c7b63be0874f4e43728584438c4c0986a2e633dd9a2d4deb
+size 6481