Upload folder using huggingface_hub

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-1000/chat_template.jinja

@@ -0,0 +1,4 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

checkpoint-1000/config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "transformers_version": "4.56.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

checkpoint-1000/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.56.1"
+}

checkpoint-1000/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba
+size 4903351912

checkpoint-1000/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b
+size 4947570872

checkpoint-1000/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6f2bef8d665529ecb588dcb90f8473fc93650581662d1d035f2a4f31757cb7a
+size 4962221464

checkpoint-1000/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29753226406161393bf76437a2737f724ebd150410d31c243e489a62e5834dc2
+size 3670322200

checkpoint-1000/model.safetensors.index.json

@@ -0,0 +1,472 @@
+{
+  "metadata": {
+    "total_parameters": 9241705984,
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71b16d66e8300731e274ad84bcd18d93d43ebc0fa9da7b962ff54c1b1e3dce87
+size 1585596266

checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:471b1c653aabe47d1dda07893496c6ec8f21353cd348bef0e1f9b8b6dd76552c
+size 14645

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1047a716c066f9f4e78c42060834e4ca319f06029bfedda77528f4fcc6970b7e
+size 1465

checkpoint-1000/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

checkpoint-1000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-1000/trainer_state.json

@@ -0,0 +1,1474 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.6,
+  "eval_steps": 100,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.9499894380569458,
+      "epoch": 0.016,
+      "grad_norm": 22.25,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 3.9966,
+      "mean_token_accuracy": 0.3689142918214202,
+      "num_tokens": 195510.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.9200712233781814,
+      "epoch": 0.032,
+      "grad_norm": 21.375,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 3.9412,
+      "mean_token_accuracy": 0.375832699611783,
+      "num_tokens": 391008.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.0196844026446343,
+      "epoch": 0.048,
+      "grad_norm": 21.0,
+      "learning_rate": 1.45e-06,
+      "loss": 3.9739,
+      "mean_token_accuracy": 0.3716651214286685,
+      "num_tokens": 590136.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.0409266516566276,
+      "epoch": 0.064,
+      "grad_norm": 18.5,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 3.8153,
+      "mean_token_accuracy": 0.37969561368227006,
+      "num_tokens": 791432.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.224095756560564,
+      "epoch": 0.08,
+      "grad_norm": 16.5,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 3.6725,
+      "mean_token_accuracy": 0.386144202388823,
+      "num_tokens": 990342.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.4430539876222612,
+      "epoch": 0.096,
+      "grad_norm": 13.875,
+      "learning_rate": 2.95e-06,
+      "loss": 3.5146,
+      "mean_token_accuracy": 0.39235076438635585,
+      "num_tokens": 1182339.0,
+      "step": 60
+    },
+    {
+      "entropy": 2.5642702370882033,
+      "epoch": 0.112,
+      "grad_norm": 10.75,
+      "learning_rate": 3.45e-06,
+      "loss": 3.251,
+      "mean_token_accuracy": 0.4070900550112128,
+      "num_tokens": 1386193.0,
+      "step": 70
+    },
+    {
+      "entropy": 2.569432505220175,
+      "epoch": 0.128,
+      "grad_norm": 7.03125,
+      "learning_rate": 3.95e-06,
+      "loss": 2.9184,
+      "mean_token_accuracy": 0.4458518948405981,
+      "num_tokens": 1583261.0,
+      "step": 80
+    },
+    {
+      "entropy": 2.533310153335333,
+      "epoch": 0.144,
+      "grad_norm": 4.4375,
+      "learning_rate": 4.450000000000001e-06,
+      "loss": 2.6751,
+      "mean_token_accuracy": 0.48293438367545605,
+      "num_tokens": 1774826.0,
+      "step": 90
+    },
+    {
+      "entropy": 2.3802088871598244,
+      "epoch": 0.16,
+      "grad_norm": 3.921875,
+      "learning_rate": 4.95e-06,
+      "loss": 2.4828,
+      "mean_token_accuracy": 0.5088552081957459,
+      "num_tokens": 1971310.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_biology_entropy": 2.1710678482055665,
+      "eval_biology_loss": 2.35992169380188,
+      "eval_biology_mean_token_accuracy": 0.5322089493274689,
+      "eval_biology_num_tokens": 1971310.0,
+      "eval_biology_runtime": 28.0843,
+      "eval_biology_samples_per_second": 17.804,
+      "eval_biology_steps_per_second": 4.451,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_chemistry_entropy": 2.305358714103699,
+      "eval_chemistry_loss": 2.695998430252075,
+      "eval_chemistry_mean_token_accuracy": 0.4696348729133606,
+      "eval_chemistry_num_tokens": 1971310.0,
+      "eval_chemistry_runtime": 35.0449,
+      "eval_chemistry_samples_per_second": 14.267,
+      "eval_chemistry_steps_per_second": 3.567,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_math_entropy": 2.620567533493042,
+      "eval_math_loss": 3.4614081382751465,
+      "eval_math_mean_token_accuracy": 0.3621794567108154,
+      "eval_math_num_tokens": 1971310.0,
+      "eval_math_runtime": 36.3198,
+      "eval_math_samples_per_second": 13.767,
+      "eval_math_steps_per_second": 3.442,
+      "step": 100
+    },
+    {
+      "epoch": 0.16,
+      "eval_physics_entropy": 2.547015338897705,
+      "eval_physics_loss": 3.113776206970215,
+      "eval_physics_mean_token_accuracy": 0.4125462465286255,
+      "eval_physics_num_tokens": 1971310.0,
+      "eval_physics_runtime": 42.232,
+      "eval_physics_samples_per_second": 11.839,
+      "eval_physics_steps_per_second": 2.96,
+      "step": 100
+    },
+    {
+      "entropy": 2.141241306066513,
+      "epoch": 0.176,
+      "grad_norm": 3.125,
+      "learning_rate": 5.450000000000001e-06,
+      "loss": 2.2605,
+      "mean_token_accuracy": 0.5384922448545695,
+      "num_tokens": 2169453.0,
+      "step": 110
+    },
+    {
+      "entropy": 2.050959923118353,
+      "epoch": 0.192,
+      "grad_norm": 3.296875,
+      "learning_rate": 5.950000000000001e-06,
+      "loss": 2.1612,
+      "mean_token_accuracy": 0.5548713574185967,
+      "num_tokens": 2366844.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.9513587825000287,
+      "epoch": 0.208,
+      "grad_norm": 3.59375,
+      "learning_rate": 6.450000000000001e-06,
+      "loss": 2.0818,
+      "mean_token_accuracy": 0.5668945079669356,
+      "num_tokens": 2559708.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.8972237318754197,
+      "epoch": 0.224,
+      "grad_norm": 3.09375,
+      "learning_rate": 6.95e-06,
+      "loss": 2.0295,
+      "mean_token_accuracy": 0.5779056932777167,
+      "num_tokens": 2756533.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.8532441429793836,
+      "epoch": 0.24,
+      "grad_norm": 4.0625,
+      "learning_rate": 7.450000000000001e-06,
+      "loss": 1.9709,
+      "mean_token_accuracy": 0.5822369705885648,
+      "num_tokens": 2948600.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.8167275488376617,
+      "epoch": 0.256,
+      "grad_norm": 3.140625,
+      "learning_rate": 7.950000000000002e-06,
+      "loss": 1.9309,
+      "mean_token_accuracy": 0.5902365429326892,
+      "num_tokens": 3141473.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.7767892055213452,
+      "epoch": 0.272,
+      "grad_norm": 2.875,
+      "learning_rate": 8.45e-06,
+      "loss": 1.8895,
+      "mean_token_accuracy": 0.5936309220269322,
+      "num_tokens": 3337285.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.7245751440525054,
+      "epoch": 0.288,
+      "grad_norm": 2.921875,
+      "learning_rate": 8.95e-06,
+      "loss": 1.8348,
+      "mean_token_accuracy": 0.6012940958142281,
+      "num_tokens": 3541221.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.7262215681374073,
+      "epoch": 0.304,
+      "grad_norm": 2.953125,
+      "learning_rate": 9.450000000000001e-06,
+      "loss": 1.8331,
+      "mean_token_accuracy": 0.6056978585198521,
+      "num_tokens": 3734952.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.7005573883652687,
+      "epoch": 0.32,
+      "grad_norm": 2.953125,
+      "learning_rate": 9.950000000000001e-06,
+      "loss": 1.8251,
+      "mean_token_accuracy": 0.6046949004754424,
+      "num_tokens": 3922209.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_biology_entropy": 1.6755634479522705,
+      "eval_biology_loss": 1.8113824129104614,
+      "eval_biology_mean_token_accuracy": 0.6135084123611451,
+      "eval_biology_num_tokens": 3922209.0,
+      "eval_biology_runtime": 28.1974,
+      "eval_biology_samples_per_second": 17.732,
+      "eval_biology_steps_per_second": 4.433,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_chemistry_entropy": 1.7215053329467773,
+      "eval_chemistry_loss": 1.8611716032028198,
+      "eval_chemistry_mean_token_accuracy": 0.5912830307483673,
+      "eval_chemistry_num_tokens": 3922209.0,
+      "eval_chemistry_runtime": 35.236,
+      "eval_chemistry_samples_per_second": 14.19,
+      "eval_chemistry_steps_per_second": 3.548,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_math_entropy": 1.995877082824707,
+      "eval_math_loss": 2.448066234588623,
+      "eval_math_mean_token_accuracy": 0.4923641927242279,
+      "eval_math_num_tokens": 3922209.0,
+      "eval_math_runtime": 36.5609,
+      "eval_math_samples_per_second": 13.676,
+      "eval_math_steps_per_second": 3.419,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "eval_physics_entropy": 1.9196847095489502,
+      "eval_physics_loss": 2.0909786224365234,
+      "eval_physics_mean_token_accuracy": 0.5556575627326965,
+      "eval_physics_num_tokens": 3922209.0,
+      "eval_physics_runtime": 41.9131,
+      "eval_physics_samples_per_second": 11.929,
+      "eval_physics_steps_per_second": 2.982,
+      "step": 200
+    },
+    {
+      "entropy": 1.662066788226366,
+      "epoch": 0.336,
+      "grad_norm": 2.890625,
+      "learning_rate": 1.045e-05,
+      "loss": 1.7669,
+      "mean_token_accuracy": 0.6151186719536781,
+      "num_tokens": 4115941.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.6733620196580887,
+      "epoch": 0.352,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.095e-05,
+      "loss": 1.7615,
+      "mean_token_accuracy": 0.613368246331811,
+      "num_tokens": 4308755.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.6339363023638724,
+      "epoch": 0.368,
+      "grad_norm": 3.140625,
+      "learning_rate": 1.145e-05,
+      "loss": 1.7387,
+      "mean_token_accuracy": 0.6185245186090469,
+      "num_tokens": 4505910.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.6267809972167016,
+      "epoch": 0.384,
+      "grad_norm": 3.171875,
+      "learning_rate": 1.195e-05,
+      "loss": 1.7257,
+      "mean_token_accuracy": 0.6196356352418662,
+      "num_tokens": 4696101.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.622489518672228,
+      "epoch": 0.4,
+      "grad_norm": 2.9375,
+      "learning_rate": 1.2450000000000003e-05,
+      "loss": 1.7156,
+      "mean_token_accuracy": 0.6213478941470385,
+      "num_tokens": 4889445.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.6228756994009017,
+      "epoch": 0.416,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.295e-05,
+      "loss": 1.7014,
+      "mean_token_accuracy": 0.6190611518919468,
+      "num_tokens": 5087624.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.6013640090823174,
+      "epoch": 0.432,
+      "grad_norm": 2.84375,
+      "learning_rate": 1.3450000000000002e-05,
+      "loss": 1.6973,
+      "mean_token_accuracy": 0.6241686562076211,
+      "num_tokens": 5273794.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.6023104600608349,
+      "epoch": 0.448,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.3950000000000002e-05,
+      "loss": 1.69,
+      "mean_token_accuracy": 0.6232402671128512,
+      "num_tokens": 5463288.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.5534578684717417,
+      "epoch": 0.464,
+      "grad_norm": 2.9375,
+      "learning_rate": 1.4450000000000002e-05,
+      "loss": 1.6338,
+      "mean_token_accuracy": 0.6332717493176461,
+      "num_tokens": 5656827.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.566382633894682,
+      "epoch": 0.48,
+      "grad_norm": 2.890625,
+      "learning_rate": 1.4950000000000003e-05,
+      "loss": 1.6471,
+      "mean_token_accuracy": 0.6299200173467397,
+      "num_tokens": 5853311.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_biology_entropy": 1.5272024364471435,
+      "eval_biology_loss": 1.6407932043075562,
+      "eval_biology_mean_token_accuracy": 0.6381200566291809,
+      "eval_biology_num_tokens": 5853311.0,
+      "eval_biology_runtime": 28.0119,
+      "eval_biology_samples_per_second": 17.85,
+      "eval_biology_steps_per_second": 4.462,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_chemistry_entropy": 1.5126174573898314,
+      "eval_chemistry_loss": 1.6662737131118774,
+      "eval_chemistry_mean_token_accuracy": 0.6235733127593994,
+      "eval_chemistry_num_tokens": 5853311.0,
+      "eval_chemistry_runtime": 35.1015,
+      "eval_chemistry_samples_per_second": 14.244,
+      "eval_chemistry_steps_per_second": 3.561,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_math_entropy": 1.7227830190658568,
+      "eval_math_loss": 2.2950356006622314,
+      "eval_math_mean_token_accuracy": 0.5230455741882324,
+      "eval_math_num_tokens": 5853311.0,
+      "eval_math_runtime": 36.2511,
+      "eval_math_samples_per_second": 13.793,
+      "eval_math_steps_per_second": 3.448,
+      "step": 300
+    },
+    {
+      "epoch": 0.48,
+      "eval_physics_entropy": 1.65383327293396,
+      "eval_physics_loss": 1.893297553062439,
+      "eval_physics_mean_token_accuracy": 0.588534206867218,
+      "eval_physics_num_tokens": 5853311.0,
+      "eval_physics_runtime": 42.0108,
+      "eval_physics_samples_per_second": 11.902,
+      "eval_physics_steps_per_second": 2.975,
+      "step": 300
+    },
+    {
+      "entropy": 1.4840284191071986,
+      "epoch": 0.496,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.545e-05,
+      "loss": 1.561,
+      "mean_token_accuracy": 0.6421158481389284,
+      "num_tokens": 6049788.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.544169555604458,
+      "epoch": 0.512,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.595e-05,
+      "loss": 1.6222,
+      "mean_token_accuracy": 0.6340210042893887,
+      "num_tokens": 6243825.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.5306921921670438,
+      "epoch": 0.528,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.645e-05,
+      "loss": 1.616,
+      "mean_token_accuracy": 0.6351876076310873,
+      "num_tokens": 6434188.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.5235325694084167,
+      "epoch": 0.544,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.6950000000000002e-05,
+      "loss": 1.6135,
+      "mean_token_accuracy": 0.6401191558688879,
+      "num_tokens": 6629832.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.5048270113766193,
+      "epoch": 0.56,
+      "grad_norm": 3.0,
+      "learning_rate": 1.7450000000000004e-05,
+      "loss": 1.5784,
+      "mean_token_accuracy": 0.6402778703719377,
+      "num_tokens": 6824668.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.4832347758114337,
+      "epoch": 0.576,
+      "grad_norm": 2.609375,
+      "learning_rate": 1.795e-05,
+      "loss": 1.5572,
+      "mean_token_accuracy": 0.6423058025538921,
+      "num_tokens": 7025923.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.4958114534616471,
+      "epoch": 0.592,
+      "grad_norm": 2.84375,
+      "learning_rate": 1.845e-05,
+      "loss": 1.5684,
+      "mean_token_accuracy": 0.6418677851557731,
+      "num_tokens": 7218224.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.4769254855811595,
+      "epoch": 0.608,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.8950000000000003e-05,
+      "loss": 1.5552,
+      "mean_token_accuracy": 0.6464388765394687,
+      "num_tokens": 7421216.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.4757872223854065,
+      "epoch": 0.624,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.9450000000000002e-05,
+      "loss": 1.5499,
+      "mean_token_accuracy": 0.6437039919197559,
+      "num_tokens": 7617258.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.4949540883302688,
+      "epoch": 0.64,
+      "grad_norm": 2.5,
+      "learning_rate": 1.9950000000000004e-05,
+      "loss": 1.5622,
+      "mean_token_accuracy": 0.6439059570431709,
+      "num_tokens": 7806173.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_biology_entropy": 1.4424342699050903,
+      "eval_biology_loss": 1.5438121557235718,
+      "eval_biology_mean_token_accuracy": 0.6533656511306762,
+      "eval_biology_num_tokens": 7806173.0,
+      "eval_biology_runtime": 28.2819,
+      "eval_biology_samples_per_second": 17.679,
+      "eval_biology_steps_per_second": 4.42,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_chemistry_entropy": 1.4317359008789063,
+      "eval_chemistry_loss": 1.545621395111084,
+      "eval_chemistry_mean_token_accuracy": 0.6464814004898072,
+      "eval_chemistry_num_tokens": 7806173.0,
+      "eval_chemistry_runtime": 35.2746,
+      "eval_chemistry_samples_per_second": 14.174,
+      "eval_chemistry_steps_per_second": 3.544,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_math_entropy": 1.6873997888565064,
+      "eval_math_loss": 2.1470043659210205,
+      "eval_math_mean_token_accuracy": 0.545379681110382,
+      "eval_math_num_tokens": 7806173.0,
+      "eval_math_runtime": 36.2473,
+      "eval_math_samples_per_second": 13.794,
+      "eval_math_steps_per_second": 3.449,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "eval_physics_entropy": 1.5928139057159423,
+      "eval_physics_loss": 1.7498213052749634,
+      "eval_physics_mean_token_accuracy": 0.6124887256622314,
+      "eval_physics_num_tokens": 7806173.0,
+      "eval_physics_runtime": 41.9016,
+      "eval_physics_samples_per_second": 11.933,
+      "eval_physics_steps_per_second": 2.983,
+      "step": 400
+    },
+    {
+      "entropy": 1.4704890355467797,
+      "epoch": 0.656,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.9950000000000004e-05,
+      "loss": 1.5465,
+      "mean_token_accuracy": 0.6461705386638641,
+      "num_tokens": 8000514.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.4818877466022968,
+      "epoch": 0.672,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.9894444444444446e-05,
+      "loss": 1.5466,
+      "mean_token_accuracy": 0.6474924240261316,
+      "num_tokens": 8187884.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.4635654278099537,
+      "epoch": 0.688,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.983888888888889e-05,
+      "loss": 1.5334,
+      "mean_token_accuracy": 0.6477241545915604,
+      "num_tokens": 8390643.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.4518470901995897,
+      "epoch": 0.704,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.9783333333333334e-05,
+      "loss": 1.5175,
+      "mean_token_accuracy": 0.6504514362663031,
+      "num_tokens": 8583266.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.403906264901161,
+      "epoch": 0.72,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.972777777777778e-05,
+      "loss": 1.4567,
+      "mean_token_accuracy": 0.6599624175578356,
+      "num_tokens": 8786233.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.4069542281329632,
+      "epoch": 0.736,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.9672222222222222e-05,
+      "loss": 1.4717,
+      "mean_token_accuracy": 0.6566756889224052,
+      "num_tokens": 8982898.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.438444308191538,
+      "epoch": 0.752,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.9616666666666668e-05,
+      "loss": 1.4996,
+      "mean_token_accuracy": 0.6532172866165638,
+      "num_tokens": 9174361.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.3892866168171167,
+      "epoch": 0.768,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.9561111111111114e-05,
+      "loss": 1.4559,
+      "mean_token_accuracy": 0.6592425998300314,
+      "num_tokens": 9373430.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.4066325765103103,
+      "epoch": 0.784,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.9505555555555556e-05,
+      "loss": 1.4672,
+      "mean_token_accuracy": 0.6569872628897429,
+      "num_tokens": 9570409.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.4496744759380817,
+      "epoch": 0.8,
+      "grad_norm": 2.46875,
+      "learning_rate": 1.9450000000000002e-05,
+      "loss": 1.5063,
+      "mean_token_accuracy": 0.6501368753612041,
+      "num_tokens": 9766360.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_biology_entropy": 1.3713347806930543,
+      "eval_biology_loss": 1.483181118965149,
+      "eval_biology_mean_token_accuracy": 0.6642049913406372,
+      "eval_biology_num_tokens": 9766360.0,
+      "eval_biology_runtime": 28.2197,
+      "eval_biology_samples_per_second": 17.718,
+      "eval_biology_steps_per_second": 4.43,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_chemistry_entropy": 1.3135048861503602,
+      "eval_chemistry_loss": 1.4522099494934082,
+      "eval_chemistry_mean_token_accuracy": 0.6647601218223572,
+      "eval_chemistry_num_tokens": 9766360.0,
+      "eval_chemistry_runtime": 35.0742,
+      "eval_chemistry_samples_per_second": 14.255,
+      "eval_chemistry_steps_per_second": 3.564,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_math_entropy": 1.577612156867981,
+      "eval_math_loss": 2.0336549282073975,
+      "eval_math_mean_token_accuracy": 0.5679818079471588,
+      "eval_math_num_tokens": 9766360.0,
+      "eval_math_runtime": 36.0793,
+      "eval_math_samples_per_second": 13.858,
+      "eval_math_steps_per_second": 3.465,
+      "step": 500
+    },
+    {
+      "epoch": 0.8,
+      "eval_physics_entropy": 1.4590512275695802,
+      "eval_physics_loss": 1.621680736541748,
+      "eval_physics_mean_token_accuracy": 0.636782910823822,
+      "eval_physics_num_tokens": 9766360.0,
+      "eval_physics_runtime": 41.6804,
+      "eval_physics_samples_per_second": 11.996,
+      "eval_physics_steps_per_second": 2.999,
+      "step": 500
+    },
+    {
+      "entropy": 1.3833812598139048,
+      "epoch": 0.816,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.9394444444444448e-05,
+      "loss": 1.4469,
+      "mean_token_accuracy": 0.6614159986376762,
+      "num_tokens": 9963834.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.3823285117745399,
+      "epoch": 0.832,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.933888888888889e-05,
+      "loss": 1.4483,
+      "mean_token_accuracy": 0.6609293859452009,
+      "num_tokens": 10160950.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.3773228887468576,
+      "epoch": 0.848,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.9283333333333332e-05,
+      "loss": 1.4364,
+      "mean_token_accuracy": 0.6627319652587176,
+      "num_tokens": 10363131.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.3971489261835814,
+      "epoch": 0.864,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.9227777777777778e-05,
+      "loss": 1.4586,
+      "mean_token_accuracy": 0.6601057507097721,
+      "num_tokens": 10558003.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.3813175525516272,
+      "epoch": 0.88,
+      "grad_norm": 2.25,
+      "learning_rate": 1.9172222222222224e-05,
+      "loss": 1.4372,
+      "mean_token_accuracy": 0.6638944070786238,
+      "num_tokens": 10754470.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.4032432597130537,
+      "epoch": 0.896,
+      "grad_norm": 2.625,
+      "learning_rate": 1.911666666666667e-05,
+      "loss": 1.4648,
+      "mean_token_accuracy": 0.6593563534319401,
+      "num_tokens": 10948943.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.3768033351749183,
+      "epoch": 0.912,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.9061111111111112e-05,
+      "loss": 1.4369,
+      "mean_token_accuracy": 0.6641619615256786,
+      "num_tokens": 11142613.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.3835541285574435,
+      "epoch": 0.928,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.9005555555555557e-05,
+      "loss": 1.4417,
+      "mean_token_accuracy": 0.6617751825600863,
+      "num_tokens": 11336779.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.3366544499993325,
+      "epoch": 0.944,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.8950000000000003e-05,
+      "loss": 1.4108,
+      "mean_token_accuracy": 0.669864046573639,
+      "num_tokens": 11536381.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.3591453194618226,
+      "epoch": 0.96,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.8894444444444446e-05,
+      "loss": 1.4175,
+      "mean_token_accuracy": 0.6659783527255059,
+      "num_tokens": 11735527.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_biology_entropy": 1.3452058339118957,
+      "eval_biology_loss": 1.4411625862121582,
+      "eval_biology_mean_token_accuracy": 0.6708439502716065,
+      "eval_biology_num_tokens": 11735527.0,
+      "eval_biology_runtime": 28.03,
+      "eval_biology_samples_per_second": 17.838,
+      "eval_biology_steps_per_second": 4.46,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_chemistry_entropy": 1.2769251098632812,
+      "eval_chemistry_loss": 1.4022718667984009,
+      "eval_chemistry_mean_token_accuracy": 0.6717345275878907,
+      "eval_chemistry_num_tokens": 11735527.0,
+      "eval_chemistry_runtime": 35.0721,
+      "eval_chemistry_samples_per_second": 14.256,
+      "eval_chemistry_steps_per_second": 3.564,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_math_entropy": 1.4816768627166748,
+      "eval_math_loss": 1.9867641925811768,
+      "eval_math_mean_token_accuracy": 0.5741805057525635,
+      "eval_math_num_tokens": 11735527.0,
+      "eval_math_runtime": 36.1019,
+      "eval_math_samples_per_second": 13.85,
+      "eval_math_steps_per_second": 3.462,
+      "step": 600
+    },
+    {
+      "epoch": 0.96,
+      "eval_physics_entropy": 1.404102375984192,
+      "eval_physics_loss": 1.5792219638824463,
+      "eval_physics_mean_token_accuracy": 0.6421908736228943,
+      "eval_physics_num_tokens": 11735527.0,
+      "eval_physics_runtime": 41.767,
+      "eval_physics_samples_per_second": 11.971,
+      "eval_physics_steps_per_second": 2.993,
+      "step": 600
+    },
+    {
+      "entropy": 1.3414951894432305,
+      "epoch": 0.976,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.883888888888889e-05,
+      "loss": 1.403,
+      "mean_token_accuracy": 0.6690400145947933,
+      "num_tokens": 11930935.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.3649388860911131,
+      "epoch": 0.992,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.8783333333333337e-05,
+      "loss": 1.4338,
+      "mean_token_accuracy": 0.6647567890584469,
+      "num_tokens": 12129585.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.3499837897717952,
+      "epoch": 1.008,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.872777777777778e-05,
+      "loss": 1.3983,
+      "mean_token_accuracy": 0.6714938268065452,
+      "num_tokens": 12325847.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.2676415912806989,
+      "epoch": 1.024,
+      "grad_norm": 2.125,
+      "learning_rate": 1.8672222222222222e-05,
+      "loss": 1.3321,
+      "mean_token_accuracy": 0.6808505415916443,
+      "num_tokens": 12530820.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.3242602664977312,
+      "epoch": 1.04,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.8616666666666667e-05,
+      "loss": 1.3811,
+      "mean_token_accuracy": 0.6721805844455957,
+      "num_tokens": 12725384.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.2782592210918664,
+      "epoch": 1.056,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.8561111111111113e-05,
+      "loss": 1.3308,
+      "mean_token_accuracy": 0.6812267541885376,
+      "num_tokens": 12924592.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.303604417666793,
+      "epoch": 1.072,
+      "grad_norm": 2.375,
+      "learning_rate": 1.8505555555555556e-05,
+      "loss": 1.3657,
+      "mean_token_accuracy": 0.6762988712638617,
+      "num_tokens": 13112848.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.2865918826311826,
+      "epoch": 1.088,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.845e-05,
+      "loss": 1.3547,
+      "mean_token_accuracy": 0.6780601013451815,
+      "num_tokens": 13305725.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.2945185869932174,
+      "epoch": 1.104,
+      "grad_norm": 2.59375,
+      "learning_rate": 1.8394444444444447e-05,
+      "loss": 1.3552,
+      "mean_token_accuracy": 0.6792182371020317,
+      "num_tokens": 13498685.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.2905282031744718,
+      "epoch": 1.12,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.833888888888889e-05,
+      "loss": 1.3539,
+      "mean_token_accuracy": 0.6796708777546883,
+      "num_tokens": 13682146.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_biology_entropy": 1.2542049832344055,
+      "eval_biology_loss": 1.4174002408981323,
+      "eval_biology_mean_token_accuracy": 0.6758444757461548,
+      "eval_biology_num_tokens": 13682146.0,
+      "eval_biology_runtime": 27.9034,
+      "eval_biology_samples_per_second": 17.919,
+      "eval_biology_steps_per_second": 4.48,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_chemistry_entropy": 1.17753240442276,
+      "eval_chemistry_loss": 1.3736826181411743,
+      "eval_chemistry_mean_token_accuracy": 0.6796552734375,
+      "eval_chemistry_num_tokens": 13682146.0,
+      "eval_chemistry_runtime": 35.3534,
+      "eval_chemistry_samples_per_second": 14.143,
+      "eval_chemistry_steps_per_second": 3.536,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_math_entropy": 1.3878496103286744,
+      "eval_math_loss": 1.964780330657959,
+      "eval_math_mean_token_accuracy": 0.5833453125953675,
+      "eval_math_num_tokens": 13682146.0,
+      "eval_math_runtime": 36.2064,
+      "eval_math_samples_per_second": 13.81,
+      "eval_math_steps_per_second": 3.452,
+      "step": 700
+    },
+    {
+      "epoch": 1.12,
+      "eval_physics_entropy": 1.3059893226623536,
+      "eval_physics_loss": 1.5306388139724731,
+      "eval_physics_mean_token_accuracy": 0.6527995052337646,
+      "eval_physics_num_tokens": 13682146.0,
+      "eval_physics_runtime": 41.914,
+      "eval_physics_samples_per_second": 11.929,
+      "eval_physics_steps_per_second": 2.982,
+      "step": 700
+    },
+    {
+      "entropy": 1.2783031422644853,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.8283333333333335e-05,
+      "loss": 1.3431,
+      "mean_token_accuracy": 0.680135428532958,
+      "num_tokens": 13876656.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.2606192819774151,
+      "epoch": 1.152,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.822777777777778e-05,
+      "loss": 1.3107,
+      "mean_token_accuracy": 0.6832978628575802,
+      "num_tokens": 14086104.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.274829576537013,
+      "epoch": 1.168,
+      "grad_norm": 1.9375,
+      "learning_rate": 1.8172222222222223e-05,
+      "loss": 1.337,
+      "mean_token_accuracy": 0.6811702467501164,
+      "num_tokens": 14274520.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.2760208252817393,
+      "epoch": 1.184,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.811666666666667e-05,
+      "loss": 1.3397,
+      "mean_token_accuracy": 0.6811570346355438,
+      "num_tokens": 14473623.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.2696042828261853,
+      "epoch": 1.2,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.806111111111111e-05,
+      "loss": 1.3374,
+      "mean_token_accuracy": 0.6804133646190167,
+      "num_tokens": 14661238.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.234127951413393,
+      "epoch": 1.216,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.8005555555555557e-05,
+      "loss": 1.2876,
+      "mean_token_accuracy": 0.6880679227411747,
+      "num_tokens": 14865915.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.2614865679293872,
+      "epoch": 1.232,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.795e-05,
+      "loss": 1.3258,
+      "mean_token_accuracy": 0.6855961695313454,
+      "num_tokens": 15055682.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.264949632063508,
+      "epoch": 1.248,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.7894444444444445e-05,
+      "loss": 1.3316,
+      "mean_token_accuracy": 0.6824632346630096,
+      "num_tokens": 15249576.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.2795803047716618,
+      "epoch": 1.264,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.783888888888889e-05,
+      "loss": 1.3318,
+      "mean_token_accuracy": 0.6813411716371774,
+      "num_tokens": 15446149.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.304668091982603,
+      "epoch": 1.28,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.7783333333333333e-05,
+      "loss": 1.3676,
+      "mean_token_accuracy": 0.6758647799491883,
+      "num_tokens": 15639317.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_biology_entropy": 1.2498101744651795,
+      "eval_biology_loss": 1.400923490524292,
+      "eval_biology_mean_token_accuracy": 0.6780072612762451,
+      "eval_biology_num_tokens": 15639317.0,
+      "eval_biology_runtime": 27.8965,
+      "eval_biology_samples_per_second": 17.923,
+      "eval_biology_steps_per_second": 4.481,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_chemistry_entropy": 1.1619028897285462,
+      "eval_chemistry_loss": 1.3499823808670044,
+      "eval_chemistry_mean_token_accuracy": 0.6855192203521728,
+      "eval_chemistry_num_tokens": 15639317.0,
+      "eval_chemistry_runtime": 35.4788,
+      "eval_chemistry_samples_per_second": 14.093,
+      "eval_chemistry_steps_per_second": 3.523,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_math_entropy": 1.342226463317871,
+      "eval_math_loss": 1.9161298274993896,
+      "eval_math_mean_token_accuracy": 0.5939471673965454,
+      "eval_math_num_tokens": 15639317.0,
+      "eval_math_runtime": 36.1236,
+      "eval_math_samples_per_second": 13.841,
+      "eval_math_steps_per_second": 3.46,
+      "step": 800
+    },
+    {
+      "epoch": 1.28,
+      "eval_physics_entropy": 1.2730433497428895,
+      "eval_physics_loss": 1.5102938413619995,
+      "eval_physics_mean_token_accuracy": 0.6596844635009765,
+      "eval_physics_num_tokens": 15639317.0,
+      "eval_physics_runtime": 42.1115,
+      "eval_physics_samples_per_second": 11.873,
+      "eval_physics_steps_per_second": 2.968,
+      "step": 800
+    },
+    {
+      "entropy": 1.2820089340209961,
+      "epoch": 1.296,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.772777777777778e-05,
+      "loss": 1.3444,
+      "mean_token_accuracy": 0.679485822096467,
+      "num_tokens": 15835805.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.2858267422765493,
+      "epoch": 1.312,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.7672222222222224e-05,
+      "loss": 1.3403,
+      "mean_token_accuracy": 0.6791896495968104,
+      "num_tokens": 16028375.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.2837634429335594,
+      "epoch": 1.328,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.761666666666667e-05,
+      "loss": 1.3353,
+      "mean_token_accuracy": 0.6799331065267324,
+      "num_tokens": 16230555.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.2417853761464357,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.7561111111111113e-05,
+      "loss": 1.3019,
+      "mean_token_accuracy": 0.6887874841690064,
+      "num_tokens": 16436297.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.2758412674069404,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.7505555555555558e-05,
+      "loss": 1.3256,
+      "mean_token_accuracy": 0.6834286849945783,
+      "num_tokens": 16630563.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.3012595981359483,
+      "epoch": 1.376,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.7450000000000004e-05,
+      "loss": 1.3584,
+      "mean_token_accuracy": 0.6760492861270905,
+      "num_tokens": 16822602.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.2583775073289871,
+      "epoch": 1.392,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.7394444444444446e-05,
+      "loss": 1.3322,
+      "mean_token_accuracy": 0.683934273943305,
+      "num_tokens": 17015750.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.264936549961567,
+      "epoch": 1.408,
+      "grad_norm": 1.8203125,
+      "learning_rate": 1.733888888888889e-05,
+      "loss": 1.3218,
+      "mean_token_accuracy": 0.6842875212430954,
+      "num_tokens": 17207248.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.2753260061144829,
+      "epoch": 1.424,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.7283333333333334e-05,
+      "loss": 1.3404,
+      "mean_token_accuracy": 0.6819597784429788,
+      "num_tokens": 17403869.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.2661269146949052,
+      "epoch": 1.44,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.722777777777778e-05,
+      "loss": 1.3228,
+      "mean_token_accuracy": 0.6832845721393823,
+      "num_tokens": 17597428.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_biology_entropy": 1.2468014574050903,
+      "eval_biology_loss": 1.3835524320602417,
+      "eval_biology_mean_token_accuracy": 0.6823186283111572,
+      "eval_biology_num_tokens": 17597428.0,
+      "eval_biology_runtime": 27.872,
+      "eval_biology_samples_per_second": 17.939,
+      "eval_biology_steps_per_second": 4.485,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_chemistry_entropy": 1.1500652661323547,
+      "eval_chemistry_loss": 1.3310225009918213,
+      "eval_chemistry_mean_token_accuracy": 0.6886456093788147,
+      "eval_chemistry_num_tokens": 17597428.0,
+      "eval_chemistry_runtime": 35.3518,
+      "eval_chemistry_samples_per_second": 14.144,
+      "eval_chemistry_steps_per_second": 3.536,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_math_entropy": 1.3305657148361205,
+      "eval_math_loss": 1.9067283868789673,
+      "eval_math_mean_token_accuracy": 0.5963297083377838,
+      "eval_math_num_tokens": 17597428.0,
+      "eval_math_runtime": 36.3806,
+      "eval_math_samples_per_second": 13.744,
+      "eval_math_steps_per_second": 3.436,
+      "step": 900
+    },
+    {
+      "epoch": 1.44,
+      "eval_physics_entropy": 1.2608039512634277,
+      "eval_physics_loss": 1.4831783771514893,
+      "eval_physics_mean_token_accuracy": 0.6645136909484863,
+      "eval_physics_num_tokens": 17597428.0,
+      "eval_physics_runtime": 41.9379,
+      "eval_physics_samples_per_second": 11.922,
+      "eval_physics_steps_per_second": 2.981,
+      "step": 900
+    },
+    {
+      "entropy": 1.2557506311684847,
+      "epoch": 1.456,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.7172222222222223e-05,
+      "loss": 1.3218,
+      "mean_token_accuracy": 0.6848054088652133,
+      "num_tokens": 17798083.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.2527935065329074,
+      "epoch": 1.472,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.7116666666666668e-05,
+      "loss": 1.3039,
+      "mean_token_accuracy": 0.6864713884890079,
+      "num_tokens": 17993802.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.2510250229388475,
+      "epoch": 1.488,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.7061111111111114e-05,
+      "loss": 1.3242,
+      "mean_token_accuracy": 0.6868655398488045,
+      "num_tokens": 18185570.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.2679124280810357,
+      "epoch": 1.504,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.7005555555555556e-05,
+      "loss": 1.3282,
+      "mean_token_accuracy": 0.6849378615617752,
+      "num_tokens": 18377903.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.2595157697796822,
+      "epoch": 1.52,
+      "grad_norm": 2.0,
+      "learning_rate": 1.6950000000000002e-05,
+      "loss": 1.3258,
+      "mean_token_accuracy": 0.6842729311436415,
+      "num_tokens": 18579284.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.2647370167076588,
+      "epoch": 1.536,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.6894444444444448e-05,
+      "loss": 1.3277,
+      "mean_token_accuracy": 0.6830333441495895,
+      "num_tokens": 18765324.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2503893371671437,
+      "epoch": 1.552,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.683888888888889e-05,
+      "loss": 1.2998,
+      "mean_token_accuracy": 0.687108751386404,
+      "num_tokens": 18966382.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.2391960687935353,
+      "epoch": 1.568,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.6783333333333332e-05,
+      "loss": 1.3038,
+      "mean_token_accuracy": 0.6893775951117277,
+      "num_tokens": 19160509.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.2362499207258224,
+      "epoch": 1.584,
+      "grad_norm": 2.125,
+      "learning_rate": 1.6727777777777778e-05,
+      "loss": 1.3027,
+      "mean_token_accuracy": 0.6873069319874048,
+      "num_tokens": 19354662.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.273169829696417,
+      "epoch": 1.6,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.6672222222222224e-05,
+      "loss": 1.3345,
+      "mean_token_accuracy": 0.6805679194629193,
+      "num_tokens": 19543088.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_biology_entropy": 1.2320180339813231,
+      "eval_biology_loss": 1.3710711002349854,
+      "eval_biology_mean_token_accuracy": 0.6841626400947571,
+      "eval_biology_num_tokens": 19543088.0,
+      "eval_biology_runtime": 28.0931,
+      "eval_biology_samples_per_second": 17.798,
+      "eval_biology_steps_per_second": 4.449,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_chemistry_entropy": 1.136505418777466,
+      "eval_chemistry_loss": 1.3159189224243164,
+      "eval_chemistry_mean_token_accuracy": 0.6926775484085083,
+      "eval_chemistry_num_tokens": 19543088.0,
+      "eval_chemistry_runtime": 35.117,
+      "eval_chemistry_samples_per_second": 14.238,
+      "eval_chemistry_steps_per_second": 3.56,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_math_entropy": 1.3251317410469055,
+      "eval_math_loss": 1.88190758228302,
+      "eval_math_mean_token_accuracy": 0.5993597855567933,
+      "eval_math_num_tokens": 19543088.0,
+      "eval_math_runtime": 36.239,
+      "eval_math_samples_per_second": 13.797,
+      "eval_math_steps_per_second": 3.449,
+      "step": 1000
+    },
+    {
+      "epoch": 1.6,
+      "eval_physics_entropy": 1.2505839891433717,
+      "eval_physics_loss": 1.4557042121887207,
+      "eval_physics_mean_token_accuracy": 0.6703379545211792,
+      "eval_physics_num_tokens": 19543088.0,
+      "eval_physics_runtime": 41.6218,
+      "eval_physics_samples_per_second": 12.013,
+      "eval_physics_steps_per_second": 3.003,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 4000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 7,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.6380077645630792e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7e39aefb9ec374912b9ea0ce26b8eabb24fc21a2c815ad514484ca79232c3ea
+size 6417