Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1524/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,62 @@
+---
+base_model: Qwen/Qwen2.5-Math-7B-Instruct
+library_name: peft
+model_name: verifier_v_7b
+tags:
+- base_model:adapter:Qwen/Qwen2.5-Math-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for verifier_v_7b
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-7B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.19.1
+- TRL: 1.8.0
+- Transformers: 5.13.1
+- Pytorch: 2.11.0+cu128
+- Datasets: 5.0.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-Math-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "k_proj",
+    "q_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fd282726016ee3ee36c9fb6ddda695227ea2b032874058191cbc67b936bcdb2
+size 323014168

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'Please reason step by step, and put your final answer within \\boxed{}.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nPlease reason step by step, and put your final answer within \\boxed{}.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoint-1524/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-Math-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-Math-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoint-1524/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-Math-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "k_proj",
+    "q_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1524/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fd282726016ee3ee36c9fb6ddda695227ea2b032874058191cbc67b936bcdb2
+size 323014168

checkpoint-1524/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'Please reason step by step, and put your final answer within \\boxed{}.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nPlease reason step by step, and put your final answer within \\boxed{}.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoint-1524/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b92e2329e7cb76efa5ccc3e734ab6162f06c3431a4f9e22667de1f59f1726a3d
+size 646259323

checkpoint-1524/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58585354dbadc85a8b1e6c6518f177a900e05fae000f4456dfc7c06bd6448d64
+size 14581

checkpoint-1524/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8092633066113ce3cbe0fe41b11261d69f4d22fbe31f753e60855ed5c4f2a31
+size 1465

checkpoint-1524/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

checkpoint-1524/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|fim_pad|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-1524/trainer_state.json

@@ -0,0 +1,1895 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 50,
+  "global_step": 1524,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 4.081731240451336,
+      "epoch": 0.01968503937007874,
+      "grad_norm": 1.2025723457336426,
+      "learning_rate": 3.91304347826087e-05,
+      "loss": 2.09478702545166,
+      "mean_token_accuracy": 0.6894429370760917,
+      "num_tokens": 115439.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.5656683981418609,
+      "epoch": 0.03937007874015748,
+      "grad_norm": 0.4732680320739746,
+      "learning_rate": 8.260869565217392e-05,
+      "loss": 1.5646931648254394,
+      "mean_token_accuracy": 0.7097038365900517,
+      "num_tokens": 225264.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.0895499642938375,
+      "epoch": 0.05905511811023622,
+      "grad_norm": 0.550152599811554,
+      "learning_rate": 0.00012608695652173915,
+      "loss": 1.1402780532836914,
+      "mean_token_accuracy": 0.7643287762999534,
+      "num_tokens": 330923.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.0377423465251923,
+      "epoch": 0.07874015748031496,
+      "grad_norm": 0.4060899317264557,
+      "learning_rate": 0.00016956521739130436,
+      "loss": 1.0394248008728026,
+      "mean_token_accuracy": 0.7670801907777787,
+      "num_tokens": 437622.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.8492720592767,
+      "epoch": 0.0984251968503937,
+      "grad_norm": 0.4656495451927185,
+      "learning_rate": 0.00019999796688511753,
+      "loss": 0.9070818901062012,
+      "mean_token_accuracy": 0.7918916828930378,
+      "num_tokens": 551175.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.0984251968503937,
+      "eval_entropy": 0.9714998945041939,
+      "eval_loss": 0.9814297556877136,
+      "eval_mean_token_accuracy": 0.7616264853212569,
+      "eval_num_tokens": 551175.0,
+      "eval_runtime": 50.076,
+      "eval_samples_per_second": 8.527,
+      "eval_steps_per_second": 1.078,
+      "step": 50
+    },
+    {
+      "entropy": 0.7805005680769682,
+      "epoch": 0.11811023622047244,
+      "grad_norm": 0.6193442940711975,
+      "learning_rate": 0.0001999618249203163,
+      "loss": 0.8954662322998047,
+      "mean_token_accuracy": 0.8002645947039128,
+      "num_tokens": 658006.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.8279133141040802,
+      "epoch": 0.1377952755905512,
+      "grad_norm": 0.6020731925964355,
+      "learning_rate": 0.00019988052141977201,
+      "loss": 0.9206802368164062,
+      "mean_token_accuracy": 0.7954164117574691,
+      "num_tokens": 771278.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.7643017668277026,
+      "epoch": 0.15748031496062992,
+      "grad_norm": 0.5231983065605164,
+      "learning_rate": 0.0001997540931154166,
+      "loss": 0.7755372047424316,
+      "mean_token_accuracy": 0.8071240559220314,
+      "num_tokens": 886071.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.7395163401961327,
+      "epoch": 0.17716535433070865,
+      "grad_norm": 0.5036323666572571,
+      "learning_rate": 0.00019958259712601903,
+      "loss": 0.8172826766967773,
+      "mean_token_accuracy": 0.8099836744368076,
+      "num_tokens": 996361.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.70047264136374,
+      "epoch": 0.1968503937007874,
+      "grad_norm": 0.53205406665802,
+      "learning_rate": 0.00019936611093137983,
+      "loss": 0.764425802230835,
+      "mean_token_accuracy": 0.8133513547480107,
+      "num_tokens": 1103999.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.1968503937007874,
+      "eval_entropy": 0.8809433656710165,
+      "eval_loss": 0.8760986924171448,
+      "eval_mean_token_accuracy": 0.7759342005959263,
+      "eval_num_tokens": 1103999.0,
+      "eval_runtime": 50.0693,
+      "eval_samples_per_second": 8.528,
+      "eval_steps_per_second": 1.079,
+      "step": 100
+    },
+    {
+      "entropy": 0.7424050349742174,
+      "epoch": 0.21653543307086615,
+      "grad_norm": 0.3994530141353607,
+      "learning_rate": 0.00019910473233732668,
+      "loss": 0.7701848030090332,
+      "mean_token_accuracy": 0.8107160992920399,
+      "num_tokens": 1216586.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.7282566852867604,
+      "epoch": 0.23622047244094488,
+      "grad_norm": 0.5450337529182434,
+      "learning_rate": 0.00019879857943152674,
+      "loss": 0.7601457118988038,
+      "mean_token_accuracy": 0.8057335019111633,
+      "num_tokens": 1324767.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.7324596978724003,
+      "epoch": 0.2559055118110236,
+      "grad_norm": 0.5821658968925476,
+      "learning_rate": 0.00019844779053013637,
+      "loss": 0.7659125804901123,
+      "mean_token_accuracy": 0.8053000435233116,
+      "num_tokens": 1432356.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.66163036134094,
+      "epoch": 0.2755905511811024,
+      "grad_norm": 0.5165016055107117,
+      "learning_rate": 0.00019805252411531126,
+      "loss": 0.7274711608886719,
+      "mean_token_accuracy": 0.8206341236829757,
+      "num_tokens": 1542545.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.7055144146084785,
+      "epoch": 0.2952755905511811,
+      "grad_norm": 0.5799933075904846,
+      "learning_rate": 0.0001976129587636064,
+      "loss": 0.7352935791015625,
+      "mean_token_accuracy": 0.8095529124140739,
+      "num_tokens": 1654462.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.2952755905511811,
+      "eval_entropy": 0.8143208810576686,
+      "eval_loss": 0.8347764611244202,
+      "eval_mean_token_accuracy": 0.7817997546107681,
+      "eval_num_tokens": 1654462.0,
+      "eval_runtime": 50.0768,
+      "eval_samples_per_second": 8.527,
+      "eval_steps_per_second": 1.078,
+      "step": 150
+    },
+    {
+      "entropy": 0.6451521161943674,
+      "epoch": 0.31496062992125984,
+      "grad_norm": 0.4720793068408966,
+      "learning_rate": 0.00019712929306529723,
+      "loss": 0.7099593162536622,
+      "mean_token_accuracy": 0.822279942035675,
+      "num_tokens": 1760181.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.6923504635691643,
+      "epoch": 0.3346456692913386,
+      "grad_norm": 0.5992543697357178,
+      "learning_rate": 0.000196601745534659,
+      "loss": 0.7766928195953369,
+      "mean_token_accuracy": 0.8206209510564804,
+      "num_tokens": 1864683.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.7083404455333948,
+      "epoch": 0.3543307086614173,
+      "grad_norm": 0.45409828424453735,
+      "learning_rate": 0.0001960305545112445,
+      "loss": 0.744833517074585,
+      "mean_token_accuracy": 0.8117265947163105,
+      "num_tokens": 1978641.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.671433940064162,
+      "epoch": 0.37401574803149606,
+      "grad_norm": 0.4698701798915863,
+      "learning_rate": 0.0001954159780522056,
+      "loss": 0.7281535625457763,
+      "mean_token_accuracy": 0.817485723644495,
+      "num_tokens": 2087989.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.7019775623455644,
+      "epoch": 0.3937007874015748,
+      "grad_norm": 0.6314639449119568,
+      "learning_rate": 0.00019475829381570566,
+      "loss": 0.7663211822509766,
+      "mean_token_accuracy": 0.8176812350749969,
+      "num_tokens": 2195494.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.3937007874015748,
+      "eval_entropy": 0.819858855119458,
+      "eval_loss": 0.8197568655014038,
+      "eval_mean_token_accuracy": 0.7837593434033571,
+      "eval_num_tokens": 2195494.0,
+      "eval_runtime": 50.0794,
+      "eval_samples_per_second": 8.526,
+      "eval_steps_per_second": 1.078,
+      "step": 200
+    },
+    {
+      "entropy": 0.6459112739190459,
+      "epoch": 0.41338582677165353,
+      "grad_norm": 0.4029120206832886,
+      "learning_rate": 0.0001940577989354773,
+      "loss": 0.7212091445922851,
+      "mean_token_accuracy": 0.8216200672090054,
+      "num_tokens": 2314228.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.6658948451280594,
+      "epoch": 0.4330708661417323,
+      "grad_norm": 0.6937551498413086,
+      "learning_rate": 0.00019331480988658083,
+      "loss": 0.7317200660705566,
+      "mean_token_accuracy": 0.8218330323696137,
+      "num_tokens": 2419122.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.6822177035734057,
+      "epoch": 0.452755905511811,
+      "grad_norm": 0.4081629514694214,
+      "learning_rate": 0.00019252966234242468,
+      "loss": 0.7487519741058349,
+      "mean_token_accuracy": 0.8097363859415054,
+      "num_tokens": 2524623.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.7053613148629665,
+      "epoch": 0.47244094488188976,
+      "grad_norm": 0.6170492172241211,
+      "learning_rate": 0.00019170271102311238,
+      "loss": 0.8214272499084473,
+      "mean_token_accuracy": 0.8059085100889206,
+      "num_tokens": 2631031.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.6398238291963935,
+      "epoch": 0.4921259842519685,
+      "grad_norm": 0.4603516161441803,
+      "learning_rate": 0.00019083432953518418,
+      "loss": 0.7328793048858643,
+      "mean_token_accuracy": 0.8248285502195358,
+      "num_tokens": 2741031.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.4921259842519685,
+      "eval_entropy": 0.8082884434196684,
+      "eval_loss": 0.8002302646636963,
+      "eval_mean_token_accuracy": 0.788091140764731,
+      "eval_num_tokens": 2741031.0,
+      "eval_runtime": 50.0693,
+      "eval_samples_per_second": 8.528,
+      "eval_steps_per_second": 1.079,
+      "step": 250
+    },
+    {
+      "entropy": 0.6552930422127247,
+      "epoch": 0.5118110236220472,
+      "grad_norm": 0.5747159719467163,
+      "learning_rate": 0.00018992491020282618,
+      "loss": 0.693137788772583,
+      "mean_token_accuracy": 0.8185193344950676,
+      "num_tokens": 2853547.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.6801722910255193,
+      "epoch": 0.531496062992126,
+      "grad_norm": 0.4311263859272003,
+      "learning_rate": 0.0001889748638906231,
+      "loss": 0.7339208602905274,
+      "mean_token_accuracy": 0.8195476584136486,
+      "num_tokens": 2968367.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.6948977328836918,
+      "epoch": 0.5511811023622047,
+      "grad_norm": 0.4513797461986542,
+      "learning_rate": 0.00018798461981793462,
+      "loss": 0.7738771438598633,
+      "mean_token_accuracy": 0.8162345252931118,
+      "num_tokens": 3075963.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.7177972780540586,
+      "epoch": 0.5708661417322834,
+      "grad_norm": 0.4137361943721771,
+      "learning_rate": 0.00018695462536497934,
+      "loss": 0.7487005710601806,
+      "mean_token_accuracy": 0.8072154425084591,
+      "num_tokens": 3185899.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.6409716093912721,
+      "epoch": 0.5905511811023622,
+      "grad_norm": 0.49643731117248535,
+      "learning_rate": 0.00018588534587071385,
+      "loss": 0.7363781452178955,
+      "mean_token_accuracy": 0.8213795177638531,
+      "num_tokens": 3297319.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.5905511811023622,
+      "eval_entropy": 0.7340650547433782,
+      "eval_loss": 0.7908466458320618,
+      "eval_mean_token_accuracy": 0.7887170259599332,
+      "eval_num_tokens": 3297319.0,
+      "eval_runtime": 50.0473,
+      "eval_samples_per_second": 8.532,
+      "eval_steps_per_second": 1.079,
+      "step": 300
+    },
+    {
+      "entropy": 0.6877464625984431,
+      "epoch": 0.610236220472441,
+      "grad_norm": 0.3918077051639557,
+      "learning_rate": 0.00018477726442259806,
+      "loss": 0.720060157775879,
+      "mean_token_accuracy": 0.811838335543871,
+      "num_tokens": 3415477.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.6839619591832161,
+      "epoch": 0.6299212598425197,
+      "grad_norm": 0.39354413747787476,
+      "learning_rate": 0.00018363088163834252,
+      "loss": 0.7302655220031739,
+      "mean_token_accuracy": 0.8125466287136078,
+      "num_tokens": 3524759.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.6844633586704731,
+      "epoch": 0.6496062992125984,
+      "grad_norm": 0.44826650619506836,
+      "learning_rate": 0.00018244671543973513,
+      "loss": 0.75206618309021,
+      "mean_token_accuracy": 0.8085944786667824,
+      "num_tokens": 3640838.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.6127769330516457,
+      "epoch": 0.6692913385826772,
+      "grad_norm": 0.5281074047088623,
+      "learning_rate": 0.0001812253008186505,
+      "loss": 0.676181697845459,
+      "mean_token_accuracy": 0.8289406672120094,
+      "num_tokens": 3756735.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.6899090738967061,
+      "epoch": 0.6889763779527559,
+      "grad_norm": 0.4868250787258148,
+      "learning_rate": 0.0001799671895953471,
+      "loss": 0.7409597396850586,
+      "mean_token_accuracy": 0.8155336409807206,
+      "num_tokens": 3870377.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.6889763779527559,
+      "eval_entropy": 0.774545039291735,
+      "eval_loss": 0.7735128402709961,
+      "eval_mean_token_accuracy": 0.7915615362149698,
+      "eval_num_tokens": 3870377.0,
+      "eval_runtime": 50.0713,
+      "eval_samples_per_second": 8.528,
+      "eval_steps_per_second": 1.078,
+      "step": 350
+    },
+    {
+      "entropy": 0.628368235193193,
+      "epoch": 0.7086614173228346,
+      "grad_norm": 0.5193262100219727,
+      "learning_rate": 0.00017867295016916167,
+      "loss": 0.686598539352417,
+      "mean_token_accuracy": 0.8207510903477668,
+      "num_tokens": 3975692.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.7101229786872864,
+      "epoch": 0.7283464566929134,
+      "grad_norm": 0.4744711220264435,
+      "learning_rate": 0.0001773431672617132,
+      "loss": 0.7801414966583252,
+      "mean_token_accuracy": 0.8085352785885334,
+      "num_tokens": 4087256.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.6297556543722749,
+      "epoch": 0.7480314960629921,
+      "grad_norm": 0.45756635069847107,
+      "learning_rate": 0.00017597844165273286,
+      "loss": 0.6692267417907715,
+      "mean_token_accuracy": 0.8241402678191662,
+      "num_tokens": 4192629.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.6269465358927846,
+      "epoch": 0.7677165354330708,
+      "grad_norm": 0.48099687695503235,
+      "learning_rate": 0.00017457938990863908,
+      "loss": 0.6762315273284912,
+      "mean_token_accuracy": 0.824905026704073,
+      "num_tokens": 4303838.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.7084620589390397,
+      "epoch": 0.7874015748031497,
+      "grad_norm": 0.4862897992134094,
+      "learning_rate": 0.00017314664410398033,
+      "loss": 0.7662469863891601,
+      "mean_token_accuracy": 0.8124067313969136,
+      "num_tokens": 4414002.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.7874015748031497,
+      "eval_entropy": 0.7462573321881117,
+      "eval_loss": 0.7633256316184998,
+      "eval_mean_token_accuracy": 0.7947341071234809,
+      "eval_num_tokens": 4414002.0,
+      "eval_runtime": 50.0471,
+      "eval_samples_per_second": 8.532,
+      "eval_steps_per_second": 1.079,
+      "step": 400
+    },
+    {
+      "entropy": 0.6137412896379828,
+      "epoch": 0.8070866141732284,
+      "grad_norm": 0.5355563759803772,
+      "learning_rate": 0.0001716808515358714,
+      "loss": 0.7326591968536377,
+      "mean_token_accuracy": 0.8191115081310272,
+      "num_tokens": 4525758.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.6132487592287361,
+      "epoch": 0.8267716535433071,
+      "grad_norm": 0.47578275203704834,
+      "learning_rate": 0.00017018267443155268,
+      "loss": 0.6466782569885254,
+      "mean_token_accuracy": 0.8353692322969437,
+      "num_tokens": 4640742.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.6744782857596874,
+      "epoch": 0.8464566929133859,
+      "grad_norm": 0.4436945617198944,
+      "learning_rate": 0.00016865278964920378,
+      "loss": 0.7423620223999023,
+      "mean_token_accuracy": 0.8134209528565407,
+      "num_tokens": 4754244.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.6488625114783645,
+      "epoch": 0.8661417322834646,
+      "grad_norm": 0.33081018924713135,
+      "learning_rate": 0.00016709188837214732,
+      "loss": 0.6952498912811279,
+      "mean_token_accuracy": 0.8194661349058151,
+      "num_tokens": 4862576.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.6501848340034485,
+      "epoch": 0.8858267716535433,
+      "grad_norm": 0.3483272194862366,
+      "learning_rate": 0.0001655006757965806,
+      "loss": 0.6860886096954346,
+      "mean_token_accuracy": 0.8210023202002048,
+      "num_tokens": 4984410.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.8858267716535433,
+      "eval_entropy": 0.7518550543873398,
+      "eval_loss": 0.7559269666671753,
+      "eval_mean_token_accuracy": 0.7948771152231429,
+      "eval_num_tokens": 4984410.0,
+      "eval_runtime": 50.0772,
+      "eval_samples_per_second": 8.527,
+      "eval_steps_per_second": 1.078,
+      "step": 450
+    },
+    {
+      "entropy": 0.6636170313693583,
+      "epoch": 0.905511811023622,
+      "grad_norm": 0.5626906752586365,
+      "learning_rate": 0.0001638798708129768,
+      "loss": 0.74765944480896,
+      "mean_token_accuracy": 0.8188274890184403,
+      "num_tokens": 5094029.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.6928009213879704,
+      "epoch": 0.9251968503937008,
+      "grad_norm": 0.37302127480506897,
+      "learning_rate": 0.00016223020568129854,
+      "loss": 0.7495561599731445,
+      "mean_token_accuracy": 0.8060568884015084,
+      "num_tokens": 5210613.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.6453812881372869,
+      "epoch": 0.9448818897637795,
+      "grad_norm": 0.6153962016105652,
+      "learning_rate": 0.00016055242570017244,
+      "loss": 0.71365327835083,
+      "mean_token_accuracy": 0.8186813399195672,
+      "num_tokens": 5322546.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.6382862586528063,
+      "epoch": 0.9645669291338582,
+      "grad_norm": 0.4038618505001068,
+      "learning_rate": 0.00015884728887017162,
+      "loss": 0.7178555965423584,
+      "mean_token_accuracy": 0.8249971143901348,
+      "num_tokens": 5434008.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.5493983221240342,
+      "epoch": 0.984251968503937,
+      "grad_norm": 0.4192914664745331,
+      "learning_rate": 0.0001571155655513606,
+      "loss": 0.6016243457794189,
+      "mean_token_accuracy": 0.8419173769652843,
+      "num_tokens": 5544086.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.984251968503937,
+      "eval_entropy": 0.7105065948433347,
+      "eval_loss": 0.7486068606376648,
+      "eval_mean_token_accuracy": 0.7960716353522407,
+      "eval_num_tokens": 5544086.0,
+      "eval_runtime": 50.0606,
+      "eval_samples_per_second": 8.53,
+      "eval_steps_per_second": 1.079,
+      "step": 500
+    },
+    {
+      "entropy": 0.5665312141180039,
+      "epoch": 1.0039370078740157,
+      "grad_norm": 0.368359237909317,
+      "learning_rate": 0.0001553580381152559,
+      "loss": 0.615216064453125,
+      "mean_token_accuracy": 0.8418210983276367,
+      "num_tokens": 5657246.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.5862001886591315,
+      "epoch": 1.0236220472440944,
+      "grad_norm": 0.45820990204811096,
+      "learning_rate": 0.0001535755005913602,
+      "loss": 0.5764771461486816,
+      "mean_token_accuracy": 0.8442600667476654,
+      "num_tokens": 5768184.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.5108475044369698,
+      "epoch": 1.0433070866141732,
+      "grad_norm": 0.6285914182662964,
+      "learning_rate": 0.00015176875830842938,
+      "loss": 0.5896134376525879,
+      "mean_token_accuracy": 0.8521038725972175,
+      "num_tokens": 5885598.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.486662133410573,
+      "epoch": 1.0629921259842519,
+      "grad_norm": 0.434353768825531,
+      "learning_rate": 0.0001499386275306348,
+      "loss": 0.5298209190368652,
+      "mean_token_accuracy": 0.8610952235758305,
+      "num_tokens": 6002120.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.4824460167437792,
+      "epoch": 1.0826771653543308,
+      "grad_norm": 0.4565293788909912,
+      "learning_rate": 0.0001480859350887852,
+      "loss": 0.5373196601867676,
+      "mean_token_accuracy": 0.8647552341222763,
+      "num_tokens": 6112085.0,
+      "step": 550
+    },
+    {
+      "epoch": 1.0826771653543308,
+      "eval_entropy": 0.6355549980092932,
+      "eval_loss": 0.7618542313575745,
+      "eval_mean_token_accuracy": 0.7958722456737801,
+      "eval_num_tokens": 6112085.0,
+      "eval_runtime": 50.0727,
+      "eval_samples_per_second": 8.528,
+      "eval_steps_per_second": 1.078,
+      "step": 550
+    },
+    {
+      "entropy": 0.501596263051033,
+      "epoch": 1.1023622047244095,
+      "grad_norm": 0.5378302335739136,
+      "learning_rate": 0.00014621151800677434,
+      "loss": 0.5560750007629395,
+      "mean_token_accuracy": 0.8573903530836106,
+      "num_tokens": 6224645.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.4562000434845686,
+      "epoch": 1.1220472440944882,
+      "grad_norm": 0.3977832496166229,
+      "learning_rate": 0.0001443162231234242,
+      "loss": 0.4738937854766846,
+      "mean_token_accuracy": 0.8670585304498672,
+      "num_tokens": 6336790.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.501401986181736,
+      "epoch": 1.141732283464567,
+      "grad_norm": 0.5541960000991821,
+      "learning_rate": 0.0001424009067098931,
+      "loss": 0.5401212692260742,
+      "mean_token_accuracy": 0.8490624278783798,
+      "num_tokens": 6450761.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.46929567167535424,
+      "epoch": 1.1614173228346456,
+      "grad_norm": 0.5008417367935181,
+      "learning_rate": 0.00014046643408282305,
+      "loss": 0.4880987167358398,
+      "mean_token_accuracy": 0.8648147113621235,
+      "num_tokens": 6562060.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.4560689639300108,
+      "epoch": 1.1811023622047245,
+      "grad_norm": 0.47965165972709656,
+      "learning_rate": 0.00013851367921340017,
+      "loss": 0.48796811103820803,
+      "mean_token_accuracy": 0.8638885729014874,
+      "num_tokens": 6676922.0,
+      "step": 600
+    },
+    {
+      "epoch": 1.1811023622047245,
+      "eval_entropy": 0.6515298419528537,
+      "eval_loss": 0.760289192199707,
+      "eval_mean_token_accuracy": 0.7941010395685831,
+      "eval_num_tokens": 6676922.0,
+      "eval_runtime": 50.0511,
+      "eval_samples_per_second": 8.531,
+      "eval_steps_per_second": 1.079,
+      "step": 600
+    },
+    {
+      "entropy": 0.5080817196518183,
+      "epoch": 1.2007874015748032,
+      "grad_norm": 0.5034531354904175,
+      "learning_rate": 0.0001365435243325051,
+      "loss": 0.5181654930114746,
+      "mean_token_accuracy": 0.8537081599235534,
+      "num_tokens": 6783497.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.4773111058399081,
+      "epoch": 1.220472440944882,
+      "grad_norm": 0.5925540328025818,
+      "learning_rate": 0.00013455685953213194,
+      "loss": 0.5433258533477783,
+      "mean_token_accuracy": 0.8583495408296585,
+      "num_tokens": 6894014.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.4819845519028604,
+      "epoch": 1.2401574803149606,
+      "grad_norm": 0.4825916290283203,
+      "learning_rate": 0.0001325545823632554,
+      "loss": 0.5122869491577149,
+      "mean_token_accuracy": 0.8658660687506199,
+      "num_tokens": 6997350.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.5107115621678531,
+      "epoch": 1.2598425196850394,
+      "grad_norm": 0.4685664474964142,
+      "learning_rate": 0.00013053759743032823,
+      "loss": 0.5442278861999512,
+      "mean_token_accuracy": 0.8509419806301594,
+      "num_tokens": 7109067.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.54763747388497,
+      "epoch": 1.279527559055118,
+      "grad_norm": 0.5677510499954224,
+      "learning_rate": 0.00012850681598259195,
+      "loss": 0.597836685180664,
+      "mean_token_accuracy": 0.8460060484707356,
+      "num_tokens": 7218739.0,
+      "step": 650
+    },
+    {
+      "epoch": 1.279527559055118,
+      "eval_entropy": 0.6216344314592855,
+      "eval_loss": 0.7540124654769897,
+      "eval_mean_token_accuracy": 0.7972306355282113,
+      "eval_num_tokens": 7218739.0,
+      "eval_runtime": 50.0442,
+      "eval_samples_per_second": 8.532,
+      "eval_steps_per_second": 1.079,
+      "step": 650
+    },
+    {
+      "entropy": 0.49238223014399407,
+      "epoch": 1.2992125984251968,
+      "grad_norm": 0.5551804304122925,
+      "learning_rate": 0.0001264631555023856,
+      "loss": 0.5532817840576172,
+      "mean_token_accuracy": 0.8577126242220402,
+      "num_tokens": 7334561.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.4327555037103593,
+      "epoch": 1.3188976377952755,
+      "grad_norm": 0.7330867052078247,
+      "learning_rate": 0.00012440753929063826,
+      "loss": 0.49375300407409667,
+      "mean_token_accuracy": 0.8675231724977494,
+      "num_tokens": 7443871.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.5022793110460043,
+      "epoch": 1.3385826771653544,
+      "grad_norm": 0.5363975167274475,
+      "learning_rate": 0.00012234089604973303,
+      "loss": 0.565674114227295,
+      "mean_token_accuracy": 0.8513440594077111,
+      "num_tokens": 7555559.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.48171619242057206,
+      "epoch": 1.358267716535433,
+      "grad_norm": 0.5590063333511353,
+      "learning_rate": 0.0001202641594639305,
+      "loss": 0.5085800170898438,
+      "mean_token_accuracy": 0.8588306061923504,
+      "num_tokens": 7669861.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.5218504227697849,
+      "epoch": 1.3779527559055118,
+      "grad_norm": 0.5833286643028259,
+      "learning_rate": 0.00011817826777754171,
+      "loss": 0.5654644966125488,
+      "mean_token_accuracy": 0.8463637113571167,
+      "num_tokens": 7782831.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.3779527559055118,
+      "eval_entropy": 0.6348862344468081,
+      "eval_loss": 0.7523996829986572,
+      "eval_mean_token_accuracy": 0.7957922904579727,
+      "eval_num_tokens": 7782831.0,
+      "eval_runtime": 50.1023,
+      "eval_samples_per_second": 8.523,
+      "eval_steps_per_second": 1.078,
+      "step": 700
+    },
+    {
+      "entropy": 0.501879290305078,
+      "epoch": 1.3976377952755905,
+      "grad_norm": 0.4697672724723816,
+      "learning_rate": 0.00011608416337104065,
+      "loss": 0.5606812953948974,
+      "mean_token_accuracy": 0.854200828820467,
+      "num_tokens": 7895511.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.45901266783475875,
+      "epoch": 1.4173228346456692,
+      "grad_norm": 0.47944504022598267,
+      "learning_rate": 0.00011398279233530829,
+      "loss": 0.5282856464385987,
+      "mean_token_accuracy": 0.8638743564486504,
+      "num_tokens": 8006186.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.5264331614598632,
+      "epoch": 1.4370078740157481,
+      "grad_norm": 0.515600860118866,
+      "learning_rate": 0.00011187510404420028,
+      "loss": 0.5334322452545166,
+      "mean_token_accuracy": 0.8458372093737125,
+      "num_tokens": 8110444.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.4978476842865348,
+      "epoch": 1.4566929133858268,
+      "grad_norm": 0.7359904050827026,
+      "learning_rate": 0.00010976205072563132,
+      "loss": 0.6107251644134521,
+      "mean_token_accuracy": 0.8548374712467194,
+      "num_tokens": 8220128.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.47486210353672503,
+      "epoch": 1.4763779527559056,
+      "grad_norm": 0.5459336042404175,
+      "learning_rate": 0.00010764458703137024,
+      "loss": 0.5048822402954102,
+      "mean_token_accuracy": 0.8645319528877735,
+      "num_tokens": 8327961.0,
+      "step": 750
+    },
+    {
+      "epoch": 1.4763779527559056,
+      "eval_entropy": 0.6082134053662971,
+      "eval_loss": 0.7509644031524658,
+      "eval_mean_token_accuracy": 0.7988089356157515,
+      "eval_num_tokens": 8327961.0,
+      "eval_runtime": 50.0575,
+      "eval_samples_per_second": 8.53,
+      "eval_steps_per_second": 1.079,
+      "step": 750
+    },
+    {
+      "entropy": 0.4768192932009697,
+      "epoch": 1.4960629921259843,
+      "grad_norm": 0.7404751181602478,
+      "learning_rate": 0.00010552366960573985,
+      "loss": 0.5644009590148926,
+      "mean_token_accuracy": 0.8573225177824497,
+      "num_tokens": 8432725.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.49236655086278913,
+      "epoch": 1.515748031496063,
+      "grad_norm": 0.3972789943218231,
+      "learning_rate": 0.00010340025665341684,
+      "loss": 0.5338064193725586,
+      "mean_token_accuracy": 0.8573864892125129,
+      "num_tokens": 8550042.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.47401853948831557,
+      "epoch": 1.5354330708661417,
+      "grad_norm": 0.4980944097042084,
+      "learning_rate": 0.00010127530750652642,
+      "loss": 0.5052824020385742,
+      "mean_token_accuracy": 0.8658686302602291,
+      "num_tokens": 8664499.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.5025391636416316,
+      "epoch": 1.5551181102362204,
+      "grad_norm": 0.709291934967041,
+      "learning_rate": 9.914978219122791e-05,
+      "loss": 0.5120546340942382,
+      "mean_token_accuracy": 0.8520957857370377,
+      "num_tokens": 8778459.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.44445893429219724,
+      "epoch": 1.574803149606299,
+      "grad_norm": 0.7344470620155334,
+      "learning_rate": 9.702464099398632e-05,
+      "loss": 0.4976132869720459,
+      "mean_token_accuracy": 0.8710284993052483,
+      "num_tokens": 8886361.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.574803149606299,
+      "eval_entropy": 0.6160008189854799,
+      "eval_loss": 0.7450118660926819,
+      "eval_mean_token_accuracy": 0.7979579280923914,
+      "eval_num_tokens": 8886361.0,
+      "eval_runtime": 50.0669,
+      "eval_samples_per_second": 8.529,
+      "eval_steps_per_second": 1.079,
+      "step": 800
+    },
+    {
+      "entropy": 0.4969221928156912,
+      "epoch": 1.594488188976378,
+      "grad_norm": 0.5188943147659302,
+      "learning_rate": 9.4900844027727e-05,
+      "loss": 0.5137925624847413,
+      "mean_token_accuracy": 0.8546112708747386,
+      "num_tokens": 8995558.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.49240641091018916,
+      "epoch": 1.6141732283464567,
+      "grad_norm": 0.5738324522972107,
+      "learning_rate": 9.277935079806795e-05,
+      "loss": 0.5383063793182373,
+      "mean_token_accuracy": 0.8596077263355255,
+      "num_tokens": 9096245.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.4618503864854574,
+      "epoch": 1.6338582677165354,
+      "grad_norm": 0.89094477891922,
+      "learning_rate": 9.066111976982712e-05,
+      "loss": 0.49991846084594727,
+      "mean_token_accuracy": 0.8642474576830864,
+      "num_tokens": 9204264.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.5017987065948546,
+      "epoch": 1.6535433070866141,
+      "grad_norm": 0.5113924145698547,
+      "learning_rate": 8.854710793399947e-05,
+      "loss": 0.5541486740112305,
+      "mean_token_accuracy": 0.8564616665244102,
+      "num_tokens": 9314940.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.47717792335897685,
+      "epoch": 1.673228346456693,
+      "grad_norm": 0.6113857626914978,
+      "learning_rate": 8.64382703754003e-05,
+      "loss": 0.5222208499908447,
+      "mean_token_accuracy": 0.8575348354876041,
+      "num_tokens": 9429496.0,
+      "step": 850
+    },
+    {
+      "epoch": 1.673228346456693,
+      "eval_entropy": 0.6118225814015777,
+      "eval_loss": 0.739860475063324,
+      "eval_mean_token_accuracy": 0.7991818443492607,
+      "eval_num_tokens": 9429496.0,
+      "eval_runtime": 50.1045,
+      "eval_samples_per_second": 8.522,
+      "eval_steps_per_second": 1.078,
+      "step": 850
+    },
+    {
+      "entropy": 0.4527157239615917,
+      "epoch": 1.6929133858267718,
+      "grad_norm": 0.48835644125938416,
+      "learning_rate": 8.433555984116937e-05,
+      "loss": 0.5060441017150878,
+      "mean_token_accuracy": 0.8712181739509106,
+      "num_tokens": 9544270.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.4360736860893667,
+      "epoch": 1.7125984251968505,
+      "grad_norm": 0.5477748513221741,
+      "learning_rate": 8.22399263103314e-05,
+      "loss": 0.5094625949859619,
+      "mean_token_accuracy": 0.8659471146762371,
+      "num_tokens": 9654198.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.4745933244936168,
+      "epoch": 1.7322834645669292,
+      "grad_norm": 0.736858606338501,
+      "learning_rate": 8.015231656460712e-05,
+      "loss": 0.5135779857635498,
+      "mean_token_accuracy": 0.8687760010361671,
+      "num_tokens": 9769457.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.4881877155974507,
+      "epoch": 1.7519685039370079,
+      "grad_norm": 0.608733594417572,
+      "learning_rate": 7.807367376066889e-05,
+      "loss": 0.5163476467132568,
+      "mean_token_accuracy": 0.8567717753350734,
+      "num_tokens": 9875831.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.4794175827410072,
+      "epoch": 1.7716535433070866,
+      "grad_norm": 0.5370113849639893,
+      "learning_rate": 7.60049370040338e-05,
+      "loss": 0.5236234664916992,
+      "mean_token_accuracy": 0.8579100959002972,
+      "num_tokens": 9991758.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.7716535433070866,
+      "eval_entropy": 0.5980368935399585,
+      "eval_loss": 0.7384271621704102,
+      "eval_mean_token_accuracy": 0.7992007765505049,
+      "eval_num_tokens": 9991758.0,
+      "eval_runtime": 50.0629,
+      "eval_samples_per_second": 8.529,
+      "eval_steps_per_second": 1.079,
+      "step": 900
+    },
+    {
+      "entropy": 0.5110580354928971,
+      "epoch": 1.7913385826771653,
+      "grad_norm": 0.5215100049972534,
+      "learning_rate": 7.394704092478742e-05,
+      "loss": 0.5709793567657471,
+      "mean_token_accuracy": 0.8531172260642051,
+      "num_tokens": 10106255.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.4726860882714391,
+      "epoch": 1.811023622047244,
+      "grad_norm": 0.5387454032897949,
+      "learning_rate": 7.190091525532921e-05,
+      "loss": 0.4944759845733643,
+      "mean_token_accuracy": 0.8637764468789101,
+      "num_tokens": 10218066.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.5084793289192021,
+      "epoch": 1.8307086614173227,
+      "grad_norm": 0.6911677718162537,
+      "learning_rate": 6.98674844103308e-05,
+      "loss": 0.554714298248291,
+      "mean_token_accuracy": 0.8502966247498989,
+      "num_tokens": 10326991.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.48612347496673464,
+      "epoch": 1.8503937007874016,
+      "grad_norm": 0.5485708117485046,
+      "learning_rate": 6.784766706909662e-05,
+      "loss": 0.5535434246063232,
+      "mean_token_accuracy": 0.8570072211325168,
+      "num_tokens": 10432558.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.47198982127010825,
+      "epoch": 1.8700787401574803,
+      "grad_norm": 0.7393401861190796,
+      "learning_rate": 6.58423757605158e-05,
+      "loss": 0.5539324760437012,
+      "mean_token_accuracy": 0.8608944483101368,
+      "num_tokens": 10541702.0,
+      "step": 950
+    },
+    {
+      "epoch": 1.8700787401574803,
+      "eval_entropy": 0.6433889176006671,
+      "eval_loss": 0.7281302809715271,
+      "eval_mean_token_accuracy": 0.7999497596864347,
+      "eval_num_tokens": 10541702.0,
+      "eval_runtime": 50.0861,
+      "eval_samples_per_second": 8.525,
+      "eval_steps_per_second": 1.078,
+      "step": 950
+    },
+    {
+      "entropy": 0.5372021682560444,
+      "epoch": 1.889763779527559,
+      "grad_norm": 0.5701485872268677,
+      "learning_rate": 6.38525164507927e-05,
+      "loss": 0.5327393531799316,
+      "mean_token_accuracy": 0.8513112954795361,
+      "num_tokens": 10652431.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.47069347724318505,
+      "epoch": 1.909448818897638,
+      "grad_norm": 0.6635864973068237,
+      "learning_rate": 6.187898813414251e-05,
+      "loss": 0.5279619216918945,
+      "mean_token_accuracy": 0.8639474131166935,
+      "num_tokens": 10766036.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.44332802165299656,
+      "epoch": 1.9291338582677167,
+      "grad_norm": 0.7213886380195618,
+      "learning_rate": 5.99226824266365e-05,
+      "loss": 0.50773606300354,
+      "mean_token_accuracy": 0.8681917086243629,
+      "num_tokens": 10869598.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.4896688494831324,
+      "epoch": 1.9488188976377954,
+      "grad_norm": 0.7005161643028259,
+      "learning_rate": 5.7984483163380764e-05,
+      "loss": 0.5368472099304199,
+      "mean_token_accuracy": 0.8613043077290058,
+      "num_tokens": 10983085.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.44838025560602546,
+      "epoch": 1.968503937007874,
+      "grad_norm": 0.63411545753479,
+      "learning_rate": 5.606526599921026e-05,
+      "loss": 0.5198753833770752,
+      "mean_token_accuracy": 0.8690416067838669,
+      "num_tokens": 11090966.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.968503937007874,
+      "eval_entropy": 0.6066061594971904,
+      "eval_loss": 0.7279917597770691,
+      "eval_mean_token_accuracy": 0.8011515019116578,
+      "eval_num_tokens": 11090966.0,
+      "eval_runtime": 50.076,
+      "eval_samples_per_second": 8.527,
+      "eval_steps_per_second": 1.078,
+      "step": 1000
+    },
+    {
+      "entropy": 0.5023897506296635,
+      "epoch": 1.9881889763779528,
+      "grad_norm": 0.6006460189819336,
+      "learning_rate": 5.416589801307874e-05,
+      "loss": 0.5540077209472656,
+      "mean_token_accuracy": 0.8556332632899284,
+      "num_tokens": 11204584.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.4389859478920698,
+      "epoch": 2.0078740157480315,
+      "grad_norm": 0.38963860273361206,
+      "learning_rate": 5.2287237316322676e-05,
+      "loss": 0.4339911937713623,
+      "mean_token_accuracy": 0.8813330098986626,
+      "num_tokens": 11314232.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.37865939307957885,
+      "epoch": 2.02755905511811,
+      "grad_norm": 0.5649951100349426,
+      "learning_rate": 5.043013266497751e-05,
+      "loss": 0.35414083003997804,
+      "mean_token_accuracy": 0.9043607592582703,
+      "num_tokens": 11422130.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.32439285619184377,
+      "epoch": 2.047244094488189,
+      "grad_norm": 0.6074209809303284,
+      "learning_rate": 4.85954230763197e-05,
+      "loss": 0.36323723793029783,
+      "mean_token_accuracy": 0.9103671722114086,
+      "num_tokens": 11538400.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.31888613002374766,
+      "epoch": 2.0669291338582676,
+      "grad_norm": 0.5395853519439697,
+      "learning_rate": 4.6783937449808856e-05,
+      "loss": 0.32643492221832277,
+      "mean_token_accuracy": 0.9092249758541584,
+      "num_tokens": 11645829.0,
+      "step": 1050
+    },
+    {
+      "epoch": 2.0669291338582676,
+      "eval_entropy": 0.5022390666935179,
+      "eval_loss": 0.7915982604026794,
+      "eval_mean_token_accuracy": 0.7972139318784078,
+      "eval_num_tokens": 11645829.0,
+      "eval_runtime": 50.0745,
+      "eval_samples_per_second": 8.527,
+      "eval_steps_per_second": 1.078,
+      "step": 1050
+    },
+    {
+      "entropy": 0.32281419560313224,
+      "epoch": 2.0866141732283463,
+      "grad_norm": 0.6258149147033691,
+      "learning_rate": 4.499649419260133e-05,
+      "loss": 0.31009109020233155,
+      "mean_token_accuracy": 0.9126963399350643,
+      "num_tokens": 11760076.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.34801057912409306,
+      "epoch": 2.106299212598425,
+      "grad_norm": 0.4270690083503723,
+      "learning_rate": 4.323390084980353e-05,
+      "loss": 0.35411341190338136,
+      "mean_token_accuracy": 0.9039115123450756,
+      "num_tokens": 11875367.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.32329632523469626,
+      "epoch": 2.1259842519685037,
+      "grad_norm": 0.6041743159294128,
+      "learning_rate": 4.1496953739633124e-05,
+      "loss": 0.31601505279541015,
+      "mean_token_accuracy": 0.9122780673205853,
+      "num_tokens": 11983766.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.32552749328315256,
+      "epoch": 2.145669291338583,
+      "grad_norm": 0.6716635227203369,
+      "learning_rate": 3.978643759365202e-05,
+      "loss": 0.33874006271362306,
+      "mean_token_accuracy": 0.9068517275154591,
+      "num_tokens": 12092559.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.32265327414497735,
+      "epoch": 2.1653543307086616,
+      "grad_norm": 0.606459379196167,
+      "learning_rate": 3.810312520223419e-05,
+      "loss": 0.33009474277496337,
+      "mean_token_accuracy": 0.9104282423853874,
+      "num_tokens": 12207307.0,
+      "step": 1100
+    },
+    {
+      "epoch": 2.1653543307086616,
+      "eval_entropy": 0.49647785005746065,
+      "eval_loss": 0.7967986464500427,
+      "eval_mean_token_accuracy": 0.7973429693116082,
+      "eval_num_tokens": 12207307.0,
+      "eval_runtime": 50.0997,
+      "eval_samples_per_second": 8.523,
+      "eval_steps_per_second": 1.078,
+      "step": 1100
+    },
+    {
+      "entropy": 0.33745345920324327,
+      "epoch": 2.1850393700787403,
+      "grad_norm": 0.5246295928955078,
+      "learning_rate": 3.644777706542845e-05,
+      "loss": 0.34748287200927735,
+      "mean_token_accuracy": 0.9026777386665344,
+      "num_tokens": 12319969.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.31724564395844934,
+      "epoch": 2.204724409448819,
+      "grad_norm": 0.5573722720146179,
+      "learning_rate": 3.4821141049373804e-05,
+      "loss": 0.2927221775054932,
+      "mean_token_accuracy": 0.9131564185023308,
+      "num_tokens": 12431027.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.3440235279500484,
+      "epoch": 2.2244094488188977,
+      "grad_norm": 0.6053721904754639,
+      "learning_rate": 3.3223952048422544e-05,
+      "loss": 0.34805119037628174,
+      "mean_token_accuracy": 0.9048855185508728,
+      "num_tokens": 12546037.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.33274165438488124,
+      "epoch": 2.2440944881889764,
+      "grad_norm": 0.6461567282676697,
+      "learning_rate": 3.165693165312395e-05,
+      "loss": 0.34774529933929443,
+      "mean_token_accuracy": 0.9065869458019733,
+      "num_tokens": 12656297.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.31052923379465935,
+      "epoch": 2.263779527559055,
+      "grad_norm": 0.6634212136268616,
+      "learning_rate": 3.0120787824218554e-05,
+      "loss": 0.3316983222961426,
+      "mean_token_accuracy": 0.9117146715521812,
+      "num_tokens": 12763743.0,
+      "step": 1150
+    },
+    {
+      "epoch": 2.263779527559055,
+      "eval_entropy": 0.49124936207577036,
+      "eval_loss": 0.7972699403762817,
+      "eval_mean_token_accuracy": 0.7969378519941259,
+      "eval_num_tokens": 12763743.0,
+      "eval_runtime": 50.0791,
+      "eval_samples_per_second": 8.527,
+      "eval_steps_per_second": 1.078,
+      "step": 1150
+    },
+    {
+      "entropy": 0.3448523377999663,
+      "epoch": 2.283464566929134,
+      "grad_norm": 0.5939217805862427,
+      "learning_rate": 2.8616214572789968e-05,
+      "loss": 0.34267549514770507,
+      "mean_token_accuracy": 0.9079354353249073,
+      "num_tokens": 12874080.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.2993648503907025,
+      "epoch": 2.3031496062992125,
+      "grad_norm": 0.5903282165527344,
+      "learning_rate": 2.7143891646719265e-05,
+      "loss": 0.3056223392486572,
+      "mean_token_accuracy": 0.9152895107865333,
+      "num_tokens": 12986029.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.3108034676872194,
+      "epoch": 2.322834645669291,
+      "grad_norm": 0.7086691856384277,
+      "learning_rate": 2.570448422358297e-05,
+      "loss": 0.3301855564117432,
+      "mean_token_accuracy": 0.9101004771888256,
+      "num_tokens": 13092818.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.31144436639733614,
+      "epoch": 2.34251968503937,
+      "grad_norm": 0.683090329170227,
+      "learning_rate": 2.4298642610134105e-05,
+      "loss": 0.3339296817779541,
+      "mean_token_accuracy": 0.9146400101482868,
+      "num_tokens": 13206493.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.3901509671472013,
+      "epoch": 2.362204724409449,
+      "grad_norm": 0.798456609249115,
+      "learning_rate": 2.292700194850147e-05,
+      "loss": 0.3838188648223877,
+      "mean_token_accuracy": 0.8957899294793605,
+      "num_tokens": 13314862.0,
+      "step": 1200
+    },
+    {
+      "epoch": 2.362204724409449,
+      "eval_entropy": 0.4911509961993606,
+      "eval_loss": 0.7939621806144714,
+      "eval_mean_token_accuracy": 0.7974915846630379,
+      "eval_num_tokens": 13314862.0,
+      "eval_runtime": 50.0789,
+      "eval_samples_per_second": 8.527,
+      "eval_steps_per_second": 1.078,
+      "step": 1200
+    },
+    {
+      "entropy": 0.3228563711978495,
+      "epoch": 2.3818897637795278,
+      "grad_norm": 0.5817623138427734,
+      "learning_rate": 2.159018192924004e-05,
+      "loss": 0.3346099853515625,
+      "mean_token_accuracy": 0.9080430999398231,
+      "num_tokens": 13423520.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.3402016874402761,
+      "epoch": 2.4015748031496065,
+      "grad_norm": 0.5687164068222046,
+      "learning_rate": 2.0288786511362525e-05,
+      "loss": 0.34886922836303713,
+      "mean_token_accuracy": 0.9053414203226566,
+      "num_tokens": 13536197.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.32414694763720037,
+      "epoch": 2.421259842519685,
+      "grad_norm": 0.6170526742935181,
+      "learning_rate": 1.9023403649477813e-05,
+      "loss": 0.3294362545013428,
+      "mean_token_accuracy": 0.9133819483220578,
+      "num_tokens": 13649116.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.30389058096334337,
+      "epoch": 2.440944881889764,
+      "grad_norm": 0.5538958311080933,
+      "learning_rate": 1.7794605028160462e-05,
+      "loss": 0.3009462594985962,
+      "mean_token_accuracy": 0.909288102388382,
+      "num_tokens": 13759803.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.3456068937666714,
+      "epoch": 2.4606299212598426,
+      "grad_norm": 0.8508191704750061,
+      "learning_rate": 1.6602945803670345e-05,
+      "loss": 0.3429473638534546,
+      "mean_token_accuracy": 0.9037580870091915,
+      "num_tokens": 13872536.0,
+      "step": 1250
+    },
+    {
+      "epoch": 2.4606299212598426,
+      "eval_entropy": 0.4867473399197614,
+      "eval_loss": 0.798021137714386,
+      "eval_mean_token_accuracy": 0.7978451141604671,
+      "eval_num_tokens": 13872536.0,
+      "eval_runtime": 50.0975,
+      "eval_samples_per_second": 8.523,
+      "eval_steps_per_second": 1.078,
+      "step": 1250
+    },
+    {
+      "entropy": 0.30477998238056897,
+      "epoch": 2.4803149606299213,
+      "grad_norm": 0.7420564889907837,
+      "learning_rate": 1.544896435313997e-05,
+      "loss": 0.323688268661499,
+      "mean_token_accuracy": 0.9099539265036583,
+      "num_tokens": 13982332.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.325347469933331,
+      "epoch": 2.5,
+      "grad_norm": 0.6256458163261414,
+      "learning_rate": 1.4333182031342185e-05,
+      "loss": 0.3231887578964233,
+      "mean_token_accuracy": 0.905095786601305,
+      "num_tokens": 14096535.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.3353913531638682,
+      "epoch": 2.5196850393700787,
+      "grad_norm": 0.500119149684906,
+      "learning_rate": 1.3256102935148496e-05,
+      "loss": 0.36284313201904295,
+      "mean_token_accuracy": 0.9058476276695728,
+      "num_tokens": 14214551.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.3284754135645926,
+      "epoch": 2.5393700787401574,
+      "grad_norm": 0.7045860886573792,
+      "learning_rate": 1.2218213675784063e-05,
+      "loss": 0.34949867725372313,
+      "mean_token_accuracy": 0.9073399856686593,
+      "num_tokens": 14322085.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.3220018117688596,
+      "epoch": 2.559055118110236,
+      "grad_norm": 0.5107280611991882,
+      "learning_rate": 1.1219983158982917e-05,
+      "loss": 0.32134926319122314,
+      "mean_token_accuracy": 0.9144444465637207,
+      "num_tokens": 14431579.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.559055118110236,
+      "eval_entropy": 0.4890395458097811,
+      "eval_loss": 0.7947290539741516,
+      "eval_mean_token_accuracy": 0.7974367803997464,
+      "eval_num_tokens": 14431579.0,
+      "eval_runtime": 50.1041,
+      "eval_samples_per_second": 8.522,
+      "eval_steps_per_second": 1.078,
+      "step": 1300
+    },
+    {
+      "entropy": 0.2850076165981591,
+      "epoch": 2.578740157480315,
+      "grad_norm": 0.7151411771774292,
+      "learning_rate": 1.0261862373141839e-05,
+      "loss": 0.2979953527450562,
+      "mean_token_accuracy": 0.9207482695579529,
+      "num_tokens": 14536887.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.2971831484232098,
+      "epoch": 2.5984251968503935,
+      "grad_norm": 0.7176497578620911,
+      "learning_rate": 9.344284185569463e-06,
+      "loss": 0.3046797037124634,
+      "mean_token_accuracy": 0.9163132011890411,
+      "num_tokens": 14649040.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.31773042744025587,
+      "epoch": 2.6181102362204722,
+      "grad_norm": 0.5870872735977173,
+      "learning_rate": 8.467663146921979e-06,
+      "loss": 0.323029899597168,
+      "mean_token_accuracy": 0.9097053371369839,
+      "num_tokens": 14757590.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.317878791783005,
+      "epoch": 2.637795275590551,
+      "grad_norm": 0.5370131134986877,
+      "learning_rate": 7.632395303914208e-06,
+      "loss": 0.3096921920776367,
+      "mean_token_accuracy": 0.9069532811641693,
+      "num_tokens": 14870411.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.30778416004031894,
+      "epoch": 2.65748031496063,
+      "grad_norm": 0.6148006319999695,
+      "learning_rate": 6.838858020390615e-06,
+      "loss": 0.33772549629211424,
+      "mean_token_accuracy": 0.913542590290308,
+      "num_tokens": 14986705.0,
+      "step": 1350
+    },
+    {
+      "epoch": 2.65748031496063,
+      "eval_entropy": 0.4901013904147678,
+      "eval_loss": 0.7947279214859009,
+      "eval_mean_token_accuracy": 0.7975728578037686,
+      "eval_num_tokens": 14986705.0,
+      "eval_runtime": 50.0762,
+      "eval_samples_per_second": 8.527,
+      "eval_steps_per_second": 1.078,
+      "step": 1350
+    },
+    {
+      "entropy": 0.3350960427895188,
+      "epoch": 2.677165354330709,
+      "grad_norm": 0.7815200686454773,
+      "learning_rate": 6.087409806836752e-06,
+      "loss": 0.3436413764953613,
+      "mean_token_accuracy": 0.9064855180680752,
+      "num_tokens": 15095469.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.2992165580391884,
+      "epoch": 2.6968503937007875,
+      "grad_norm": 0.6063485741615295,
+      "learning_rate": 5.378390158408731e-06,
+      "loss": 0.3226536512374878,
+      "mean_token_accuracy": 0.9164511568844318,
+      "num_tokens": 15202248.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.3059391477610916,
+      "epoch": 2.716535433070866,
+      "grad_norm": 0.582249104976654,
+      "learning_rate": 4.712119401553394e-06,
+      "loss": 0.2904583692550659,
+      "mean_token_accuracy": 0.9138661302626133,
+      "num_tokens": 15312525.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.3175976496189833,
+      "epoch": 2.736220472440945,
+      "grad_norm": 0.5042891502380371,
+      "learning_rate": 4.088898549288689e-06,
+      "loss": 0.33177206516265867,
+      "mean_token_accuracy": 0.9126643091440201,
+      "num_tokens": 15424901.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.3237245835363865,
+      "epoch": 2.7559055118110236,
+      "grad_norm": 0.6042836308479309,
+      "learning_rate": 3.5090091652096977e-06,
+      "loss": 0.3255606174468994,
+      "mean_token_accuracy": 0.911313808709383,
+      "num_tokens": 15534177.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.7559055118110236,
+      "eval_entropy": 0.49696798898555616,
+      "eval_loss": 0.7927594780921936,
+      "eval_mean_token_accuracy": 0.7980255352126228,
+      "eval_num_tokens": 15534177.0,
+      "eval_runtime": 50.0812,
+      "eval_samples_per_second": 8.526,
+      "eval_steps_per_second": 1.078,
+      "step": 1400
+    },
+    {
+      "entropy": 0.3428220834583044,
+      "epoch": 2.7755905511811023,
+      "grad_norm": 0.728273332118988,
+      "learning_rate": 2.972713236281632e-06,
+      "loss": 0.3655885934829712,
+      "mean_token_accuracy": 0.9057191483676433,
+      "num_tokens": 15642725.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.3044391763396561,
+      "epoch": 2.795275590551181,
+      "grad_norm": 0.5648255944252014,
+      "learning_rate": 2.480253054477244e-06,
+      "loss": 0.3287725210189819,
+      "mean_token_accuracy": 0.9158088378608227,
+      "num_tokens": 15749590.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.32088840757496656,
+      "epoch": 2.8149606299212597,
+      "grad_norm": 0.5879103541374207,
+      "learning_rate": 2.031851107312366e-06,
+      "loss": 0.3544080018997192,
+      "mean_token_accuracy": 0.9135449744760991,
+      "num_tokens": 15865143.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.3113178635016084,
+      "epoch": 2.8346456692913384,
+      "grad_norm": 0.6843683123588562,
+      "learning_rate": 1.6277099773286951e-06,
+      "loss": 0.29128711223602294,
+      "mean_token_accuracy": 0.9160408519208432,
+      "num_tokens": 15967667.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.32423784183338283,
+      "epoch": 2.8543307086614176,
+      "grad_norm": 0.6659854650497437,
+      "learning_rate": 1.2680122505694502e-06,
+      "loss": 0.3179208755493164,
+      "mean_token_accuracy": 0.9091328486800194,
+      "num_tokens": 16077301.0,
+      "step": 1450
+    },
+    {
+      "epoch": 2.8543307086614176,
+      "eval_entropy": 0.4910302360852559,
+      "eval_loss": 0.792415976524353,
+      "eval_mean_token_accuracy": 0.798325040826091,
+      "eval_num_tokens": 16077301.0,
+      "eval_runtime": 50.0532,
+      "eval_samples_per_second": 8.531,
+      "eval_steps_per_second": 1.079,
+      "step": 1450
+    },
+    {
+      "entropy": 0.3308924221433699,
+      "epoch": 2.8740157480314963,
+      "grad_norm": 0.6177695989608765,
+      "learning_rate": 9.529204340892772e-07,
+      "loss": 0.34883484840393064,
+      "mean_token_accuracy": 0.907434081286192,
+      "num_tokens": 16190635.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.32041617911309006,
+      "epoch": 2.893700787401575,
+      "grad_norm": 0.9388318657875061,
+      "learning_rate": 6.825768825355017e-07,
+      "loss": 0.33657877445220946,
+      "mean_token_accuracy": 0.9068625263869763,
+      "num_tokens": 16304259.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.29119140561670065,
+      "epoch": 2.9133858267716537,
+      "grad_norm": 0.6246579885482788,
+      "learning_rate": 4.5710373383401895e-07,
+      "loss": 0.28482346534729003,
+      "mean_token_accuracy": 0.9181540630757808,
+      "num_tokens": 16414959.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.30836315331980585,
+      "epoch": 2.9330708661417324,
+      "grad_norm": 0.6171616911888123,
+      "learning_rate": 2.766028540089227e-07,
+      "loss": 0.3132902145385742,
+      "mean_token_accuracy": 0.9110966674983502,
+      "num_tokens": 16531149.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.2961795757524669,
+      "epoch": 2.952755905511811,
+      "grad_norm": 0.7841485142707825,
+      "learning_rate": 1.4115579116061873e-07,
+      "loss": 0.3022057771682739,
+      "mean_token_accuracy": 0.9193245336413384,
+      "num_tokens": 16634608.0,
+      "step": 1500
+    },
+    {
+      "epoch": 2.952755905511811,
+      "eval_entropy": 0.490779525703854,
+      "eval_loss": 0.792911946773529,
+      "eval_mean_token_accuracy": 0.7981838837817863,
+      "eval_num_tokens": 16634608.0,
+      "eval_runtime": 50.0697,
+      "eval_samples_per_second": 8.528,
+      "eval_steps_per_second": 1.078,
+      "step": 1500
+    },
+    {
+      "entropy": 0.3136138510890305,
+      "epoch": 2.97244094488189,
+      "grad_norm": 0.5390800833702087,
+      "learning_rate": 5.0823738623417826e-08,
+      "loss": 0.3194080591201782,
+      "mean_token_accuracy": 0.9103287599980832,
+      "num_tokens": 16742519.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.31666473839432,
+      "epoch": 2.9921259842519685,
+      "grad_norm": 0.5248324275016785,
+      "learning_rate": 5.647507319117562e-09,
+      "loss": 0.31403298377990724,
+      "mean_token_accuracy": 0.9132906906306744,
+      "num_tokens": 16855391.0,
+      "step": 1520
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 0.49082624525935564,
+      "eval_loss": 0.7929432988166809,
+      "eval_mean_token_accuracy": 0.7976736547770323,
+      "eval_num_tokens": 16901880.0,
+      "eval_runtime": 50.0698,
+      "eval_samples_per_second": 8.528,
+      "eval_steps_per_second": 1.078,
+      "step": 1524
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1524,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.956265936390922e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1524/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c565827a95d76fd524e57e1a6f572b1e00f75c93f00b63781d130a3eb57a2eb6
+size 5713

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|fim_pad|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c565827a95d76fd524e57e1a6f572b1e00f75c93f00b63781d130a3eb57a2eb6
+size 5713