Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1206/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,61 @@
+---
+base_model: numind/NuExtract-2.0-2B
+library_name: peft
+model_name: nuextract_lora_adapter_2b_split
+tags:
+- base_model:adapter:numind/NuExtract-2.0-2B
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for nuextract_lora_adapter_2b_split
+
+This model is a fine-tuned version of [numind/NuExtract-2.0-2B](https://huggingface.co/numind/NuExtract-2.0-2B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.28.0
+- Transformers: 5.2.0
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "numind/NuExtract-2.0-2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75111344ac6827da35318c801df614a13fec222a63157a24e959fa27b46ac70d
+size 36987736

chat_template.jinja

@@ -0,0 +1,138 @@
+{%- set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' -%}
+{%- for message in messages -%}
+    {%- if message['role'] == 'system' -%}
+        {{- '<|im_start|>system
+' -}}
+        {%- if message['content'] is string -%}
+            {{- message['content'] | trim -}}
+        {%- endif -%}
+        {{- '<|im_end|>
+' -}}
+    {%- elif message['role'] == 'user' -%}
+        {%- if loop.first -%}
+            {{- '<|im_start|>system
+' -}}
+            {%- if template -%}
+                {#--- If template, extraction task ---#}
+                {{- 'You are NuExtract, an information extraction tool created by NuMind.' -}}
+            {%- else -%}
+                {#--- Else, template generation task ---#}
+                {{- 'You are a helpful assistant.' -}}
+            {%- endif -%}
+            {{ '<|im_end|>
+' }}
+        {%- endif -%}
+        {{- '<|im_start|>' + message['role'] + '
+' -}}
+        {%- if template -%}
+            {#--- Template Section ---#}
+            {{- '# Template:
+' -}}
+            {{- template -}}
+            {{- '
+' -}}
+
+            {%- if examples -%}
+                {#--- Examples can only exist in the extraction task ---#}
+                {{- '# Examples:
+' -}}
+                {%- for example in examples -%}
+                    {{- '## Input:
+' -}}
+                    {%- if example['input'] is mapping and (example['input']['type'] == 'image' or example['input']['type'] == 'image_url') -%}
+                        {{- image_placeholder | trim -}}
+                    {%- elif example['input'] == '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                    {%- else -%}
+                        {#--- Text input example ---#}
+                        {{- example['input'] -}}
+                    {%- endif -%}
+                    {{- '
+' -}}
+                    {{- '## Output:
+' -}}
+                    {{- example['output'] -}}
+                    {{- '
+' -}}
+                {%- endfor -%}
+            {%- endif -%}
+            {{- '# Context:
+' -}}
+        {%- endif -%}
+
+        {%- if message['content'] is string -%}
+            {#--- Simple string content ---#}
+            {{- message['content'] | trim -}}
+        {%- elif message['content'] is mapping and (message['content']['type'] == 'image' or message['content']['type'] == 'image_url') -%}
+            {{- image_placeholder | trim -}}
+        {%- else -%}
+            {#--- List of content items (mixed text/images) ---#}
+            {#--- First, determine what the actual input content is (not ICL images) ---#}
+            {%- set ns = namespace(has_text_input=false, text_content='') -%}
+
+            {#--- Count content types and identify actual input document ---#}
+            {%- for content in message['content'] -%}
+                {%- if content is mapping and content.get('type') == 'text' -%}
+                    {%- if content.get('text') != '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {%- set ns.has_text_input = true -%}
+                        {%- set ns.text_content = content['text'] -%}
+                    {%- endif -%}
+                {%- elif content is string -%}
+                    {%- if content != '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {%- set ns.has_text_input = true -%}
+                        {%- set ns.text_content = content -%}
+                    {%- endif -%}
+                {%- endif -%}
+            {%- endfor -%}
+
+            {#--- Determine what to output based on actual input type ---#}
+            {%- if ns.has_text_input -%}
+                {#--- Main input is text, so output the text content ---#}
+                {{- ns.text_content | trim -}}
+            {%- else -%}
+                {#--- Main input is image or <image> placeholder ---#}
+                {%- set ns2 = namespace(found_image=false) -%}
+                {%- for content in message['content'] -%}
+                    {%- if content is mapping and (content.get('type') == 'image' or content.get('type') == 'image_url') and not ns2.found_image -%}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' and not ns2.found_image -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- elif content is string and content == '<image>' and not ns2.found_image -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+        {%- endif -%}
+        {{- '<|im_end|>
+'}}
+
+    {%- elif message['role'] == 'assistant' -%}
+        {{- '<|im_start|>assistant
+' -}}
+        {%- if message['content'] is string -%}
+            {{- message['content'] | trim -}}
+        {%- elif message['content'] is iterable and message['content'] is not string -%}
+            {%- for content in message['content'] -%}
+                {%- if content is mapping and content.get('type') == 'text' -%}
+                    {{- content['text'] | trim -}}
+                {%- elif content is string -%}
+                    {{- content | trim -}}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- '<|im_end|>
+' -}}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- '<|im_start|>assistant
+' -}}
+{%- endif -%}

checkpoint-1000/README.md

@@ -0,0 +1,209 @@
+---
+base_model: numind/NuExtract-2.0-2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:numind/NuExtract-2.0-2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-1000/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "numind/NuExtract-2.0-2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5c4dbe8236fe54809b4e05229bacd2a242d70ce8265231dedfe3345aab7b164
+size 36987736

checkpoint-1000/chat_template.jinja

@@ -0,0 +1,138 @@
+{%- set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' -%}
+{%- for message in messages -%}
+    {%- if message['role'] == 'system' -%}
+        {{- '<|im_start|>system
+' -}}
+        {%- if message['content'] is string -%}
+            {{- message['content'] | trim -}}
+        {%- endif -%}
+        {{- '<|im_end|>
+' -}}
+    {%- elif message['role'] == 'user' -%}
+        {%- if loop.first -%}
+            {{- '<|im_start|>system
+' -}}
+            {%- if template -%}
+                {#--- If template, extraction task ---#}
+                {{- 'You are NuExtract, an information extraction tool created by NuMind.' -}}
+            {%- else -%}
+                {#--- Else, template generation task ---#}
+                {{- 'You are a helpful assistant.' -}}
+            {%- endif -%}
+            {{ '<|im_end|>
+' }}
+        {%- endif -%}
+        {{- '<|im_start|>' + message['role'] + '
+' -}}
+        {%- if template -%}
+            {#--- Template Section ---#}
+            {{- '# Template:
+' -}}
+            {{- template -}}
+            {{- '
+' -}}
+
+            {%- if examples -%}
+                {#--- Examples can only exist in the extraction task ---#}
+                {{- '# Examples:
+' -}}
+                {%- for example in examples -%}
+                    {{- '## Input:
+' -}}
+                    {%- if example['input'] is mapping and (example['input']['type'] == 'image' or example['input']['type'] == 'image_url') -%}
+                        {{- image_placeholder | trim -}}
+                    {%- elif example['input'] == '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                    {%- else -%}
+                        {#--- Text input example ---#}
+                        {{- example['input'] -}}
+                    {%- endif -%}
+                    {{- '
+' -}}
+                    {{- '## Output:
+' -}}
+                    {{- example['output'] -}}
+                    {{- '
+' -}}
+                {%- endfor -%}
+            {%- endif -%}
+            {{- '# Context:
+' -}}
+        {%- endif -%}
+
+        {%- if message['content'] is string -%}
+            {#--- Simple string content ---#}
+            {{- message['content'] | trim -}}
+        {%- elif message['content'] is mapping and (message['content']['type'] == 'image' or message['content']['type'] == 'image_url') -%}
+            {{- image_placeholder | trim -}}
+        {%- else -%}
+            {#--- List of content items (mixed text/images) ---#}
+            {#--- First, determine what the actual input content is (not ICL images) ---#}
+            {%- set ns = namespace(has_text_input=false, text_content='') -%}
+
+            {#--- Count content types and identify actual input document ---#}
+            {%- for content in message['content'] -%}
+                {%- if content is mapping and content.get('type') == 'text' -%}
+                    {%- if content.get('text') != '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {%- set ns.has_text_input = true -%}
+                        {%- set ns.text_content = content['text'] -%}
+                    {%- endif -%}
+                {%- elif content is string -%}
+                    {%- if content != '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {%- set ns.has_text_input = true -%}
+                        {%- set ns.text_content = content -%}
+                    {%- endif -%}
+                {%- endif -%}
+            {%- endfor -%}
+
+            {#--- Determine what to output based on actual input type ---#}
+            {%- if ns.has_text_input -%}
+                {#--- Main input is text, so output the text content ---#}
+                {{- ns.text_content | trim -}}
+            {%- else -%}
+                {#--- Main input is image or <image> placeholder ---#}
+                {%- set ns2 = namespace(found_image=false) -%}
+                {%- for content in message['content'] -%}
+                    {%- if content is mapping and (content.get('type') == 'image' or content.get('type') == 'image_url') and not ns2.found_image -%}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' and not ns2.found_image -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- elif content is string and content == '<image>' and not ns2.found_image -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+        {%- endif -%}
+        {{- '<|im_end|>
+'}}
+
+    {%- elif message['role'] == 'assistant' -%}
+        {{- '<|im_start|>assistant
+' -}}
+        {%- if message['content'] is string -%}
+            {{- message['content'] | trim -}}
+        {%- elif message['content'] is iterable and message['content'] is not string -%}
+            {%- for content in message['content'] -%}
+                {%- if content is mapping and content.get('type') == 'text' -%}
+                    {{- content['text'] | trim -}}
+                {%- elif content is string -%}
+                    {{- content | trim -}}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- '<|im_end|>
+' -}}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- '<|im_start|>assistant
+' -}}
+{%- endif -%}

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:879d24d360f3ca5ca839c8edf41c787f4e2ef84db201c4b0bf8d7c5d6ec7ae62
+size 37974661

checkpoint-1000/processor_config.json

@@ -0,0 +1,63 @@
+{
+  "image_processor": {
+    "data_format": "channels_first",
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "Qwen2VLImageProcessorFast",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "merge_size": 2,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 23000000,
+      "shortest_edge": 200704
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen2_5_VLProcessor",
+  "video_processor": {
+    "data_format": "channels_first",
+    "default_to_square": true,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": false,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 23000000,
+      "shortest_edge": 200704
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen2VLVideoProcessor"
+  }
+}

checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9915f3c83949abaf128c7e49e29c23e22d42e3acb940456a8ba772e63e58a1ae
+size 14645

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80a59461368f07e3f04d9f2e341b48e67443a0f97cb03fd267e1f8ff0620ba91
+size 1465

checkpoint-1000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a7aabe7a3b44651592a99607bb33066e6965260336be9218cccd53af161491d
+size 11422060

checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "_commit_hash": null,
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "is_local": false,
+  "max_length": null,
+  "max_pixels": 23000000,
+  "min_pixels": 200704,
+  "model_max_length": 131072,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "processor_class": "Qwen2_5_VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-1000/trainer_state.json

@@ -0,0 +1,2056 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.487713841368585,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.7456206619739532,
+      "epoch": 0.012441679626749611,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 1.0810810810810812e-05,
+      "loss": 0.31386559009552,
+      "mean_token_accuracy": 0.9072110533714295,
+      "num_tokens": 27963.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.8172702223062516,
+      "epoch": 0.024883359253499222,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 2.4324324324324327e-05,
+      "loss": 0.28579437732696533,
+      "mean_token_accuracy": 0.9151854813098907,
+      "num_tokens": 51393.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.7869735598564147,
+      "epoch": 0.03732503888024884,
+      "grad_norm": 0.046875,
+      "learning_rate": 3.783783783783784e-05,
+      "loss": 0.26468374729156496,
+      "mean_token_accuracy": 0.9194357767701149,
+      "num_tokens": 76323.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.7407771855592729,
+      "epoch": 0.049766718506998445,
+      "grad_norm": 0.0546875,
+      "learning_rate": 5.135135135135135e-05,
+      "loss": 0.22699759006500245,
+      "mean_token_accuracy": 0.936237809062004,
+      "num_tokens": 101144.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.8171133548021317,
+      "epoch": 0.06220839813374806,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 6.486486486486487e-05,
+      "loss": 0.21160719394683838,
+      "mean_token_accuracy": 0.9410519078373909,
+      "num_tokens": 125653.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.6494354471564292,
+      "epoch": 0.07465007776049767,
+      "grad_norm": 0.068359375,
+      "learning_rate": 7.837837837837838e-05,
+      "loss": 0.1776595950126648,
+      "mean_token_accuracy": 0.9463573560118675,
+      "num_tokens": 150968.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.5880378365516663,
+      "epoch": 0.08709175738724728,
+      "grad_norm": 0.076171875,
+      "learning_rate": 9.18918918918919e-05,
+      "loss": 0.18801350593566896,
+      "mean_token_accuracy": 0.9474205821752548,
+      "num_tokens": 176859.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.5649296075105668,
+      "epoch": 0.09953343701399689,
+      "grad_norm": 0.064453125,
+      "learning_rate": 9.99992777797229e-05,
+      "loss": 0.18648279905319215,
+      "mean_token_accuracy": 0.9436241507530212,
+      "num_tokens": 198909.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.5894027322530746,
+      "epoch": 0.1119751166407465,
+      "grad_norm": 0.05322265625,
+      "learning_rate": 9.999115304121457e-05,
+      "loss": 0.18634642362594606,
+      "mean_token_accuracy": 0.9470892950892449,
+      "num_tokens": 228344.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.596228164434433,
+      "epoch": 0.12441679626749612,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.997400226068578e-05,
+      "loss": 0.14496899843215943,
+      "mean_token_accuracy": 0.9579829111695289,
+      "num_tokens": 257062.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.509284293651581,
+      "epoch": 0.1368584758942457,
+      "grad_norm": 0.03369140625,
+      "learning_rate": 9.994782853475774e-05,
+      "loss": 0.14144277572631836,
+      "mean_token_accuracy": 0.9537434980273247,
+      "num_tokens": 283149.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.5500743955373764,
+      "epoch": 0.14930015552099535,
+      "grad_norm": 0.0546875,
+      "learning_rate": 9.99126365891696e-05,
+      "loss": 0.15906004905700682,
+      "mean_token_accuracy": 0.9526497855782509,
+      "num_tokens": 313192.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.5956889361143112,
+      "epoch": 0.16174183514774496,
+      "grad_norm": 0.061279296875,
+      "learning_rate": 9.986843277792523e-05,
+      "loss": 0.17970917224884034,
+      "mean_token_accuracy": 0.9503620356321335,
+      "num_tokens": 340137.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.6294057995080948,
+      "epoch": 0.17418351477449456,
+      "grad_norm": 0.055908203125,
+      "learning_rate": 9.9815225082146e-05,
+      "loss": 0.14261267185211182,
+      "mean_token_accuracy": 0.9562160760164261,
+      "num_tokens": 364984.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.589295208454132,
+      "epoch": 0.18662519440124417,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 9.975302310862964e-05,
+      "loss": 0.1694250226020813,
+      "mean_token_accuracy": 0.9456220239400863,
+      "num_tokens": 390651.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.583991777896881,
+      "epoch": 0.19906687402799378,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.968183808811586e-05,
+      "loss": 0.17054636478424073,
+      "mean_token_accuracy": 0.9493257701396942,
+      "num_tokens": 418166.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.5742847174406052,
+      "epoch": 0.2115085536547434,
+      "grad_norm": 0.06103515625,
+      "learning_rate": 9.960168287325852e-05,
+      "loss": 0.17596685886383057,
+      "mean_token_accuracy": 0.9445611819624901,
+      "num_tokens": 442501.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.5073494076728822,
+      "epoch": 0.223950233281493,
+      "grad_norm": 0.06640625,
+      "learning_rate": 9.951257193630507e-05,
+      "loss": 0.13867353200912474,
+      "mean_token_accuracy": 0.9568744376301765,
+      "num_tokens": 468083.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.5407251685857772,
+      "epoch": 0.2363919129082426,
+      "grad_norm": 0.0673828125,
+      "learning_rate": 9.941452136648359e-05,
+      "loss": 0.13775357007980346,
+      "mean_token_accuracy": 0.9594547316431999,
+      "num_tokens": 491914.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.4568117886781693,
+      "epoch": 0.24883359253499224,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 9.930754886709772e-05,
+      "loss": 0.13010139465332032,
+      "mean_token_accuracy": 0.9607375919818878,
+      "num_tokens": 518984.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.4679558753967286,
+      "epoch": 0.26127527216174184,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 9.919167375233041e-05,
+      "loss": 0.15394901037216185,
+      "mean_token_accuracy": 0.9542355045676232,
+      "num_tokens": 544627.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.5308140426874162,
+      "epoch": 0.2737169517884914,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 9.906691694375656e-05,
+      "loss": 0.14595698118209838,
+      "mean_token_accuracy": 0.9553182557225227,
+      "num_tokens": 568970.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.5124749779701232,
+      "epoch": 0.28615863141524106,
+      "grad_norm": 0.08056640625,
+      "learning_rate": 9.893330096656574e-05,
+      "loss": 0.1251847267150879,
+      "mean_token_accuracy": 0.9631539225578308,
+      "num_tokens": 594854.0,
+      "step": 115
+    },
+    {
+      "entropy": 1.5146708011627197,
+      "epoch": 0.2986003110419907,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 9.879084994549503e-05,
+      "loss": 0.12504571676254272,
+      "mean_token_accuracy": 0.9637450009584427,
+      "num_tokens": 621481.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.5158167153596878,
+      "epoch": 0.3110419906687403,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 9.863958960047335e-05,
+      "loss": 0.11272231340408326,
+      "mean_token_accuracy": 0.9642089486122132,
+      "num_tokens": 648057.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.4378292948007583,
+      "epoch": 0.3234836702954899,
+      "grad_norm": 0.05078125,
+      "learning_rate": 9.847954724197757e-05,
+      "loss": 0.1520946979522705,
+      "mean_token_accuracy": 0.9551170215010643,
+      "num_tokens": 676272.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.4547880798578263,
+      "epoch": 0.3359253499222395,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 9.831075176610163e-05,
+      "loss": 0.13464858531951904,
+      "mean_token_accuracy": 0.9599997282028199,
+      "num_tokens": 702496.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.5038821995258331,
+      "epoch": 0.3483670295489891,
+      "grad_norm": 0.058349609375,
+      "learning_rate": 9.813323364933915e-05,
+      "loss": 0.13690755367279053,
+      "mean_token_accuracy": 0.9559143736958504,
+      "num_tokens": 726296.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.5725213587284088,
+      "epoch": 0.3608087091757387,
+      "grad_norm": 0.05615234375,
+      "learning_rate": 9.794702494308093e-05,
+      "loss": 0.14729396104812623,
+      "mean_token_accuracy": 0.9551976174116135,
+      "num_tokens": 752371.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.4945250004529953,
+      "epoch": 0.37325038880248834,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.775215926782788e-05,
+      "loss": 0.1206730842590332,
+      "mean_token_accuracy": 0.9687894850969314,
+      "num_tokens": 775896.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.4557270616292954,
+      "epoch": 0.3856920684292379,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 9.754867180712082e-05,
+      "loss": 0.1341879963874817,
+      "mean_token_accuracy": 0.9626043424010277,
+      "num_tokens": 798172.0,
+      "step": 155
+    },
+    {
+      "entropy": 1.4879502207040787,
+      "epoch": 0.39813374805598756,
+      "grad_norm": 0.05224609375,
+      "learning_rate": 9.733659930118793e-05,
+      "loss": 0.13069820404052734,
+      "mean_token_accuracy": 0.9548368141055107,
+      "num_tokens": 824452.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.5602718144655228,
+      "epoch": 0.4105754276827372,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 9.711598004031131e-05,
+      "loss": 0.13032547235488892,
+      "mean_token_accuracy": 0.9589100196957588,
+      "num_tokens": 849316.0,
+      "step": 165
+    },
+    {
+      "entropy": 1.488172698020935,
+      "epoch": 0.4230171073094868,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 9.688685385791339e-05,
+      "loss": 0.13182138204574584,
+      "mean_token_accuracy": 0.9608529090881348,
+      "num_tokens": 878357.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.4886444747447967,
+      "epoch": 0.4354587869362364,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 9.664926212336505e-05,
+      "loss": 0.1238294005393982,
+      "mean_token_accuracy": 0.960505859553814,
+      "num_tokens": 904192.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.5064076393842698,
+      "epoch": 0.447900466562986,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 9.640324773451622e-05,
+      "loss": 0.167707359790802,
+      "mean_token_accuracy": 0.9532883360981941,
+      "num_tokens": 930575.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.445479767024517,
+      "epoch": 0.4603421461897356,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 9.614885510995047e-05,
+      "loss": 0.12214440107345581,
+      "mean_token_accuracy": 0.9636711940169335,
+      "num_tokens": 954713.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.508093237876892,
+      "epoch": 0.4727838258164852,
+      "grad_norm": 0.052734375,
+      "learning_rate": 9.588613018096529e-05,
+      "loss": 0.13071714639663695,
+      "mean_token_accuracy": 0.9601568162441254,
+      "num_tokens": 981241.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.4727303981781006,
+      "epoch": 0.48522550544323484,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 9.561512038327882e-05,
+      "loss": 0.11298023462295533,
+      "mean_token_accuracy": 0.9675653219223023,
+      "num_tokens": 1007954.0,
+      "step": 195
+    },
+    {
+      "entropy": 1.463455791771412,
+      "epoch": 0.4976671850699845,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 9.533587464846548e-05,
+      "loss": 0.13673393726348876,
+      "mean_token_accuracy": 0.9588131874799728,
+      "num_tokens": 1037256.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.455343420803547,
+      "epoch": 0.5101088646967341,
+      "grad_norm": 0.060302734375,
+      "learning_rate": 9.504844339512095e-05,
+      "loss": 0.16279250383377075,
+      "mean_token_accuracy": 0.952251310646534,
+      "num_tokens": 1059784.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.512709778547287,
+      "epoch": 0.5225505443234837,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 9.475287851975925e-05,
+      "loss": 0.10544586181640625,
+      "mean_token_accuracy": 0.9677229434251785,
+      "num_tokens": 1082209.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.462220585346222,
+      "epoch": 0.5349922239502333,
+      "grad_norm": 0.0400390625,
+      "learning_rate": 9.444923338744239e-05,
+      "loss": 0.11979058980941773,
+      "mean_token_accuracy": 0.9626576647162437,
+      "num_tokens": 1110595.0,
+      "step": 215
+    },
+    {
+      "entropy": 1.4548005178570746,
+      "epoch": 0.5474339035769828,
+      "grad_norm": 0.04345703125,
+      "learning_rate": 9.413756282214537e-05,
+      "loss": 0.1187555432319641,
+      "mean_token_accuracy": 0.9617565467953682,
+      "num_tokens": 1135832.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.493788743019104,
+      "epoch": 0.5598755832037325,
+      "grad_norm": 0.06591796875,
+      "learning_rate": 9.381792309685746e-05,
+      "loss": 0.15725988149642944,
+      "mean_token_accuracy": 0.9544133111834526,
+      "num_tokens": 1163621.0,
+      "step": 225
+    },
+    {
+      "entropy": 1.4669800892472267,
+      "epoch": 0.5723172628304821,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 9.3490371923422e-05,
+      "loss": 0.12038854360580445,
+      "mean_token_accuracy": 0.9577056676149368,
+      "num_tokens": 1188345.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.5317447245121003,
+      "epoch": 0.5847589424572317,
+      "grad_norm": 0.05615234375,
+      "learning_rate": 9.31549684421163e-05,
+      "loss": 0.12890284061431884,
+      "mean_token_accuracy": 0.9609759420156478,
+      "num_tokens": 1215187.0,
+      "step": 235
+    },
+    {
+      "entropy": 1.502908742427826,
+      "epoch": 0.5972006220839814,
+      "grad_norm": 0.048828125,
+      "learning_rate": 9.28117732109738e-05,
+      "loss": 0.12224407196044922,
+      "mean_token_accuracy": 0.9638051286339759,
+      "num_tokens": 1241282.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.4564243242144586,
+      "epoch": 0.609642301710731,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 9.246084819485004e-05,
+      "loss": 0.13289834260940553,
+      "mean_token_accuracy": 0.9601684510707855,
+      "num_tokens": 1268251.0,
+      "step": 245
+    },
+    {
+      "entropy": 1.4659890502691268,
+      "epoch": 0.6220839813374806,
+      "grad_norm": 0.0595703125,
+      "learning_rate": 9.21022567542348e-05,
+      "loss": 0.14783012866973877,
+      "mean_token_accuracy": 0.9558995455503464,
+      "num_tokens": 1292958.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.4668870717287064,
+      "epoch": 0.6345256609642301,
+      "grad_norm": 0.057861328125,
+      "learning_rate": 9.173606363381219e-05,
+      "loss": 0.12078696489334106,
+      "mean_token_accuracy": 0.965465834736824,
+      "num_tokens": 1318689.0,
+      "step": 255
+    },
+    {
+      "entropy": 1.4572956532239913,
+      "epoch": 0.6469673405909798,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 9.136233495077077e-05,
+      "loss": 0.1338449239730835,
+      "mean_token_accuracy": 0.9598589226603508,
+      "num_tokens": 1345620.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.436805287003517,
+      "epoch": 0.6594090202177294,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 9.098113818286595e-05,
+      "loss": 0.1269620180130005,
+      "mean_token_accuracy": 0.963837206363678,
+      "num_tokens": 1369989.0,
+      "step": 265
+    },
+    {
+      "entropy": 1.430183658003807,
+      "epoch": 0.671850699844479,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 9.059254215623668e-05,
+      "loss": 0.09294362068176269,
+      "mean_token_accuracy": 0.9718702122569084,
+      "num_tokens": 1399090.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.4391476839780808,
+      "epoch": 0.6842923794712286,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 9.01966170329787e-05,
+      "loss": 0.11170387268066406,
+      "mean_token_accuracy": 0.9664130583405495,
+      "num_tokens": 1425851.0,
+      "step": 275
+    },
+    {
+      "entropy": 1.5080994993448258,
+      "epoch": 0.6967340590979783,
+      "grad_norm": 0.06005859375,
+      "learning_rate": 8.979343429847653e-05,
+      "loss": 0.1207460641860962,
+      "mean_token_accuracy": 0.96175517141819,
+      "num_tokens": 1450517.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.4918249040842055,
+      "epoch": 0.7091757387247278,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 8.93830667484967e-05,
+      "loss": 0.11671255826950074,
+      "mean_token_accuracy": 0.9663726434111595,
+      "num_tokens": 1474715.0,
+      "step": 285
+    },
+    {
+      "entropy": 1.4864625990390778,
+      "epoch": 0.7216174183514774,
+      "grad_norm": 0.044921875,
+      "learning_rate": 8.896558847604414e-05,
+      "loss": 0.12517218589782714,
+      "mean_token_accuracy": 0.9652746349573136,
+      "num_tokens": 1499442.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.490417730808258,
+      "epoch": 0.7340590979782271,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 8.854107485798458e-05,
+      "loss": 0.11757193803787232,
+      "mean_token_accuracy": 0.9664239585399628,
+      "num_tokens": 1523873.0,
+      "step": 295
+    },
+    {
+      "entropy": 1.4401051640510558,
+      "epoch": 0.7465007776049767,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 8.810960254143499e-05,
+      "loss": 0.13470059633255005,
+      "mean_token_accuracy": 0.9591680407524109,
+      "num_tokens": 1548575.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.5322502881288529,
+      "epoch": 0.7589424572317263,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 8.767124942992475e-05,
+      "loss": 0.12030593156814576,
+      "mean_token_accuracy": 0.9647545605897904,
+      "num_tokens": 1575643.0,
+      "step": 305
+    },
+    {
+      "entropy": 1.458931715786457,
+      "epoch": 0.7713841368584758,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 8.722609466932996e-05,
+      "loss": 0.09307305216789245,
+      "mean_token_accuracy": 0.9665900468826294,
+      "num_tokens": 1600333.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.458182653784752,
+      "epoch": 0.7838258164852255,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 8.677421863358341e-05,
+      "loss": 0.12296299934387207,
+      "mean_token_accuracy": 0.9640052378177643,
+      "num_tokens": 1625117.0,
+      "step": 315
+    },
+    {
+      "entropy": 1.4836479336023332,
+      "epoch": 0.7962674961119751,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 8.631570291016284e-05,
+      "loss": 0.12105728387832641,
+      "mean_token_accuracy": 0.9649034470319748,
+      "num_tokens": 1649929.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.5023894742131234,
+      "epoch": 0.8087091757387247,
+      "grad_norm": 0.044921875,
+      "learning_rate": 8.585063028536016e-05,
+      "loss": 0.10724372863769531,
+      "mean_token_accuracy": 0.965858943760395,
+      "num_tokens": 1675782.0,
+      "step": 325
+    },
+    {
+      "entropy": 1.4469050377607346,
+      "epoch": 0.8211508553654744,
+      "grad_norm": 0.052490234375,
+      "learning_rate": 8.537908472933402e-05,
+      "loss": 0.13695430755615234,
+      "mean_token_accuracy": 0.9561882585287094,
+      "num_tokens": 1703718.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.5462682336568832,
+      "epoch": 0.833592534992224,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 8.490115138094889e-05,
+      "loss": 0.14301273822784424,
+      "mean_token_accuracy": 0.9608543425798416,
+      "num_tokens": 1731614.0,
+      "step": 335
+    },
+    {
+      "entropy": 1.5303865879774095,
+      "epoch": 0.8460342146189735,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 8.4416916532403e-05,
+      "loss": 0.1558545470237732,
+      "mean_token_accuracy": 0.9530998334288597,
+      "num_tokens": 1758723.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.451111488044262,
+      "epoch": 0.8584758942457231,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 8.392646761364789e-05,
+      "loss": 0.10284805297851562,
+      "mean_token_accuracy": 0.9650755852460862,
+      "num_tokens": 1785721.0,
+      "step": 345
+    },
+    {
+      "entropy": 1.484094935655594,
+      "epoch": 0.8709175738724728,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 8.342989317660289e-05,
+      "loss": 0.10030006170272827,
+      "mean_token_accuracy": 0.9675216883420944,
+      "num_tokens": 1814257.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.5280012369155884,
+      "epoch": 0.8833592534992224,
+      "grad_norm": 0.0546875,
+      "learning_rate": 8.292728287916665e-05,
+      "loss": 0.09518902897834777,
+      "mean_token_accuracy": 0.9697460547089577,
+      "num_tokens": 1838719.0,
+      "step": 355
+    },
+    {
+      "entropy": 1.4606290251016616,
+      "epoch": 0.895800933125972,
+      "grad_norm": 0.05810546875,
+      "learning_rate": 8.241872746902935e-05,
+      "loss": 0.09836371541023255,
+      "mean_token_accuracy": 0.9680053278803825,
+      "num_tokens": 1860428.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.4977456852793694,
+      "epoch": 0.9082426127527217,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 8.190431876728772e-05,
+      "loss": 0.11404428482055665,
+      "mean_token_accuracy": 0.9683067321777343,
+      "num_tokens": 1887249.0,
+      "step": 365
+    },
+    {
+      "entropy": 1.4481744587421417,
+      "epoch": 0.9206842923794712,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 8.138414965186667e-05,
+      "loss": 0.11055492162704468,
+      "mean_token_accuracy": 0.9680417776107788,
+      "num_tokens": 1913193.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.4087932407855988,
+      "epoch": 0.9331259720062208,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 8.085831404074984e-05,
+      "loss": 0.08752046227455139,
+      "mean_token_accuracy": 0.9722029373049736,
+      "num_tokens": 1935245.0,
+      "step": 375
+    },
+    {
+      "entropy": 1.4568452775478362,
+      "epoch": 0.9455676516329704,
+      "grad_norm": 0.0576171875,
+      "learning_rate": 8.03269068750225e-05,
+      "loss": 0.11881341934204101,
+      "mean_token_accuracy": 0.959407502412796,
+      "num_tokens": 1961480.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.4280684739351273,
+      "epoch": 0.9580093312597201,
+      "grad_norm": 0.05078125,
+      "learning_rate": 7.979002410172954e-05,
+      "loss": 0.14049121141433715,
+      "mean_token_accuracy": 0.9576024219393731,
+      "num_tokens": 1986643.0,
+      "step": 385
+    },
+    {
+      "entropy": 1.4764668315649032,
+      "epoch": 0.9704510108864697,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.924776265655216e-05,
+      "loss": 0.11259770393371582,
+      "mean_token_accuracy": 0.9635465934872627,
+      "num_tokens": 2012509.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.4508350789546967,
+      "epoch": 0.9828926905132193,
+      "grad_norm": 0.0361328125,
+      "learning_rate": 7.870022044630569e-05,
+      "loss": 0.11488029956817628,
+      "mean_token_accuracy": 0.9670460253953934,
+      "num_tokens": 2039717.0,
+      "step": 395
+    },
+    {
+      "entropy": 1.5077543556690216,
+      "epoch": 0.995334370139969,
+      "grad_norm": 0.06201171875,
+      "learning_rate": 7.814749633126235e-05,
+      "loss": 0.1424098014831543,
+      "mean_token_accuracy": 0.9543160945177078,
+      "num_tokens": 2064948.0,
+      "step": 400
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.4550913160425485,
+      "eval_loss": 0.12082157284021378,
+      "eval_mean_token_accuracy": 0.9625284673763668,
+      "eval_num_tokens": 2076268.0,
+      "eval_runtime": 24.7531,
+      "eval_samples_per_second": 12.16,
+      "eval_steps_per_second": 12.16,
+      "step": 402
+    },
+    {
+      "entropy": 1.4730579455693562,
+      "epoch": 1.0074650077760499,
+      "grad_norm": 0.0322265625,
+      "learning_rate": 7.75896901073017e-05,
+      "loss": 0.08911717534065247,
+      "mean_token_accuracy": 0.9732377681976709,
+      "num_tokens": 2088687.0,
+      "step": 405
+    },
+    {
+      "entropy": 1.436254122853279,
+      "epoch": 1.0199066874027993,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 7.702690248789223e-05,
+      "loss": 0.10224579572677613,
+      "mean_token_accuracy": 0.9676861181855202,
+      "num_tokens": 2114020.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.4225786730647088,
+      "epoch": 1.032348367029549,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 7.645923508590726e-05,
+      "loss": 0.09414710998535156,
+      "mean_token_accuracy": 0.9725460812449456,
+      "num_tokens": 2143268.0,
+      "step": 415
+    },
+    {
+      "entropy": 1.4235260367393494,
+      "epoch": 1.0447900466562987,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.588679039527848e-05,
+      "loss": 0.0781755805015564,
+      "mean_token_accuracy": 0.9752240985631943,
+      "num_tokens": 2168154.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.3582488745450974,
+      "epoch": 1.0572317262830482,
+      "grad_norm": 0.03564453125,
+      "learning_rate": 7.530967177249036e-05,
+      "loss": 0.06534834504127503,
+      "mean_token_accuracy": 0.9809993267059326,
+      "num_tokens": 2195567.0,
+      "step": 425
+    },
+    {
+      "entropy": 1.4120464980602265,
+      "epoch": 1.0696734059097979,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 7.472798341791877e-05,
+      "loss": 0.08980979323387146,
+      "mean_token_accuracy": 0.9729388952255249,
+      "num_tokens": 2220651.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.3695552781224252,
+      "epoch": 1.0821150855365473,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 7.414183035701749e-05,
+      "loss": 0.07374750971794128,
+      "mean_token_accuracy": 0.9773057013750076,
+      "num_tokens": 2244016.0,
+      "step": 435
+    },
+    {
+      "entropy": 1.361125336587429,
+      "epoch": 1.094556765163297,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.355131842135543e-05,
+      "loss": 0.09960444569587708,
+      "mean_token_accuracy": 0.9707440197467804,
+      "num_tokens": 2269688.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.4232443049550056,
+      "epoch": 1.1069984447900467,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.295655422950849e-05,
+      "loss": 0.08172831535339356,
+      "mean_token_accuracy": 0.9749625563621521,
+      "num_tokens": 2294075.0,
+      "step": 445
+    },
+    {
+      "entropy": 1.4120786994695664,
+      "epoch": 1.1194401244167962,
+      "grad_norm": 0.0341796875,
+      "learning_rate": 7.235764516780927e-05,
+      "loss": 0.06837162375450134,
+      "mean_token_accuracy": 0.9771177500486374,
+      "num_tokens": 2319276.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.4431693792343139,
+      "epoch": 1.1318818040435459,
+      "grad_norm": 0.044677734375,
+      "learning_rate": 7.175469937095818e-05,
+      "loss": 0.08680518865585327,
+      "mean_token_accuracy": 0.9729270681738853,
+      "num_tokens": 2347219.0,
+      "step": 455
+    },
+    {
+      "entropy": 1.4526385068893433,
+      "epoch": 1.1443234836702956,
+      "grad_norm": 0.10302734375,
+      "learning_rate": 7.114782570249943e-05,
+      "loss": 0.09112498164176941,
+      "mean_token_accuracy": 0.973096051812172,
+      "num_tokens": 2372584.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.3945886969566346,
+      "epoch": 1.156765163297045,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 7.053713373516538e-05,
+      "loss": 0.07018510103225709,
+      "mean_token_accuracy": 0.9799379587173462,
+      "num_tokens": 2398752.0,
+      "step": 465
+    },
+    {
+      "entropy": 1.3700096011161804,
+      "epoch": 1.1692068429237947,
+      "grad_norm": 0.055908203125,
+      "learning_rate": 6.992273373109288e-05,
+      "loss": 0.06986131072044373,
+      "mean_token_accuracy": 0.9791282191872597,
+      "num_tokens": 2421888.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.3979035422205925,
+      "epoch": 1.1816485225505442,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 6.930473662191517e-05,
+      "loss": 0.09055956602096557,
+      "mean_token_accuracy": 0.9743469759821892,
+      "num_tokens": 2448705.0,
+      "step": 475
+    },
+    {
+      "entropy": 1.3682101637125015,
+      "epoch": 1.194090202177294,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 6.868325398873284e-05,
+      "loss": 0.07687475085258484,
+      "mean_token_accuracy": 0.9755479469895363,
+      "num_tokens": 2474561.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.36375695168972,
+      "epoch": 1.2065318818040436,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 6.805839804196757e-05,
+      "loss": 0.09423953294754028,
+      "mean_token_accuracy": 0.9717281922698021,
+      "num_tokens": 2501620.0,
+      "step": 485
+    },
+    {
+      "entropy": 1.3528606280684472,
+      "epoch": 1.218973561430793,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 6.743028160110215e-05,
+      "loss": 0.07231994867324829,
+      "mean_token_accuracy": 0.976399302482605,
+      "num_tokens": 2527481.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.4269449830055236,
+      "epoch": 1.2314152410575427,
+      "grad_norm": 0.061279296875,
+      "learning_rate": 6.679901807431073e-05,
+      "loss": 0.0847695529460907,
+      "mean_token_accuracy": 0.9742431849241256,
+      "num_tokens": 2553527.0,
+      "step": 495
+    },
+    {
+      "entropy": 1.4285764783620833,
+      "epoch": 1.2438569206842924,
+      "grad_norm": 0.044677734375,
+      "learning_rate": 6.616472143798261e-05,
+      "loss": 0.07603438496589661,
+      "mean_token_accuracy": 0.9767945379018783,
+      "num_tokens": 2580242.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.4113897368311883,
+      "epoch": 1.256298600311042,
+      "grad_norm": 0.05078125,
+      "learning_rate": 6.552750621614337e-05,
+      "loss": 0.09946245551109315,
+      "mean_token_accuracy": 0.9701174065470696,
+      "num_tokens": 2604019.0,
+      "step": 505
+    },
+    {
+      "entropy": 1.2941894769668578,
+      "epoch": 1.2687402799377916,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 6.488748745977746e-05,
+      "loss": 0.0612099289894104,
+      "mean_token_accuracy": 0.981476029753685,
+      "num_tokens": 2628945.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.4310174494981767,
+      "epoch": 1.2811819595645413,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 6.424478072605522e-05,
+      "loss": 0.06989500522613526,
+      "mean_token_accuracy": 0.9810906872153282,
+      "num_tokens": 2652720.0,
+      "step": 515
+    },
+    {
+      "entropy": 1.3148574769496917,
+      "epoch": 1.2936236391912908,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 6.359950205746881e-05,
+      "loss": 0.07661423683166504,
+      "mean_token_accuracy": 0.9741629838943482,
+      "num_tokens": 2677256.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.3354602426290512,
+      "epoch": 1.3060653188180404,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 6.295176796088045e-05,
+      "loss": 0.0708840012550354,
+      "mean_token_accuracy": 0.9720481187105179,
+      "num_tokens": 2705462.0,
+      "step": 525
+    },
+    {
+      "entropy": 1.3693901777267456,
+      "epoch": 1.3185069984447901,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 6.230169538648667e-05,
+      "loss": 0.06346817016601562,
+      "mean_token_accuracy": 0.9793296322226525,
+      "num_tokens": 2731465.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.3165668994188309,
+      "epoch": 1.3309486780715396,
+      "grad_norm": 0.0625,
+      "learning_rate": 6.164940170670266e-05,
+      "loss": 0.08328393697738648,
+      "mean_token_accuracy": 0.971780526638031,
+      "num_tokens": 2759478.0,
+      "step": 535
+    },
+    {
+      "entropy": 1.34098874181509,
+      "epoch": 1.3433903576982893,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 6.09950046949705e-05,
+      "loss": 0.08074995279312133,
+      "mean_token_accuracy": 0.9730972275137901,
+      "num_tokens": 2783456.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.385405807197094,
+      "epoch": 1.355832037325039,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 6.0338622504494604e-05,
+      "loss": 0.06526267528533936,
+      "mean_token_accuracy": 0.9802431628108025,
+      "num_tokens": 2808812.0,
+      "step": 545
+    },
+    {
+      "entropy": 1.393546462059021,
+      "epoch": 1.3682737169517885,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 5.968037364690897e-05,
+      "loss": 0.08440290689468384,
+      "mean_token_accuracy": 0.9747256502509117,
+      "num_tokens": 2837088.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.4094155162572861,
+      "epoch": 1.3807153965785381,
+      "grad_norm": 0.058837890625,
+      "learning_rate": 5.902037697087957e-05,
+      "loss": 0.07816668152809143,
+      "mean_token_accuracy": 0.9740476593375206,
+      "num_tokens": 2862524.0,
+      "step": 555
+    },
+    {
+      "entropy": 1.350576937198639,
+      "epoch": 1.3931570762052878,
+      "grad_norm": 0.055419921875,
+      "learning_rate": 5.8358751640645835e-05,
+      "loss": 0.06395477056503296,
+      "mean_token_accuracy": 0.9800668403506279,
+      "num_tokens": 2884137.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.403828603029251,
+      "epoch": 1.4055987558320373,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 5.769561711450527e-05,
+      "loss": 0.0748115599155426,
+      "mean_token_accuracy": 0.9773760348558426,
+      "num_tokens": 2906231.0,
+      "step": 565
+    },
+    {
+      "entropy": 1.3695515617728233,
+      "epoch": 1.418040435458787,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 5.7031093123244925e-05,
+      "loss": 0.08527232408523559,
+      "mean_token_accuracy": 0.9715872302651405,
+      "num_tokens": 2932820.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.4267593204975129,
+      "epoch": 1.4304821150855365,
+      "grad_norm": 0.05859375,
+      "learning_rate": 5.6365299648523605e-05,
+      "loss": 0.10021589994430542,
+      "mean_token_accuracy": 0.9709113940596581,
+      "num_tokens": 2956192.0,
+      "step": 575
+    },
+    {
+      "entropy": 1.3429571866989136,
+      "epoch": 1.4429237947122862,
+      "grad_norm": 0.06005859375,
+      "learning_rate": 5.5698356901208925e-05,
+      "loss": 0.107265305519104,
+      "mean_token_accuracy": 0.9646854192018509,
+      "num_tokens": 2980056.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.4046858251094818,
+      "epoch": 1.4553654743390358,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 5.5030385299672884e-05,
+      "loss": 0.08158871531486511,
+      "mean_token_accuracy": 0.9777654156088829,
+      "num_tokens": 3004045.0,
+      "step": 585
+    },
+    {
+      "entropy": 1.4346278131008148,
+      "epoch": 1.4678071539657853,
+      "grad_norm": 0.06201171875,
+      "learning_rate": 5.436150544804994e-05,
+      "loss": 0.07914494276046753,
+      "mean_token_accuracy": 0.9757034927606583,
+      "num_tokens": 3027916.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.3918647408485412,
+      "epoch": 1.480248833592535,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 5.3691838114461626e-05,
+      "loss": 0.09960976839065552,
+      "mean_token_accuracy": 0.9678563356399537,
+      "num_tokens": 3053663.0,
+      "step": 595
+    },
+    {
+      "entropy": 1.3449461698532104,
+      "epoch": 1.4926905132192845,
+      "grad_norm": 0.06982421875,
+      "learning_rate": 5.302150420921161e-05,
+      "loss": 0.08556955456733703,
+      "mean_token_accuracy": 0.9714176893234253,
+      "num_tokens": 3080272.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.3727866768836976,
+      "epoch": 1.5051321928460342,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 5.2350624762954884e-05,
+      "loss": 0.0717516303062439,
+      "mean_token_accuracy": 0.9767438977956772,
+      "num_tokens": 3107201.0,
+      "step": 605
+    },
+    {
+      "entropy": 1.3760264009237289,
+      "epoch": 1.5175738724727839,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 5.1679320904845365e-05,
+      "loss": 0.08696765303611756,
+      "mean_token_accuracy": 0.9729200676083565,
+      "num_tokens": 3135290.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.3937256038188934,
+      "epoch": 1.5300155520995333,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 5.100771384066575e-05,
+      "loss": 0.06904829740524292,
+      "mean_token_accuracy": 0.9765424266457557,
+      "num_tokens": 3161468.0,
+      "step": 615
+    },
+    {
+      "entropy": 1.3464481949806213,
+      "epoch": 1.542457231726283,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 5.033592483094344e-05,
+      "loss": 0.08148046731948852,
+      "mean_token_accuracy": 0.9747312605381012,
+      "num_tokens": 3187972.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.3713692784309388,
+      "epoch": 1.5548989113530327,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 4.966407516905657e-05,
+      "loss": 0.09401994347572326,
+      "mean_token_accuracy": 0.9700708135962486,
+      "num_tokens": 3218206.0,
+      "step": 625
+    },
+    {
+      "entropy": 1.4130443289875985,
+      "epoch": 1.5673405909797822,
+      "grad_norm": 0.05517578125,
+      "learning_rate": 4.899228615933427e-05,
+      "loss": 0.102409827709198,
+      "mean_token_accuracy": 0.9645006597042084,
+      "num_tokens": 3243182.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.312636035680771,
+      "epoch": 1.5797822706065319,
+      "grad_norm": 0.05078125,
+      "learning_rate": 4.832067909515467e-05,
+      "loss": 0.07581937909126282,
+      "mean_token_accuracy": 0.977123475074768,
+      "num_tokens": 3270355.0,
+      "step": 635
+    },
+    {
+      "entropy": 1.2894637912511826,
+      "epoch": 1.5922239502332816,
+      "grad_norm": 0.0546875,
+      "learning_rate": 4.7649375237045135e-05,
+      "loss": 0.0760753870010376,
+      "mean_token_accuracy": 0.9742821797728538,
+      "num_tokens": 3295761.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.3615785390138626,
+      "epoch": 1.604665629860031,
+      "grad_norm": 0.052001953125,
+      "learning_rate": 4.697849579078838e-05,
+      "loss": 0.08204466700553895,
+      "mean_token_accuracy": 0.9755400061607361,
+      "num_tokens": 3322763.0,
+      "step": 645
+    },
+    {
+      "entropy": 1.4578467637300492,
+      "epoch": 1.6171073094867807,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 4.630816188553837e-05,
+      "loss": 0.10771067142486572,
+      "mean_token_accuracy": 0.9648271799087524,
+      "num_tokens": 3347850.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.3541809171438217,
+      "epoch": 1.6295489891135304,
+      "grad_norm": 0.044921875,
+      "learning_rate": 4.563849455195007e-05,
+      "loss": 0.08120843768119812,
+      "mean_token_accuracy": 0.9748451858758926,
+      "num_tokens": 3378295.0,
+      "step": 655
+    },
+    {
+      "entropy": 1.3386278599500656,
+      "epoch": 1.6419906687402799,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 4.496961470032713e-05,
+      "loss": 0.07182853221893311,
+      "mean_token_accuracy": 0.9755081802606582,
+      "num_tokens": 3407238.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.3182463720440865,
+      "epoch": 1.6544323483670296,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 4.4301643098791086e-05,
+      "loss": 0.0790963888168335,
+      "mean_token_accuracy": 0.9719948589801788,
+      "num_tokens": 3430188.0,
+      "step": 665
+    },
+    {
+      "entropy": 1.3785580173134804,
+      "epoch": 1.6668740279937793,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 4.363470035147641e-05,
+      "loss": 0.0732508659362793,
+      "mean_token_accuracy": 0.9752413406968117,
+      "num_tokens": 3454755.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.392490154504776,
+      "epoch": 1.6793157076205287,
+      "grad_norm": 0.041015625,
+      "learning_rate": 4.29689068767551e-05,
+      "loss": 0.07593491673469543,
+      "mean_token_accuracy": 0.9777896001935005,
+      "num_tokens": 3481020.0,
+      "step": 675
+    },
+    {
+      "entropy": 1.333091938495636,
+      "epoch": 1.6917573872472784,
+      "grad_norm": 0.052734375,
+      "learning_rate": 4.2304382885494753e-05,
+      "loss": 0.0870925784111023,
+      "mean_token_accuracy": 0.9741413682699204,
+      "num_tokens": 3507708.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.320798073709011,
+      "epoch": 1.7041990668740281,
+      "grad_norm": 0.04833984375,
+      "learning_rate": 4.164124835935418e-05,
+      "loss": 0.0877147912979126,
+      "mean_token_accuracy": 0.9753073915839195,
+      "num_tokens": 3534100.0,
+      "step": 685
+    },
+    {
+      "entropy": 1.3160912215709686,
+      "epoch": 1.7166407465007776,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 4.0979623029120436e-05,
+      "loss": 0.08963274955749512,
+      "mean_token_accuracy": 0.9698545068502427,
+      "num_tokens": 3558716.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.3380629509687423,
+      "epoch": 1.729082426127527,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 4.031962635309104e-05,
+      "loss": 0.07874792218208312,
+      "mean_token_accuracy": 0.9740808755159378,
+      "num_tokens": 3586065.0,
+      "step": 695
+    },
+    {
+      "entropy": 1.3654002964496612,
+      "epoch": 1.741524105754277,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 3.96613774955054e-05,
+      "loss": 0.08268317580223083,
+      "mean_token_accuracy": 0.9741670668125153,
+      "num_tokens": 3613513.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.3774547845125198,
+      "epoch": 1.7539657853810264,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 3.900499530502951e-05,
+      "loss": 0.08645263314247131,
+      "mean_token_accuracy": 0.9758756011724472,
+      "num_tokens": 3640806.0,
+      "step": 705
+    },
+    {
+      "entropy": 1.3115302637219428,
+      "epoch": 1.766407465007776,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 3.835059829329735e-05,
+      "loss": 0.07381963729858398,
+      "mean_token_accuracy": 0.9768764480948449,
+      "num_tokens": 3665922.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.3822921440005302,
+      "epoch": 1.7788491446345258,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 3.769830461351336e-05,
+      "loss": 0.09703359007835388,
+      "mean_token_accuracy": 0.9744590654969215,
+      "num_tokens": 3694612.0,
+      "step": 715
+    },
+    {
+      "entropy": 1.3549542352557182,
+      "epoch": 1.7912908242612753,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 3.704823203911957e-05,
+      "loss": 0.09285060167312623,
+      "mean_token_accuracy": 0.9735767096281052,
+      "num_tokens": 3718319.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.3544731348752976,
+      "epoch": 1.8037325038880248,
+      "grad_norm": 0.07421875,
+      "learning_rate": 3.6400497942531186e-05,
+      "loss": 0.07869815826416016,
+      "mean_token_accuracy": 0.977102306485176,
+      "num_tokens": 3742125.0,
+      "step": 725
+    },
+    {
+      "entropy": 1.3644321769475938,
+      "epoch": 1.8161741835147744,
+      "grad_norm": 0.0546875,
+      "learning_rate": 3.57552192739448e-05,
+      "loss": 0.08890414834022523,
+      "mean_token_accuracy": 0.973346334695816,
+      "num_tokens": 3765739.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.3641284555196762,
+      "epoch": 1.8286158631415241,
+      "grad_norm": 0.045654296875,
+      "learning_rate": 3.511251254022255e-05,
+      "loss": 0.11303408145904541,
+      "mean_token_accuracy": 0.9671000927686692,
+      "num_tokens": 3790908.0,
+      "step": 735
+    },
+    {
+      "entropy": 1.352848869562149,
+      "epoch": 1.8410575427682736,
+      "grad_norm": 0.046875,
+      "learning_rate": 3.447249378385663e-05,
+      "loss": 0.07621319890022278,
+      "mean_token_accuracy": 0.9736784845590591,
+      "num_tokens": 3817021.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.3620782911777496,
+      "epoch": 1.8534992223950233,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 3.38352785620174e-05,
+      "loss": 0.07505900859832763,
+      "mean_token_accuracy": 0.9756825506687165,
+      "num_tokens": 3845607.0,
+      "step": 745
+    },
+    {
+      "entropy": 1.387294201552868,
+      "epoch": 1.865940902021773,
+      "grad_norm": 0.040771484375,
+      "learning_rate": 3.3200981925689264e-05,
+      "loss": 0.07677844166755676,
+      "mean_token_accuracy": 0.9746143981814385,
+      "num_tokens": 3871956.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.3148253604769706,
+      "epoch": 1.8783825816485225,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 3.256971839889787e-05,
+      "loss": 0.062203264236450194,
+      "mean_token_accuracy": 0.9806876540184021,
+      "num_tokens": 3896461.0,
+      "step": 755
+    },
+    {
+      "entropy": 1.297327271103859,
+      "epoch": 1.8908242612752721,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 3.194160195803245e-05,
+      "loss": 0.07807562947273254,
+      "mean_token_accuracy": 0.9751886636018753,
+      "num_tokens": 3924652.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.2678332865238189,
+      "epoch": 1.9032659409020218,
+      "grad_norm": 0.04833984375,
+      "learning_rate": 3.1316746011267155e-05,
+      "loss": 0.0622839629650116,
+      "mean_token_accuracy": 0.980354443192482,
+      "num_tokens": 3950947.0,
+      "step": 765
+    },
+    {
+      "entropy": 1.3001187965273857,
+      "epoch": 1.9157076205287713,
+      "grad_norm": 0.05322265625,
+      "learning_rate": 3.0695263378084834e-05,
+      "loss": 0.07793177962303162,
+      "mean_token_accuracy": 0.9762747883796692,
+      "num_tokens": 3977231.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.343767161667347,
+      "epoch": 1.928149300155521,
+      "grad_norm": 0.058349609375,
+      "learning_rate": 3.0077266268907124e-05,
+      "loss": 0.09248242378234864,
+      "mean_token_accuracy": 0.9737435877323151,
+      "num_tokens": 4003918.0,
+      "step": 775
+    },
+    {
+      "entropy": 1.2801260307431221,
+      "epoch": 1.9405909797822707,
+      "grad_norm": 0.0673828125,
+      "learning_rate": 2.946286626483463e-05,
+      "loss": 0.07414901256561279,
+      "mean_token_accuracy": 0.978515362739563,
+      "num_tokens": 4027613.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.3166591823101044,
+      "epoch": 1.9530326594090202,
+      "grad_norm": 0.0634765625,
+      "learning_rate": 2.8852174297500567e-05,
+      "loss": 0.0782233715057373,
+      "mean_token_accuracy": 0.9734146118164062,
+      "num_tokens": 4053231.0,
+      "step": 785
+    },
+    {
+      "entropy": 1.2959586173295974,
+      "epoch": 1.9654743390357698,
+      "grad_norm": 0.05078125,
+      "learning_rate": 2.8245300629041838e-05,
+      "loss": 0.08309885859489441,
+      "mean_token_accuracy": 0.9741686061024666,
+      "num_tokens": 4081234.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.362495419383049,
+      "epoch": 1.9779160186625195,
+      "grad_norm": 0.047119140625,
+      "learning_rate": 2.7642354832190747e-05,
+      "loss": 0.09300965070724487,
+      "mean_token_accuracy": 0.9714553162455559,
+      "num_tokens": 4106601.0,
+      "step": 795
+    },
+    {
+      "entropy": 1.324322985112667,
+      "epoch": 1.990357698289269,
+      "grad_norm": 0.05078125,
+      "learning_rate": 2.7043445770491526e-05,
+      "loss": 0.0799511969089508,
+      "mean_token_accuracy": 0.9754261270165443,
+      "num_tokens": 4133337.0,
+      "step": 800
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 1.3053412522588457,
+      "eval_loss": 0.10880886763334274,
+      "eval_mean_token_accuracy": 0.9662994710709961,
+      "eval_num_tokens": 4152536.0,
+      "eval_runtime": 24.7392,
+      "eval_samples_per_second": 12.167,
+      "eval_steps_per_second": 12.167,
+      "step": 804
+    },
+    {
+      "entropy": 1.3139420319826176,
+      "epoch": 2.00248833592535,
+      "grad_norm": 0.0311279296875,
+      "learning_rate": 2.644868157864458e-05,
+      "loss": 0.07132983803749085,
+      "mean_token_accuracy": 0.9790512231680063,
+      "num_tokens": 4157848.0,
+      "step": 805
+    },
+    {
+      "entropy": 1.3346404328942298,
+      "epoch": 2.0149300155520997,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 2.5858169642982515e-05,
+      "loss": 0.06443278789520264,
+      "mean_token_accuracy": 0.9790809944272041,
+      "num_tokens": 4183018.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.3496920108795165,
+      "epoch": 2.027371695178849,
+      "grad_norm": 0.046142578125,
+      "learning_rate": 2.5272016582081236e-05,
+      "loss": 0.046033966541290286,
+      "mean_token_accuracy": 0.9862025916576386,
+      "num_tokens": 4207722.0,
+      "step": 815
+    },
+    {
+      "entropy": 1.2798842176795007,
+      "epoch": 2.0398133748055987,
+      "grad_norm": 0.033203125,
+      "learning_rate": 2.4690328227509668e-05,
+      "loss": 0.04474189877510071,
+      "mean_token_accuracy": 0.9851935356855392,
+      "num_tokens": 4233615.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.292403693497181,
+      "epoch": 2.0522550544323486,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 2.411320960472152e-05,
+      "loss": 0.05720994472503662,
+      "mean_token_accuracy": 0.9835459992289544,
+      "num_tokens": 4260265.0,
+      "step": 825
+    },
+    {
+      "entropy": 1.2822298243641854,
+      "epoch": 2.064696734059098,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 2.354076491409274e-05,
+      "loss": 0.06071372032165527,
+      "mean_token_accuracy": 0.9809239432215691,
+      "num_tokens": 4287688.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.3030796691775322,
+      "epoch": 2.0771384136858475,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 2.2973097512107796e-05,
+      "loss": 0.04943784773349762,
+      "mean_token_accuracy": 0.9850927606225014,
+      "num_tokens": 4311396.0,
+      "step": 835
+    },
+    {
+      "entropy": 1.2815598845481873,
+      "epoch": 2.0895800933125974,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 2.2410309892698322e-05,
+      "loss": 0.06365938782691956,
+      "mean_token_accuracy": 0.9812412708997726,
+      "num_tokens": 4338280.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.2353304088115693,
+      "epoch": 2.102021772939347,
+      "grad_norm": 0.0228271484375,
+      "learning_rate": 2.185250366873765e-05,
+      "loss": 0.03775741457939148,
+      "mean_token_accuracy": 0.988109527528286,
+      "num_tokens": 4361954.0,
+      "step": 845
+    },
+    {
+      "entropy": 1.3121544525027276,
+      "epoch": 2.1144634525660964,
+      "grad_norm": 0.052734375,
+      "learning_rate": 2.1299779553694323e-05,
+      "loss": 0.06021370887756348,
+      "mean_token_accuracy": 0.9813222840428353,
+      "num_tokens": 4389868.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.2844686791300775,
+      "epoch": 2.126905132192846,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 2.075223734344785e-05,
+      "loss": 0.043541711568832395,
+      "mean_token_accuracy": 0.9842599242925644,
+      "num_tokens": 4415143.0,
+      "step": 855
+    },
+    {
+      "entropy": 1.2730493247509003,
+      "epoch": 2.1393468118195957,
+      "grad_norm": 0.051025390625,
+      "learning_rate": 2.0209975898270454e-05,
+      "loss": 0.060882335901260375,
+      "mean_token_accuracy": 0.9831152334809303,
+      "num_tokens": 4439690.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.3221701458096504,
+      "epoch": 2.151788491446345,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 1.967309312497752e-05,
+      "loss": 0.06385582089424133,
+      "mean_token_accuracy": 0.980524268746376,
+      "num_tokens": 4468649.0,
+      "step": 865
+    },
+    {
+      "entropy": 1.308930653333664,
+      "epoch": 2.1642301710730947,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 1.914168595925016e-05,
+      "loss": 0.05137070417404175,
+      "mean_token_accuracy": 0.9829948961734771,
+      "num_tokens": 4494900.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.278279796242714,
+      "epoch": 2.1766718506998446,
+      "grad_norm": 0.044921875,
+      "learning_rate": 1.861585034813334e-05,
+      "loss": 0.053401678800582886,
+      "mean_token_accuracy": 0.9820586636662483,
+      "num_tokens": 4521907.0,
+      "step": 875
+    },
+    {
+      "entropy": 1.2940535843372345,
+      "epoch": 2.189113530326594,
+      "grad_norm": 0.033935546875,
+      "learning_rate": 1.809568123271231e-05,
+      "loss": 0.05988807678222656,
+      "mean_token_accuracy": 0.9810460910201073,
+      "num_tokens": 4546991.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.3210156977176666,
+      "epoch": 2.2015552099533435,
+      "grad_norm": 0.03515625,
+      "learning_rate": 1.7581272530970667e-05,
+      "loss": 0.05812197327613831,
+      "mean_token_accuracy": 0.9826369822025299,
+      "num_tokens": 4573645.0,
+      "step": 885
+    },
+    {
+      "entropy": 1.2087926417589188,
+      "epoch": 2.2139968895800934,
+      "grad_norm": 0.03564453125,
+      "learning_rate": 1.707271712083335e-05,
+      "loss": 0.059379422664642335,
+      "mean_token_accuracy": 0.9827022448182106,
+      "num_tokens": 4598343.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.2919662535190581,
+      "epoch": 2.226438569206843,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 1.6570106823397142e-05,
+      "loss": 0.05405360460281372,
+      "mean_token_accuracy": 0.9803063213825226,
+      "num_tokens": 4625264.0,
+      "step": 895
+    },
+    {
+      "entropy": 1.2397317215800285,
+      "epoch": 2.2388802488335924,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 1.607353238635212e-05,
+      "loss": 0.05695806741714478,
+      "mean_token_accuracy": 0.9809326454997063,
+      "num_tokens": 4655699.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.2710261449217797,
+      "epoch": 2.2513219284603423,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 1.558308346759701e-05,
+      "loss": 0.05667018294334412,
+      "mean_token_accuracy": 0.9820555970072746,
+      "num_tokens": 4682556.0,
+      "step": 905
+    },
+    {
+      "entropy": 1.3200107291340828,
+      "epoch": 2.2637636080870918,
+      "grad_norm": 0.0712890625,
+      "learning_rate": 1.5098848619051109e-05,
+      "loss": 0.058824723958969115,
+      "mean_token_accuracy": 0.9803817018866539,
+      "num_tokens": 4703229.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.3161411359906197,
+      "epoch": 2.2762052877138412,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 1.4620915270665992e-05,
+      "loss": 0.05733857154846191,
+      "mean_token_accuracy": 0.9827383399009705,
+      "num_tokens": 4732396.0,
+      "step": 915
+    },
+    {
+      "entropy": 1.27952651232481,
+      "epoch": 2.288646967340591,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 1.4149369714639853e-05,
+      "loss": 0.0526383638381958,
+      "mean_token_accuracy": 0.9817092105746269,
+      "num_tokens": 4758145.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.2732232689857483,
+      "epoch": 2.3010886469673406,
+      "grad_norm": 0.0281982421875,
+      "learning_rate": 1.3684297089837155e-05,
+      "loss": 0.055611425638198854,
+      "mean_token_accuracy": 0.982058934867382,
+      "num_tokens": 4784880.0,
+      "step": 925
+    },
+    {
+      "entropy": 1.2462307259440422,
+      "epoch": 2.31353032659409,
+      "grad_norm": 0.0498046875,
+      "learning_rate": 1.3225781366416595e-05,
+      "loss": 0.06674134135246276,
+      "mean_token_accuracy": 0.9809619382023811,
+      "num_tokens": 4814123.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.3400009661912917,
+      "epoch": 2.32597200622084,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 1.2773905330670038e-05,
+      "loss": 0.059016406536102295,
+      "mean_token_accuracy": 0.9813900917768479,
+      "num_tokens": 4838037.0,
+      "step": 935
+    },
+    {
+      "entropy": 1.2777298644185067,
+      "epoch": 2.3384136858475895,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 1.2328750570075265e-05,
+      "loss": 0.04491693377494812,
+      "mean_token_accuracy": 0.9859692931175232,
+      "num_tokens": 4865586.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.284477137029171,
+      "epoch": 2.350855365474339,
+      "grad_norm": 0.07763671875,
+      "learning_rate": 1.1890397458565022e-05,
+      "loss": 0.06391626596450806,
+      "mean_token_accuracy": 0.9819939568638801,
+      "num_tokens": 4889503.0,
+      "step": 945
+    },
+    {
+      "entropy": 1.2908324316143989,
+      "epoch": 2.3632970451010884,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 1.1458925142015431e-05,
+      "loss": 0.06202779412269592,
+      "mean_token_accuracy": 0.9813487082719803,
+      "num_tokens": 4918754.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.277482558786869,
+      "epoch": 2.3757387247278383,
+      "grad_norm": 0.031982421875,
+      "learning_rate": 1.103441152395588e-05,
+      "loss": 0.06154012680053711,
+      "mean_token_accuracy": 0.9820781499147415,
+      "num_tokens": 4945316.0,
+      "step": 955
+    },
+    {
+      "entropy": 1.2095389053225518,
+      "epoch": 2.388180404354588,
+      "grad_norm": 0.041748046875,
+      "learning_rate": 1.061693325150332e-05,
+      "loss": 0.06242812871932983,
+      "mean_token_accuracy": 0.9817898482084274,
+      "num_tokens": 4974069.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2972484081983566,
+      "epoch": 2.4006220839813377,
+      "grad_norm": 0.046875,
+      "learning_rate": 1.0206565701523479e-05,
+      "loss": 0.05046294927597046,
+      "mean_token_accuracy": 0.9853808850049972,
+      "num_tokens": 4999019.0,
+      "step": 965
+    },
+    {
+      "entropy": 1.3197123885154725,
+      "epoch": 2.413063763608087,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 9.803382967021318e-06,
+      "loss": 0.049368086457252505,
+      "mean_token_accuracy": 0.9852947980165482,
+      "num_tokens": 5023412.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.3319877043366433,
+      "epoch": 2.4255054432348366,
+      "grad_norm": 0.0693359375,
+      "learning_rate": 9.407457843763324e-06,
+      "loss": 0.0752966582775116,
+      "mean_token_accuracy": 0.9780165493488312,
+      "num_tokens": 5050267.0,
+      "step": 975
+    },
+    {
+      "entropy": 1.3291449189186095,
+      "epoch": 2.437947122861586,
+      "grad_norm": 0.056396484375,
+      "learning_rate": 9.01886181713405e-06,
+      "loss": 0.05530819296836853,
+      "mean_token_accuracy": 0.9837485626339912,
+      "num_tokens": 5074502.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.3494797691702842,
+      "epoch": 2.450388802488336,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 8.637665049229243e-06,
+      "loss": 0.052686119079589845,
+      "mean_token_accuracy": 0.9853983774781228,
+      "num_tokens": 5099489.0,
+      "step": 985
+    },
+    {
+      "entropy": 1.253180342912674,
+      "epoch": 2.4628304821150855,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 8.263936366187824e-06,
+      "loss": 0.06666571497917176,
+      "mean_token_accuracy": 0.9831083133816719,
+      "num_tokens": 5121181.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.3052298992872238,
+      "epoch": 2.4752721617418354,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 7.89774324576521e-06,
+      "loss": 0.046898335218429565,
+      "mean_token_accuracy": 0.9847671329975128,
+      "num_tokens": 5144616.0,
+      "step": 995
+    },
+    {
+      "entropy": 1.2638304322957992,
+      "epoch": 2.487713841368585,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 7.539151805149985e-06,
+      "loss": 0.045440280437469484,
+      "mean_token_accuracy": 0.9862535953521728,
+      "num_tokens": 5173304.0,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1206,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.189578894138573e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8b76582097e27a940c0a231a0d53cf8cdc986bf57bf76ad6533b70015ba5c67
+size 5713

checkpoint-1206/README.md

@@ -0,0 +1,209 @@
+---
+base_model: numind/NuExtract-2.0-2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:numind/NuExtract-2.0-2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-1206/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "numind/NuExtract-2.0-2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1206/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75111344ac6827da35318c801df614a13fec222a63157a24e959fa27b46ac70d
+size 36987736

checkpoint-1206/chat_template.jinja

@@ -0,0 +1,138 @@
+{%- set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' -%}
+{%- for message in messages -%}
+    {%- if message['role'] == 'system' -%}
+        {{- '<|im_start|>system
+' -}}
+        {%- if message['content'] is string -%}
+            {{- message['content'] | trim -}}
+        {%- endif -%}
+        {{- '<|im_end|>
+' -}}
+    {%- elif message['role'] == 'user' -%}
+        {%- if loop.first -%}
+            {{- '<|im_start|>system
+' -}}
+            {%- if template -%}
+                {#--- If template, extraction task ---#}
+                {{- 'You are NuExtract, an information extraction tool created by NuMind.' -}}
+            {%- else -%}
+                {#--- Else, template generation task ---#}
+                {{- 'You are a helpful assistant.' -}}
+            {%- endif -%}
+            {{ '<|im_end|>
+' }}
+        {%- endif -%}
+        {{- '<|im_start|>' + message['role'] + '
+' -}}
+        {%- if template -%}
+            {#--- Template Section ---#}
+            {{- '# Template:
+' -}}
+            {{- template -}}
+            {{- '
+' -}}
+
+            {%- if examples -%}
+                {#--- Examples can only exist in the extraction task ---#}
+                {{- '# Examples:
+' -}}
+                {%- for example in examples -%}
+                    {{- '## Input:
+' -}}
+                    {%- if example['input'] is mapping and (example['input']['type'] == 'image' or example['input']['type'] == 'image_url') -%}
+                        {{- image_placeholder | trim -}}
+                    {%- elif example['input'] == '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                    {%- else -%}
+                        {#--- Text input example ---#}
+                        {{- example['input'] -}}
+                    {%- endif -%}
+                    {{- '
+' -}}
+                    {{- '## Output:
+' -}}
+                    {{- example['output'] -}}
+                    {{- '
+' -}}
+                {%- endfor -%}
+            {%- endif -%}
+            {{- '# Context:
+' -}}
+        {%- endif -%}
+
+        {%- if message['content'] is string -%}
+            {#--- Simple string content ---#}
+            {{- message['content'] | trim -}}
+        {%- elif message['content'] is mapping and (message['content']['type'] == 'image' or message['content']['type'] == 'image_url') -%}
+            {{- image_placeholder | trim -}}
+        {%- else -%}
+            {#--- List of content items (mixed text/images) ---#}
+            {#--- First, determine what the actual input content is (not ICL images) ---#}
+            {%- set ns = namespace(has_text_input=false, text_content='') -%}
+
+            {#--- Count content types and identify actual input document ---#}
+            {%- for content in message['content'] -%}
+                {%- if content is mapping and content.get('type') == 'text' -%}
+                    {%- if content.get('text') != '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {%- set ns.has_text_input = true -%}
+                        {%- set ns.text_content = content['text'] -%}
+                    {%- endif -%}
+                {%- elif content is string -%}
+                    {%- if content != '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {%- set ns.has_text_input = true -%}
+                        {%- set ns.text_content = content -%}
+                    {%- endif -%}
+                {%- endif -%}
+            {%- endfor -%}
+
+            {#--- Determine what to output based on actual input type ---#}
+            {%- if ns.has_text_input -%}
+                {#--- Main input is text, so output the text content ---#}
+                {{- ns.text_content | trim -}}
+            {%- else -%}
+                {#--- Main input is image or <image> placeholder ---#}
+                {%- set ns2 = namespace(found_image=false) -%}
+                {%- for content in message['content'] -%}
+                    {%- if content is mapping and (content.get('type') == 'image' or content.get('type') == 'image_url') and not ns2.found_image -%}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' and not ns2.found_image -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- elif content is string and content == '<image>' and not ns2.found_image -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+        {%- endif -%}
+        {{- '<|im_end|>
+'}}
+
+    {%- elif message['role'] == 'assistant' -%}
+        {{- '<|im_start|>assistant
+' -}}
+        {%- if message['content'] is string -%}
+            {{- message['content'] | trim -}}
+        {%- elif message['content'] is iterable and message['content'] is not string -%}
+            {%- for content in message['content'] -%}
+                {%- if content is mapping and content.get('type') == 'text' -%}
+                    {{- content['text'] | trim -}}
+                {%- elif content is string -%}
+                    {{- content | trim -}}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- '<|im_end|>
+' -}}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- '<|im_start|>assistant
+' -}}
+{%- endif -%}

checkpoint-1206/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f0cfb0f30c22bf8617409e96248871065ed248abe62399d13d128c94275e423
+size 37974661

checkpoint-1206/processor_config.json

@@ -0,0 +1,63 @@
+{
+  "image_processor": {
+    "data_format": "channels_first",
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "Qwen2VLImageProcessorFast",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "merge_size": 2,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 23000000,
+      "shortest_edge": 200704
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen2_5_VLProcessor",
+  "video_processor": {
+    "data_format": "channels_first",
+    "default_to_square": true,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": false,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 23000000,
+      "shortest_edge": 200704
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen2VLVideoProcessor"
+  }
+}

checkpoint-1206/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57c507d4737f8fd4956b786b54960e0705ec80a763e92e698acb6ee8fbaf248f
+size 14645

checkpoint-1206/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97bd5c49788fb1087c4805233ee40c9595252b330c52d191e3568fcd3d120784
+size 1465

checkpoint-1206/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a7aabe7a3b44651592a99607bb33066e6965260336be9218cccd53af161491d
+size 11422060

checkpoint-1206/tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "_commit_hash": null,
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "is_local": false,
+  "max_length": null,
+  "max_pixels": 23000000,
+  "min_pixels": 200704,
+  "model_max_length": 131072,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "processor_class": "Qwen2_5_VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-1206/trainer_state.json

@@ -0,0 +1,2466 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 1206,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.7456206619739532,
+      "epoch": 0.012441679626749611,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 1.0810810810810812e-05,
+      "loss": 0.31386559009552,
+      "mean_token_accuracy": 0.9072110533714295,
+      "num_tokens": 27963.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.8172702223062516,
+      "epoch": 0.024883359253499222,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 2.4324324324324327e-05,
+      "loss": 0.28579437732696533,
+      "mean_token_accuracy": 0.9151854813098907,
+      "num_tokens": 51393.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.7869735598564147,
+      "epoch": 0.03732503888024884,
+      "grad_norm": 0.046875,
+      "learning_rate": 3.783783783783784e-05,
+      "loss": 0.26468374729156496,
+      "mean_token_accuracy": 0.9194357767701149,
+      "num_tokens": 76323.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.7407771855592729,
+      "epoch": 0.049766718506998445,
+      "grad_norm": 0.0546875,
+      "learning_rate": 5.135135135135135e-05,
+      "loss": 0.22699759006500245,
+      "mean_token_accuracy": 0.936237809062004,
+      "num_tokens": 101144.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.8171133548021317,
+      "epoch": 0.06220839813374806,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 6.486486486486487e-05,
+      "loss": 0.21160719394683838,
+      "mean_token_accuracy": 0.9410519078373909,
+      "num_tokens": 125653.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.6494354471564292,
+      "epoch": 0.07465007776049767,
+      "grad_norm": 0.068359375,
+      "learning_rate": 7.837837837837838e-05,
+      "loss": 0.1776595950126648,
+      "mean_token_accuracy": 0.9463573560118675,
+      "num_tokens": 150968.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.5880378365516663,
+      "epoch": 0.08709175738724728,
+      "grad_norm": 0.076171875,
+      "learning_rate": 9.18918918918919e-05,
+      "loss": 0.18801350593566896,
+      "mean_token_accuracy": 0.9474205821752548,
+      "num_tokens": 176859.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.5649296075105668,
+      "epoch": 0.09953343701399689,
+      "grad_norm": 0.064453125,
+      "learning_rate": 9.99992777797229e-05,
+      "loss": 0.18648279905319215,
+      "mean_token_accuracy": 0.9436241507530212,
+      "num_tokens": 198909.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.5894027322530746,
+      "epoch": 0.1119751166407465,
+      "grad_norm": 0.05322265625,
+      "learning_rate": 9.999115304121457e-05,
+      "loss": 0.18634642362594606,
+      "mean_token_accuracy": 0.9470892950892449,
+      "num_tokens": 228344.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.596228164434433,
+      "epoch": 0.12441679626749612,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.997400226068578e-05,
+      "loss": 0.14496899843215943,
+      "mean_token_accuracy": 0.9579829111695289,
+      "num_tokens": 257062.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.509284293651581,
+      "epoch": 0.1368584758942457,
+      "grad_norm": 0.03369140625,
+      "learning_rate": 9.994782853475774e-05,
+      "loss": 0.14144277572631836,
+      "mean_token_accuracy": 0.9537434980273247,
+      "num_tokens": 283149.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.5500743955373764,
+      "epoch": 0.14930015552099535,
+      "grad_norm": 0.0546875,
+      "learning_rate": 9.99126365891696e-05,
+      "loss": 0.15906004905700682,
+      "mean_token_accuracy": 0.9526497855782509,
+      "num_tokens": 313192.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.5956889361143112,
+      "epoch": 0.16174183514774496,
+      "grad_norm": 0.061279296875,
+      "learning_rate": 9.986843277792523e-05,
+      "loss": 0.17970917224884034,
+      "mean_token_accuracy": 0.9503620356321335,
+      "num_tokens": 340137.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.6294057995080948,
+      "epoch": 0.17418351477449456,
+      "grad_norm": 0.055908203125,
+      "learning_rate": 9.9815225082146e-05,
+      "loss": 0.14261267185211182,
+      "mean_token_accuracy": 0.9562160760164261,
+      "num_tokens": 364984.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.589295208454132,
+      "epoch": 0.18662519440124417,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 9.975302310862964e-05,
+      "loss": 0.1694250226020813,
+      "mean_token_accuracy": 0.9456220239400863,
+      "num_tokens": 390651.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.583991777896881,
+      "epoch": 0.19906687402799378,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.968183808811586e-05,
+      "loss": 0.17054636478424073,
+      "mean_token_accuracy": 0.9493257701396942,
+      "num_tokens": 418166.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.5742847174406052,
+      "epoch": 0.2115085536547434,
+      "grad_norm": 0.06103515625,
+      "learning_rate": 9.960168287325852e-05,
+      "loss": 0.17596685886383057,
+      "mean_token_accuracy": 0.9445611819624901,
+      "num_tokens": 442501.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.5073494076728822,
+      "epoch": 0.223950233281493,
+      "grad_norm": 0.06640625,
+      "learning_rate": 9.951257193630507e-05,
+      "loss": 0.13867353200912474,
+      "mean_token_accuracy": 0.9568744376301765,
+      "num_tokens": 468083.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.5407251685857772,
+      "epoch": 0.2363919129082426,
+      "grad_norm": 0.0673828125,
+      "learning_rate": 9.941452136648359e-05,
+      "loss": 0.13775357007980346,
+      "mean_token_accuracy": 0.9594547316431999,
+      "num_tokens": 491914.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.4568117886781693,
+      "epoch": 0.24883359253499224,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 9.930754886709772e-05,
+      "loss": 0.13010139465332032,
+      "mean_token_accuracy": 0.9607375919818878,
+      "num_tokens": 518984.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.4679558753967286,
+      "epoch": 0.26127527216174184,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 9.919167375233041e-05,
+      "loss": 0.15394901037216185,
+      "mean_token_accuracy": 0.9542355045676232,
+      "num_tokens": 544627.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.5308140426874162,
+      "epoch": 0.2737169517884914,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 9.906691694375656e-05,
+      "loss": 0.14595698118209838,
+      "mean_token_accuracy": 0.9553182557225227,
+      "num_tokens": 568970.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.5124749779701232,
+      "epoch": 0.28615863141524106,
+      "grad_norm": 0.08056640625,
+      "learning_rate": 9.893330096656574e-05,
+      "loss": 0.1251847267150879,
+      "mean_token_accuracy": 0.9631539225578308,
+      "num_tokens": 594854.0,
+      "step": 115
+    },
+    {
+      "entropy": 1.5146708011627197,
+      "epoch": 0.2986003110419907,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 9.879084994549503e-05,
+      "loss": 0.12504571676254272,
+      "mean_token_accuracy": 0.9637450009584427,
+      "num_tokens": 621481.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.5158167153596878,
+      "epoch": 0.3110419906687403,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 9.863958960047335e-05,
+      "loss": 0.11272231340408326,
+      "mean_token_accuracy": 0.9642089486122132,
+      "num_tokens": 648057.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.4378292948007583,
+      "epoch": 0.3234836702954899,
+      "grad_norm": 0.05078125,
+      "learning_rate": 9.847954724197757e-05,
+      "loss": 0.1520946979522705,
+      "mean_token_accuracy": 0.9551170215010643,
+      "num_tokens": 676272.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.4547880798578263,
+      "epoch": 0.3359253499222395,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 9.831075176610163e-05,
+      "loss": 0.13464858531951904,
+      "mean_token_accuracy": 0.9599997282028199,
+      "num_tokens": 702496.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.5038821995258331,
+      "epoch": 0.3483670295489891,
+      "grad_norm": 0.058349609375,
+      "learning_rate": 9.813323364933915e-05,
+      "loss": 0.13690755367279053,
+      "mean_token_accuracy": 0.9559143736958504,
+      "num_tokens": 726296.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.5725213587284088,
+      "epoch": 0.3608087091757387,
+      "grad_norm": 0.05615234375,
+      "learning_rate": 9.794702494308093e-05,
+      "loss": 0.14729396104812623,
+      "mean_token_accuracy": 0.9551976174116135,
+      "num_tokens": 752371.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.4945250004529953,
+      "epoch": 0.37325038880248834,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.775215926782788e-05,
+      "loss": 0.1206730842590332,
+      "mean_token_accuracy": 0.9687894850969314,
+      "num_tokens": 775896.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.4557270616292954,
+      "epoch": 0.3856920684292379,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 9.754867180712082e-05,
+      "loss": 0.1341879963874817,
+      "mean_token_accuracy": 0.9626043424010277,
+      "num_tokens": 798172.0,
+      "step": 155
+    },
+    {
+      "entropy": 1.4879502207040787,
+      "epoch": 0.39813374805598756,
+      "grad_norm": 0.05224609375,
+      "learning_rate": 9.733659930118793e-05,
+      "loss": 0.13069820404052734,
+      "mean_token_accuracy": 0.9548368141055107,
+      "num_tokens": 824452.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.5602718144655228,
+      "epoch": 0.4105754276827372,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 9.711598004031131e-05,
+      "loss": 0.13032547235488892,
+      "mean_token_accuracy": 0.9589100196957588,
+      "num_tokens": 849316.0,
+      "step": 165
+    },
+    {
+      "entropy": 1.488172698020935,
+      "epoch": 0.4230171073094868,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 9.688685385791339e-05,
+      "loss": 0.13182138204574584,
+      "mean_token_accuracy": 0.9608529090881348,
+      "num_tokens": 878357.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.4886444747447967,
+      "epoch": 0.4354587869362364,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 9.664926212336505e-05,
+      "loss": 0.1238294005393982,
+      "mean_token_accuracy": 0.960505859553814,
+      "num_tokens": 904192.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.5064076393842698,
+      "epoch": 0.447900466562986,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 9.640324773451622e-05,
+      "loss": 0.167707359790802,
+      "mean_token_accuracy": 0.9532883360981941,
+      "num_tokens": 930575.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.445479767024517,
+      "epoch": 0.4603421461897356,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 9.614885510995047e-05,
+      "loss": 0.12214440107345581,
+      "mean_token_accuracy": 0.9636711940169335,
+      "num_tokens": 954713.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.508093237876892,
+      "epoch": 0.4727838258164852,
+      "grad_norm": 0.052734375,
+      "learning_rate": 9.588613018096529e-05,
+      "loss": 0.13071714639663695,
+      "mean_token_accuracy": 0.9601568162441254,
+      "num_tokens": 981241.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.4727303981781006,
+      "epoch": 0.48522550544323484,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 9.561512038327882e-05,
+      "loss": 0.11298023462295533,
+      "mean_token_accuracy": 0.9675653219223023,
+      "num_tokens": 1007954.0,
+      "step": 195
+    },
+    {
+      "entropy": 1.463455791771412,
+      "epoch": 0.4976671850699845,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 9.533587464846548e-05,
+      "loss": 0.13673393726348876,
+      "mean_token_accuracy": 0.9588131874799728,
+      "num_tokens": 1037256.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.455343420803547,
+      "epoch": 0.5101088646967341,
+      "grad_norm": 0.060302734375,
+      "learning_rate": 9.504844339512095e-05,
+      "loss": 0.16279250383377075,
+      "mean_token_accuracy": 0.952251310646534,
+      "num_tokens": 1059784.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.512709778547287,
+      "epoch": 0.5225505443234837,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 9.475287851975925e-05,
+      "loss": 0.10544586181640625,
+      "mean_token_accuracy": 0.9677229434251785,
+      "num_tokens": 1082209.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.462220585346222,
+      "epoch": 0.5349922239502333,
+      "grad_norm": 0.0400390625,
+      "learning_rate": 9.444923338744239e-05,
+      "loss": 0.11979058980941773,
+      "mean_token_accuracy": 0.9626576647162437,
+      "num_tokens": 1110595.0,
+      "step": 215
+    },
+    {
+      "entropy": 1.4548005178570746,
+      "epoch": 0.5474339035769828,
+      "grad_norm": 0.04345703125,
+      "learning_rate": 9.413756282214537e-05,
+      "loss": 0.1187555432319641,
+      "mean_token_accuracy": 0.9617565467953682,
+      "num_tokens": 1135832.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.493788743019104,
+      "epoch": 0.5598755832037325,
+      "grad_norm": 0.06591796875,
+      "learning_rate": 9.381792309685746e-05,
+      "loss": 0.15725988149642944,
+      "mean_token_accuracy": 0.9544133111834526,
+      "num_tokens": 1163621.0,
+      "step": 225
+    },
+    {
+      "entropy": 1.4669800892472267,
+      "epoch": 0.5723172628304821,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 9.3490371923422e-05,
+      "loss": 0.12038854360580445,
+      "mean_token_accuracy": 0.9577056676149368,
+      "num_tokens": 1188345.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.5317447245121003,
+      "epoch": 0.5847589424572317,
+      "grad_norm": 0.05615234375,
+      "learning_rate": 9.31549684421163e-05,
+      "loss": 0.12890284061431884,
+      "mean_token_accuracy": 0.9609759420156478,
+      "num_tokens": 1215187.0,
+      "step": 235
+    },
+    {
+      "entropy": 1.502908742427826,
+      "epoch": 0.5972006220839814,
+      "grad_norm": 0.048828125,
+      "learning_rate": 9.28117732109738e-05,
+      "loss": 0.12224407196044922,
+      "mean_token_accuracy": 0.9638051286339759,
+      "num_tokens": 1241282.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.4564243242144586,
+      "epoch": 0.609642301710731,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 9.246084819485004e-05,
+      "loss": 0.13289834260940553,
+      "mean_token_accuracy": 0.9601684510707855,
+      "num_tokens": 1268251.0,
+      "step": 245
+    },
+    {
+      "entropy": 1.4659890502691268,
+      "epoch": 0.6220839813374806,
+      "grad_norm": 0.0595703125,
+      "learning_rate": 9.21022567542348e-05,
+      "loss": 0.14783012866973877,
+      "mean_token_accuracy": 0.9558995455503464,
+      "num_tokens": 1292958.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.4668870717287064,
+      "epoch": 0.6345256609642301,
+      "grad_norm": 0.057861328125,
+      "learning_rate": 9.173606363381219e-05,
+      "loss": 0.12078696489334106,
+      "mean_token_accuracy": 0.965465834736824,
+      "num_tokens": 1318689.0,
+      "step": 255
+    },
+    {
+      "entropy": 1.4572956532239913,
+      "epoch": 0.6469673405909798,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 9.136233495077077e-05,
+      "loss": 0.1338449239730835,
+      "mean_token_accuracy": 0.9598589226603508,
+      "num_tokens": 1345620.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.436805287003517,
+      "epoch": 0.6594090202177294,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 9.098113818286595e-05,
+      "loss": 0.1269620180130005,
+      "mean_token_accuracy": 0.963837206363678,
+      "num_tokens": 1369989.0,
+      "step": 265
+    },
+    {
+      "entropy": 1.430183658003807,
+      "epoch": 0.671850699844479,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 9.059254215623668e-05,
+      "loss": 0.09294362068176269,
+      "mean_token_accuracy": 0.9718702122569084,
+      "num_tokens": 1399090.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.4391476839780808,
+      "epoch": 0.6842923794712286,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 9.01966170329787e-05,
+      "loss": 0.11170387268066406,
+      "mean_token_accuracy": 0.9664130583405495,
+      "num_tokens": 1425851.0,
+      "step": 275
+    },
+    {
+      "entropy": 1.5080994993448258,
+      "epoch": 0.6967340590979783,
+      "grad_norm": 0.06005859375,
+      "learning_rate": 8.979343429847653e-05,
+      "loss": 0.1207460641860962,
+      "mean_token_accuracy": 0.96175517141819,
+      "num_tokens": 1450517.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.4918249040842055,
+      "epoch": 0.7091757387247278,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 8.93830667484967e-05,
+      "loss": 0.11671255826950074,
+      "mean_token_accuracy": 0.9663726434111595,
+      "num_tokens": 1474715.0,
+      "step": 285
+    },
+    {
+      "entropy": 1.4864625990390778,
+      "epoch": 0.7216174183514774,
+      "grad_norm": 0.044921875,
+      "learning_rate": 8.896558847604414e-05,
+      "loss": 0.12517218589782714,
+      "mean_token_accuracy": 0.9652746349573136,
+      "num_tokens": 1499442.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.490417730808258,
+      "epoch": 0.7340590979782271,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 8.854107485798458e-05,
+      "loss": 0.11757193803787232,
+      "mean_token_accuracy": 0.9664239585399628,
+      "num_tokens": 1523873.0,
+      "step": 295
+    },
+    {
+      "entropy": 1.4401051640510558,
+      "epoch": 0.7465007776049767,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 8.810960254143499e-05,
+      "loss": 0.13470059633255005,
+      "mean_token_accuracy": 0.9591680407524109,
+      "num_tokens": 1548575.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.5322502881288529,
+      "epoch": 0.7589424572317263,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 8.767124942992475e-05,
+      "loss": 0.12030593156814576,
+      "mean_token_accuracy": 0.9647545605897904,
+      "num_tokens": 1575643.0,
+      "step": 305
+    },
+    {
+      "entropy": 1.458931715786457,
+      "epoch": 0.7713841368584758,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 8.722609466932996e-05,
+      "loss": 0.09307305216789245,
+      "mean_token_accuracy": 0.9665900468826294,
+      "num_tokens": 1600333.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.458182653784752,
+      "epoch": 0.7838258164852255,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 8.677421863358341e-05,
+      "loss": 0.12296299934387207,
+      "mean_token_accuracy": 0.9640052378177643,
+      "num_tokens": 1625117.0,
+      "step": 315
+    },
+    {
+      "entropy": 1.4836479336023332,
+      "epoch": 0.7962674961119751,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 8.631570291016284e-05,
+      "loss": 0.12105728387832641,
+      "mean_token_accuracy": 0.9649034470319748,
+      "num_tokens": 1649929.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.5023894742131234,
+      "epoch": 0.8087091757387247,
+      "grad_norm": 0.044921875,
+      "learning_rate": 8.585063028536016e-05,
+      "loss": 0.10724372863769531,
+      "mean_token_accuracy": 0.965858943760395,
+      "num_tokens": 1675782.0,
+      "step": 325
+    },
+    {
+      "entropy": 1.4469050377607346,
+      "epoch": 0.8211508553654744,
+      "grad_norm": 0.052490234375,
+      "learning_rate": 8.537908472933402e-05,
+      "loss": 0.13695430755615234,
+      "mean_token_accuracy": 0.9561882585287094,
+      "num_tokens": 1703718.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.5462682336568832,
+      "epoch": 0.833592534992224,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 8.490115138094889e-05,
+      "loss": 0.14301273822784424,
+      "mean_token_accuracy": 0.9608543425798416,
+      "num_tokens": 1731614.0,
+      "step": 335
+    },
+    {
+      "entropy": 1.5303865879774095,
+      "epoch": 0.8460342146189735,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 8.4416916532403e-05,
+      "loss": 0.1558545470237732,
+      "mean_token_accuracy": 0.9530998334288597,
+      "num_tokens": 1758723.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.451111488044262,
+      "epoch": 0.8584758942457231,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 8.392646761364789e-05,
+      "loss": 0.10284805297851562,
+      "mean_token_accuracy": 0.9650755852460862,
+      "num_tokens": 1785721.0,
+      "step": 345
+    },
+    {
+      "entropy": 1.484094935655594,
+      "epoch": 0.8709175738724728,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 8.342989317660289e-05,
+      "loss": 0.10030006170272827,
+      "mean_token_accuracy": 0.9675216883420944,
+      "num_tokens": 1814257.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.5280012369155884,
+      "epoch": 0.8833592534992224,
+      "grad_norm": 0.0546875,
+      "learning_rate": 8.292728287916665e-05,
+      "loss": 0.09518902897834777,
+      "mean_token_accuracy": 0.9697460547089577,
+      "num_tokens": 1838719.0,
+      "step": 355
+    },
+    {
+      "entropy": 1.4606290251016616,
+      "epoch": 0.895800933125972,
+      "grad_norm": 0.05810546875,
+      "learning_rate": 8.241872746902935e-05,
+      "loss": 0.09836371541023255,
+      "mean_token_accuracy": 0.9680053278803825,
+      "num_tokens": 1860428.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.4977456852793694,
+      "epoch": 0.9082426127527217,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 8.190431876728772e-05,
+      "loss": 0.11404428482055665,
+      "mean_token_accuracy": 0.9683067321777343,
+      "num_tokens": 1887249.0,
+      "step": 365
+    },
+    {
+      "entropy": 1.4481744587421417,
+      "epoch": 0.9206842923794712,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 8.138414965186667e-05,
+      "loss": 0.11055492162704468,
+      "mean_token_accuracy": 0.9680417776107788,
+      "num_tokens": 1913193.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.4087932407855988,
+      "epoch": 0.9331259720062208,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 8.085831404074984e-05,
+      "loss": 0.08752046227455139,
+      "mean_token_accuracy": 0.9722029373049736,
+      "num_tokens": 1935245.0,
+      "step": 375
+    },
+    {
+      "entropy": 1.4568452775478362,
+      "epoch": 0.9455676516329704,
+      "grad_norm": 0.0576171875,
+      "learning_rate": 8.03269068750225e-05,
+      "loss": 0.11881341934204101,
+      "mean_token_accuracy": 0.959407502412796,
+      "num_tokens": 1961480.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.4280684739351273,
+      "epoch": 0.9580093312597201,
+      "grad_norm": 0.05078125,
+      "learning_rate": 7.979002410172954e-05,
+      "loss": 0.14049121141433715,
+      "mean_token_accuracy": 0.9576024219393731,
+      "num_tokens": 1986643.0,
+      "step": 385
+    },
+    {
+      "entropy": 1.4764668315649032,
+      "epoch": 0.9704510108864697,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.924776265655216e-05,
+      "loss": 0.11259770393371582,
+      "mean_token_accuracy": 0.9635465934872627,
+      "num_tokens": 2012509.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.4508350789546967,
+      "epoch": 0.9828926905132193,
+      "grad_norm": 0.0361328125,
+      "learning_rate": 7.870022044630569e-05,
+      "loss": 0.11488029956817628,
+      "mean_token_accuracy": 0.9670460253953934,
+      "num_tokens": 2039717.0,
+      "step": 395
+    },
+    {
+      "entropy": 1.5077543556690216,
+      "epoch": 0.995334370139969,
+      "grad_norm": 0.06201171875,
+      "learning_rate": 7.814749633126235e-05,
+      "loss": 0.1424098014831543,
+      "mean_token_accuracy": 0.9543160945177078,
+      "num_tokens": 2064948.0,
+      "step": 400
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.4550913160425485,
+      "eval_loss": 0.12082157284021378,
+      "eval_mean_token_accuracy": 0.9625284673763668,
+      "eval_num_tokens": 2076268.0,
+      "eval_runtime": 24.7531,
+      "eval_samples_per_second": 12.16,
+      "eval_steps_per_second": 12.16,
+      "step": 402
+    },
+    {
+      "entropy": 1.4730579455693562,
+      "epoch": 1.0074650077760499,
+      "grad_norm": 0.0322265625,
+      "learning_rate": 7.75896901073017e-05,
+      "loss": 0.08911717534065247,
+      "mean_token_accuracy": 0.9732377681976709,
+      "num_tokens": 2088687.0,
+      "step": 405
+    },
+    {
+      "entropy": 1.436254122853279,
+      "epoch": 1.0199066874027993,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 7.702690248789223e-05,
+      "loss": 0.10224579572677613,
+      "mean_token_accuracy": 0.9676861181855202,
+      "num_tokens": 2114020.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.4225786730647088,
+      "epoch": 1.032348367029549,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 7.645923508590726e-05,
+      "loss": 0.09414710998535156,
+      "mean_token_accuracy": 0.9725460812449456,
+      "num_tokens": 2143268.0,
+      "step": 415
+    },
+    {
+      "entropy": 1.4235260367393494,
+      "epoch": 1.0447900466562987,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.588679039527848e-05,
+      "loss": 0.0781755805015564,
+      "mean_token_accuracy": 0.9752240985631943,
+      "num_tokens": 2168154.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.3582488745450974,
+      "epoch": 1.0572317262830482,
+      "grad_norm": 0.03564453125,
+      "learning_rate": 7.530967177249036e-05,
+      "loss": 0.06534834504127503,
+      "mean_token_accuracy": 0.9809993267059326,
+      "num_tokens": 2195567.0,
+      "step": 425
+    },
+    {
+      "entropy": 1.4120464980602265,
+      "epoch": 1.0696734059097979,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 7.472798341791877e-05,
+      "loss": 0.08980979323387146,
+      "mean_token_accuracy": 0.9729388952255249,
+      "num_tokens": 2220651.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.3695552781224252,
+      "epoch": 1.0821150855365473,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 7.414183035701749e-05,
+      "loss": 0.07374750971794128,
+      "mean_token_accuracy": 0.9773057013750076,
+      "num_tokens": 2244016.0,
+      "step": 435
+    },
+    {
+      "entropy": 1.361125336587429,
+      "epoch": 1.094556765163297,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.355131842135543e-05,
+      "loss": 0.09960444569587708,
+      "mean_token_accuracy": 0.9707440197467804,
+      "num_tokens": 2269688.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.4232443049550056,
+      "epoch": 1.1069984447900467,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.295655422950849e-05,
+      "loss": 0.08172831535339356,
+      "mean_token_accuracy": 0.9749625563621521,
+      "num_tokens": 2294075.0,
+      "step": 445
+    },
+    {
+      "entropy": 1.4120786994695664,
+      "epoch": 1.1194401244167962,
+      "grad_norm": 0.0341796875,
+      "learning_rate": 7.235764516780927e-05,
+      "loss": 0.06837162375450134,
+      "mean_token_accuracy": 0.9771177500486374,
+      "num_tokens": 2319276.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.4431693792343139,
+      "epoch": 1.1318818040435459,
+      "grad_norm": 0.044677734375,
+      "learning_rate": 7.175469937095818e-05,
+      "loss": 0.08680518865585327,
+      "mean_token_accuracy": 0.9729270681738853,
+      "num_tokens": 2347219.0,
+      "step": 455
+    },
+    {
+      "entropy": 1.4526385068893433,
+      "epoch": 1.1443234836702956,
+      "grad_norm": 0.10302734375,
+      "learning_rate": 7.114782570249943e-05,
+      "loss": 0.09112498164176941,
+      "mean_token_accuracy": 0.973096051812172,
+      "num_tokens": 2372584.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.3945886969566346,
+      "epoch": 1.156765163297045,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 7.053713373516538e-05,
+      "loss": 0.07018510103225709,
+      "mean_token_accuracy": 0.9799379587173462,
+      "num_tokens": 2398752.0,
+      "step": 465
+    },
+    {
+      "entropy": 1.3700096011161804,
+      "epoch": 1.1692068429237947,
+      "grad_norm": 0.055908203125,
+      "learning_rate": 6.992273373109288e-05,
+      "loss": 0.06986131072044373,
+      "mean_token_accuracy": 0.9791282191872597,
+      "num_tokens": 2421888.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.3979035422205925,
+      "epoch": 1.1816485225505442,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 6.930473662191517e-05,
+      "loss": 0.09055956602096557,
+      "mean_token_accuracy": 0.9743469759821892,
+      "num_tokens": 2448705.0,
+      "step": 475
+    },
+    {
+      "entropy": 1.3682101637125015,
+      "epoch": 1.194090202177294,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 6.868325398873284e-05,
+      "loss": 0.07687475085258484,
+      "mean_token_accuracy": 0.9755479469895363,
+      "num_tokens": 2474561.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.36375695168972,
+      "epoch": 1.2065318818040436,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 6.805839804196757e-05,
+      "loss": 0.09423953294754028,
+      "mean_token_accuracy": 0.9717281922698021,
+      "num_tokens": 2501620.0,
+      "step": 485
+    },
+    {
+      "entropy": 1.3528606280684472,
+      "epoch": 1.218973561430793,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 6.743028160110215e-05,
+      "loss": 0.07231994867324829,
+      "mean_token_accuracy": 0.976399302482605,
+      "num_tokens": 2527481.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.4269449830055236,
+      "epoch": 1.2314152410575427,
+      "grad_norm": 0.061279296875,
+      "learning_rate": 6.679901807431073e-05,
+      "loss": 0.0847695529460907,
+      "mean_token_accuracy": 0.9742431849241256,
+      "num_tokens": 2553527.0,
+      "step": 495
+    },
+    {
+      "entropy": 1.4285764783620833,
+      "epoch": 1.2438569206842924,
+      "grad_norm": 0.044677734375,
+      "learning_rate": 6.616472143798261e-05,
+      "loss": 0.07603438496589661,
+      "mean_token_accuracy": 0.9767945379018783,
+      "num_tokens": 2580242.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.4113897368311883,
+      "epoch": 1.256298600311042,
+      "grad_norm": 0.05078125,
+      "learning_rate": 6.552750621614337e-05,
+      "loss": 0.09946245551109315,
+      "mean_token_accuracy": 0.9701174065470696,
+      "num_tokens": 2604019.0,
+      "step": 505
+    },
+    {
+      "entropy": 1.2941894769668578,
+      "epoch": 1.2687402799377916,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 6.488748745977746e-05,
+      "loss": 0.0612099289894104,
+      "mean_token_accuracy": 0.981476029753685,
+      "num_tokens": 2628945.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.4310174494981767,
+      "epoch": 1.2811819595645413,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 6.424478072605522e-05,
+      "loss": 0.06989500522613526,
+      "mean_token_accuracy": 0.9810906872153282,
+      "num_tokens": 2652720.0,
+      "step": 515
+    },
+    {
+      "entropy": 1.3148574769496917,
+      "epoch": 1.2936236391912908,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 6.359950205746881e-05,
+      "loss": 0.07661423683166504,
+      "mean_token_accuracy": 0.9741629838943482,
+      "num_tokens": 2677256.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.3354602426290512,
+      "epoch": 1.3060653188180404,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 6.295176796088045e-05,
+      "loss": 0.0708840012550354,
+      "mean_token_accuracy": 0.9720481187105179,
+      "num_tokens": 2705462.0,
+      "step": 525
+    },
+    {
+      "entropy": 1.3693901777267456,
+      "epoch": 1.3185069984447901,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 6.230169538648667e-05,
+      "loss": 0.06346817016601562,
+      "mean_token_accuracy": 0.9793296322226525,
+      "num_tokens": 2731465.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.3165668994188309,
+      "epoch": 1.3309486780715396,
+      "grad_norm": 0.0625,
+      "learning_rate": 6.164940170670266e-05,
+      "loss": 0.08328393697738648,
+      "mean_token_accuracy": 0.971780526638031,
+      "num_tokens": 2759478.0,
+      "step": 535
+    },
+    {
+      "entropy": 1.34098874181509,
+      "epoch": 1.3433903576982893,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 6.09950046949705e-05,
+      "loss": 0.08074995279312133,
+      "mean_token_accuracy": 0.9730972275137901,
+      "num_tokens": 2783456.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.385405807197094,
+      "epoch": 1.355832037325039,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 6.0338622504494604e-05,
+      "loss": 0.06526267528533936,
+      "mean_token_accuracy": 0.9802431628108025,
+      "num_tokens": 2808812.0,
+      "step": 545
+    },
+    {
+      "entropy": 1.393546462059021,
+      "epoch": 1.3682737169517885,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 5.968037364690897e-05,
+      "loss": 0.08440290689468384,
+      "mean_token_accuracy": 0.9747256502509117,
+      "num_tokens": 2837088.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.4094155162572861,
+      "epoch": 1.3807153965785381,
+      "grad_norm": 0.058837890625,
+      "learning_rate": 5.902037697087957e-05,
+      "loss": 0.07816668152809143,
+      "mean_token_accuracy": 0.9740476593375206,
+      "num_tokens": 2862524.0,
+      "step": 555
+    },
+    {
+      "entropy": 1.350576937198639,
+      "epoch": 1.3931570762052878,
+      "grad_norm": 0.055419921875,
+      "learning_rate": 5.8358751640645835e-05,
+      "loss": 0.06395477056503296,
+      "mean_token_accuracy": 0.9800668403506279,
+      "num_tokens": 2884137.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.403828603029251,
+      "epoch": 1.4055987558320373,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 5.769561711450527e-05,
+      "loss": 0.0748115599155426,
+      "mean_token_accuracy": 0.9773760348558426,
+      "num_tokens": 2906231.0,
+      "step": 565
+    },
+    {
+      "entropy": 1.3695515617728233,
+      "epoch": 1.418040435458787,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 5.7031093123244925e-05,
+      "loss": 0.08527232408523559,
+      "mean_token_accuracy": 0.9715872302651405,
+      "num_tokens": 2932820.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.4267593204975129,
+      "epoch": 1.4304821150855365,
+      "grad_norm": 0.05859375,
+      "learning_rate": 5.6365299648523605e-05,
+      "loss": 0.10021589994430542,
+      "mean_token_accuracy": 0.9709113940596581,
+      "num_tokens": 2956192.0,
+      "step": 575
+    },
+    {
+      "entropy": 1.3429571866989136,
+      "epoch": 1.4429237947122862,
+      "grad_norm": 0.06005859375,
+      "learning_rate": 5.5698356901208925e-05,
+      "loss": 0.107265305519104,
+      "mean_token_accuracy": 0.9646854192018509,
+      "num_tokens": 2980056.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.4046858251094818,
+      "epoch": 1.4553654743390358,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 5.5030385299672884e-05,
+      "loss": 0.08158871531486511,
+      "mean_token_accuracy": 0.9777654156088829,
+      "num_tokens": 3004045.0,
+      "step": 585
+    },
+    {
+      "entropy": 1.4346278131008148,
+      "epoch": 1.4678071539657853,
+      "grad_norm": 0.06201171875,
+      "learning_rate": 5.436150544804994e-05,
+      "loss": 0.07914494276046753,
+      "mean_token_accuracy": 0.9757034927606583,
+      "num_tokens": 3027916.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.3918647408485412,
+      "epoch": 1.480248833592535,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 5.3691838114461626e-05,
+      "loss": 0.09960976839065552,
+      "mean_token_accuracy": 0.9678563356399537,
+      "num_tokens": 3053663.0,
+      "step": 595
+    },
+    {
+      "entropy": 1.3449461698532104,
+      "epoch": 1.4926905132192845,
+      "grad_norm": 0.06982421875,
+      "learning_rate": 5.302150420921161e-05,
+      "loss": 0.08556955456733703,
+      "mean_token_accuracy": 0.9714176893234253,
+      "num_tokens": 3080272.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.3727866768836976,
+      "epoch": 1.5051321928460342,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 5.2350624762954884e-05,
+      "loss": 0.0717516303062439,
+      "mean_token_accuracy": 0.9767438977956772,
+      "num_tokens": 3107201.0,
+      "step": 605
+    },
+    {
+      "entropy": 1.3760264009237289,
+      "epoch": 1.5175738724727839,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 5.1679320904845365e-05,
+      "loss": 0.08696765303611756,
+      "mean_token_accuracy": 0.9729200676083565,
+      "num_tokens": 3135290.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.3937256038188934,
+      "epoch": 1.5300155520995333,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 5.100771384066575e-05,
+      "loss": 0.06904829740524292,
+      "mean_token_accuracy": 0.9765424266457557,
+      "num_tokens": 3161468.0,
+      "step": 615
+    },
+    {
+      "entropy": 1.3464481949806213,
+      "epoch": 1.542457231726283,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 5.033592483094344e-05,
+      "loss": 0.08148046731948852,
+      "mean_token_accuracy": 0.9747312605381012,
+      "num_tokens": 3187972.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.3713692784309388,
+      "epoch": 1.5548989113530327,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 4.966407516905657e-05,
+      "loss": 0.09401994347572326,
+      "mean_token_accuracy": 0.9700708135962486,
+      "num_tokens": 3218206.0,
+      "step": 625
+    },
+    {
+      "entropy": 1.4130443289875985,
+      "epoch": 1.5673405909797822,
+      "grad_norm": 0.05517578125,
+      "learning_rate": 4.899228615933427e-05,
+      "loss": 0.102409827709198,
+      "mean_token_accuracy": 0.9645006597042084,
+      "num_tokens": 3243182.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.312636035680771,
+      "epoch": 1.5797822706065319,
+      "grad_norm": 0.05078125,
+      "learning_rate": 4.832067909515467e-05,
+      "loss": 0.07581937909126282,
+      "mean_token_accuracy": 0.977123475074768,
+      "num_tokens": 3270355.0,
+      "step": 635
+    },
+    {
+      "entropy": 1.2894637912511826,
+      "epoch": 1.5922239502332816,
+      "grad_norm": 0.0546875,
+      "learning_rate": 4.7649375237045135e-05,
+      "loss": 0.0760753870010376,
+      "mean_token_accuracy": 0.9742821797728538,
+      "num_tokens": 3295761.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.3615785390138626,
+      "epoch": 1.604665629860031,
+      "grad_norm": 0.052001953125,
+      "learning_rate": 4.697849579078838e-05,
+      "loss": 0.08204466700553895,
+      "mean_token_accuracy": 0.9755400061607361,
+      "num_tokens": 3322763.0,
+      "step": 645
+    },
+    {
+      "entropy": 1.4578467637300492,
+      "epoch": 1.6171073094867807,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 4.630816188553837e-05,
+      "loss": 0.10771067142486572,
+      "mean_token_accuracy": 0.9648271799087524,
+      "num_tokens": 3347850.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.3541809171438217,
+      "epoch": 1.6295489891135304,
+      "grad_norm": 0.044921875,
+      "learning_rate": 4.563849455195007e-05,
+      "loss": 0.08120843768119812,
+      "mean_token_accuracy": 0.9748451858758926,
+      "num_tokens": 3378295.0,
+      "step": 655
+    },
+    {
+      "entropy": 1.3386278599500656,
+      "epoch": 1.6419906687402799,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 4.496961470032713e-05,
+      "loss": 0.07182853221893311,
+      "mean_token_accuracy": 0.9755081802606582,
+      "num_tokens": 3407238.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.3182463720440865,
+      "epoch": 1.6544323483670296,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 4.4301643098791086e-05,
+      "loss": 0.0790963888168335,
+      "mean_token_accuracy": 0.9719948589801788,
+      "num_tokens": 3430188.0,
+      "step": 665
+    },
+    {
+      "entropy": 1.3785580173134804,
+      "epoch": 1.6668740279937793,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 4.363470035147641e-05,
+      "loss": 0.0732508659362793,
+      "mean_token_accuracy": 0.9752413406968117,
+      "num_tokens": 3454755.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.392490154504776,
+      "epoch": 1.6793157076205287,
+      "grad_norm": 0.041015625,
+      "learning_rate": 4.29689068767551e-05,
+      "loss": 0.07593491673469543,
+      "mean_token_accuracy": 0.9777896001935005,
+      "num_tokens": 3481020.0,
+      "step": 675
+    },
+    {
+      "entropy": 1.333091938495636,
+      "epoch": 1.6917573872472784,
+      "grad_norm": 0.052734375,
+      "learning_rate": 4.2304382885494753e-05,
+      "loss": 0.0870925784111023,
+      "mean_token_accuracy": 0.9741413682699204,
+      "num_tokens": 3507708.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.320798073709011,
+      "epoch": 1.7041990668740281,
+      "grad_norm": 0.04833984375,
+      "learning_rate": 4.164124835935418e-05,
+      "loss": 0.0877147912979126,
+      "mean_token_accuracy": 0.9753073915839195,
+      "num_tokens": 3534100.0,
+      "step": 685
+    },
+    {
+      "entropy": 1.3160912215709686,
+      "epoch": 1.7166407465007776,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 4.0979623029120436e-05,
+      "loss": 0.08963274955749512,
+      "mean_token_accuracy": 0.9698545068502427,
+      "num_tokens": 3558716.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.3380629509687423,
+      "epoch": 1.729082426127527,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 4.031962635309104e-05,
+      "loss": 0.07874792218208312,
+      "mean_token_accuracy": 0.9740808755159378,
+      "num_tokens": 3586065.0,
+      "step": 695
+    },
+    {
+      "entropy": 1.3654002964496612,
+      "epoch": 1.741524105754277,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 3.96613774955054e-05,
+      "loss": 0.08268317580223083,
+      "mean_token_accuracy": 0.9741670668125153,
+      "num_tokens": 3613513.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.3774547845125198,
+      "epoch": 1.7539657853810264,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 3.900499530502951e-05,
+      "loss": 0.08645263314247131,
+      "mean_token_accuracy": 0.9758756011724472,
+      "num_tokens": 3640806.0,
+      "step": 705
+    },
+    {
+      "entropy": 1.3115302637219428,
+      "epoch": 1.766407465007776,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 3.835059829329735e-05,
+      "loss": 0.07381963729858398,
+      "mean_token_accuracy": 0.9768764480948449,
+      "num_tokens": 3665922.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.3822921440005302,
+      "epoch": 1.7788491446345258,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 3.769830461351336e-05,
+      "loss": 0.09703359007835388,
+      "mean_token_accuracy": 0.9744590654969215,
+      "num_tokens": 3694612.0,
+      "step": 715
+    },
+    {
+      "entropy": 1.3549542352557182,
+      "epoch": 1.7912908242612753,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 3.704823203911957e-05,
+      "loss": 0.09285060167312623,
+      "mean_token_accuracy": 0.9735767096281052,
+      "num_tokens": 3718319.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.3544731348752976,
+      "epoch": 1.8037325038880248,
+      "grad_norm": 0.07421875,
+      "learning_rate": 3.6400497942531186e-05,
+      "loss": 0.07869815826416016,
+      "mean_token_accuracy": 0.977102306485176,
+      "num_tokens": 3742125.0,
+      "step": 725
+    },
+    {
+      "entropy": 1.3644321769475938,
+      "epoch": 1.8161741835147744,
+      "grad_norm": 0.0546875,
+      "learning_rate": 3.57552192739448e-05,
+      "loss": 0.08890414834022523,
+      "mean_token_accuracy": 0.973346334695816,
+      "num_tokens": 3765739.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.3641284555196762,
+      "epoch": 1.8286158631415241,
+      "grad_norm": 0.045654296875,
+      "learning_rate": 3.511251254022255e-05,
+      "loss": 0.11303408145904541,
+      "mean_token_accuracy": 0.9671000927686692,
+      "num_tokens": 3790908.0,
+      "step": 735
+    },
+    {
+      "entropy": 1.352848869562149,
+      "epoch": 1.8410575427682736,
+      "grad_norm": 0.046875,
+      "learning_rate": 3.447249378385663e-05,
+      "loss": 0.07621319890022278,
+      "mean_token_accuracy": 0.9736784845590591,
+      "num_tokens": 3817021.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.3620782911777496,
+      "epoch": 1.8534992223950233,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 3.38352785620174e-05,
+      "loss": 0.07505900859832763,
+      "mean_token_accuracy": 0.9756825506687165,
+      "num_tokens": 3845607.0,
+      "step": 745
+    },
+    {
+      "entropy": 1.387294201552868,
+      "epoch": 1.865940902021773,
+      "grad_norm": 0.040771484375,
+      "learning_rate": 3.3200981925689264e-05,
+      "loss": 0.07677844166755676,
+      "mean_token_accuracy": 0.9746143981814385,
+      "num_tokens": 3871956.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.3148253604769706,
+      "epoch": 1.8783825816485225,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 3.256971839889787e-05,
+      "loss": 0.062203264236450194,
+      "mean_token_accuracy": 0.9806876540184021,
+      "num_tokens": 3896461.0,
+      "step": 755
+    },
+    {
+      "entropy": 1.297327271103859,
+      "epoch": 1.8908242612752721,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 3.194160195803245e-05,
+      "loss": 0.07807562947273254,
+      "mean_token_accuracy": 0.9751886636018753,
+      "num_tokens": 3924652.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.2678332865238189,
+      "epoch": 1.9032659409020218,
+      "grad_norm": 0.04833984375,
+      "learning_rate": 3.1316746011267155e-05,
+      "loss": 0.0622839629650116,
+      "mean_token_accuracy": 0.980354443192482,
+      "num_tokens": 3950947.0,
+      "step": 765
+    },
+    {
+      "entropy": 1.3001187965273857,
+      "epoch": 1.9157076205287713,
+      "grad_norm": 0.05322265625,
+      "learning_rate": 3.0695263378084834e-05,
+      "loss": 0.07793177962303162,
+      "mean_token_accuracy": 0.9762747883796692,
+      "num_tokens": 3977231.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.343767161667347,
+      "epoch": 1.928149300155521,
+      "grad_norm": 0.058349609375,
+      "learning_rate": 3.0077266268907124e-05,
+      "loss": 0.09248242378234864,
+      "mean_token_accuracy": 0.9737435877323151,
+      "num_tokens": 4003918.0,
+      "step": 775
+    },
+    {
+      "entropy": 1.2801260307431221,
+      "epoch": 1.9405909797822707,
+      "grad_norm": 0.0673828125,
+      "learning_rate": 2.946286626483463e-05,
+      "loss": 0.07414901256561279,
+      "mean_token_accuracy": 0.978515362739563,
+      "num_tokens": 4027613.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.3166591823101044,
+      "epoch": 1.9530326594090202,
+      "grad_norm": 0.0634765625,
+      "learning_rate": 2.8852174297500567e-05,
+      "loss": 0.0782233715057373,
+      "mean_token_accuracy": 0.9734146118164062,
+      "num_tokens": 4053231.0,
+      "step": 785
+    },
+    {
+      "entropy": 1.2959586173295974,
+      "epoch": 1.9654743390357698,
+      "grad_norm": 0.05078125,
+      "learning_rate": 2.8245300629041838e-05,
+      "loss": 0.08309885859489441,
+      "mean_token_accuracy": 0.9741686061024666,
+      "num_tokens": 4081234.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.362495419383049,
+      "epoch": 1.9779160186625195,
+      "grad_norm": 0.047119140625,
+      "learning_rate": 2.7642354832190747e-05,
+      "loss": 0.09300965070724487,
+      "mean_token_accuracy": 0.9714553162455559,
+      "num_tokens": 4106601.0,
+      "step": 795
+    },
+    {
+      "entropy": 1.324322985112667,
+      "epoch": 1.990357698289269,
+      "grad_norm": 0.05078125,
+      "learning_rate": 2.7043445770491526e-05,
+      "loss": 0.0799511969089508,
+      "mean_token_accuracy": 0.9754261270165443,
+      "num_tokens": 4133337.0,
+      "step": 800
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 1.3053412522588457,
+      "eval_loss": 0.10880886763334274,
+      "eval_mean_token_accuracy": 0.9662994710709961,
+      "eval_num_tokens": 4152536.0,
+      "eval_runtime": 24.7392,
+      "eval_samples_per_second": 12.167,
+      "eval_steps_per_second": 12.167,
+      "step": 804
+    },
+    {
+      "entropy": 1.3139420319826176,
+      "epoch": 2.00248833592535,
+      "grad_norm": 0.0311279296875,
+      "learning_rate": 2.644868157864458e-05,
+      "loss": 0.07132983803749085,
+      "mean_token_accuracy": 0.9790512231680063,
+      "num_tokens": 4157848.0,
+      "step": 805
+    },
+    {
+      "entropy": 1.3346404328942298,
+      "epoch": 2.0149300155520997,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 2.5858169642982515e-05,
+      "loss": 0.06443278789520264,
+      "mean_token_accuracy": 0.9790809944272041,
+      "num_tokens": 4183018.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.3496920108795165,
+      "epoch": 2.027371695178849,
+      "grad_norm": 0.046142578125,
+      "learning_rate": 2.5272016582081236e-05,
+      "loss": 0.046033966541290286,
+      "mean_token_accuracy": 0.9862025916576386,
+      "num_tokens": 4207722.0,
+      "step": 815
+    },
+    {
+      "entropy": 1.2798842176795007,
+      "epoch": 2.0398133748055987,
+      "grad_norm": 0.033203125,
+      "learning_rate": 2.4690328227509668e-05,
+      "loss": 0.04474189877510071,
+      "mean_token_accuracy": 0.9851935356855392,
+      "num_tokens": 4233615.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.292403693497181,
+      "epoch": 2.0522550544323486,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 2.411320960472152e-05,
+      "loss": 0.05720994472503662,
+      "mean_token_accuracy": 0.9835459992289544,
+      "num_tokens": 4260265.0,
+      "step": 825
+    },
+    {
+      "entropy": 1.2822298243641854,
+      "epoch": 2.064696734059098,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 2.354076491409274e-05,
+      "loss": 0.06071372032165527,
+      "mean_token_accuracy": 0.9809239432215691,
+      "num_tokens": 4287688.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.3030796691775322,
+      "epoch": 2.0771384136858475,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 2.2973097512107796e-05,
+      "loss": 0.04943784773349762,
+      "mean_token_accuracy": 0.9850927606225014,
+      "num_tokens": 4311396.0,
+      "step": 835
+    },
+    {
+      "entropy": 1.2815598845481873,
+      "epoch": 2.0895800933125974,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 2.2410309892698322e-05,
+      "loss": 0.06365938782691956,
+      "mean_token_accuracy": 0.9812412708997726,
+      "num_tokens": 4338280.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.2353304088115693,
+      "epoch": 2.102021772939347,
+      "grad_norm": 0.0228271484375,
+      "learning_rate": 2.185250366873765e-05,
+      "loss": 0.03775741457939148,
+      "mean_token_accuracy": 0.988109527528286,
+      "num_tokens": 4361954.0,
+      "step": 845
+    },
+    {
+      "entropy": 1.3121544525027276,
+      "epoch": 2.1144634525660964,
+      "grad_norm": 0.052734375,
+      "learning_rate": 2.1299779553694323e-05,
+      "loss": 0.06021370887756348,
+      "mean_token_accuracy": 0.9813222840428353,
+      "num_tokens": 4389868.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.2844686791300775,
+      "epoch": 2.126905132192846,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 2.075223734344785e-05,
+      "loss": 0.043541711568832395,
+      "mean_token_accuracy": 0.9842599242925644,
+      "num_tokens": 4415143.0,
+      "step": 855
+    },
+    {
+      "entropy": 1.2730493247509003,
+      "epoch": 2.1393468118195957,
+      "grad_norm": 0.051025390625,
+      "learning_rate": 2.0209975898270454e-05,
+      "loss": 0.060882335901260375,
+      "mean_token_accuracy": 0.9831152334809303,
+      "num_tokens": 4439690.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.3221701458096504,
+      "epoch": 2.151788491446345,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 1.967309312497752e-05,
+      "loss": 0.06385582089424133,
+      "mean_token_accuracy": 0.980524268746376,
+      "num_tokens": 4468649.0,
+      "step": 865
+    },
+    {
+      "entropy": 1.308930653333664,
+      "epoch": 2.1642301710730947,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 1.914168595925016e-05,
+      "loss": 0.05137070417404175,
+      "mean_token_accuracy": 0.9829948961734771,
+      "num_tokens": 4494900.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.278279796242714,
+      "epoch": 2.1766718506998446,
+      "grad_norm": 0.044921875,
+      "learning_rate": 1.861585034813334e-05,
+      "loss": 0.053401678800582886,
+      "mean_token_accuracy": 0.9820586636662483,
+      "num_tokens": 4521907.0,
+      "step": 875
+    },
+    {
+      "entropy": 1.2940535843372345,
+      "epoch": 2.189113530326594,
+      "grad_norm": 0.033935546875,
+      "learning_rate": 1.809568123271231e-05,
+      "loss": 0.05988807678222656,
+      "mean_token_accuracy": 0.9810460910201073,
+      "num_tokens": 4546991.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.3210156977176666,
+      "epoch": 2.2015552099533435,
+      "grad_norm": 0.03515625,
+      "learning_rate": 1.7581272530970667e-05,
+      "loss": 0.05812197327613831,
+      "mean_token_accuracy": 0.9826369822025299,
+      "num_tokens": 4573645.0,
+      "step": 885
+    },
+    {
+      "entropy": 1.2087926417589188,
+      "epoch": 2.2139968895800934,
+      "grad_norm": 0.03564453125,
+      "learning_rate": 1.707271712083335e-05,
+      "loss": 0.059379422664642335,
+      "mean_token_accuracy": 0.9827022448182106,
+      "num_tokens": 4598343.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.2919662535190581,
+      "epoch": 2.226438569206843,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 1.6570106823397142e-05,
+      "loss": 0.05405360460281372,
+      "mean_token_accuracy": 0.9803063213825226,
+      "num_tokens": 4625264.0,
+      "step": 895
+    },
+    {
+      "entropy": 1.2397317215800285,
+      "epoch": 2.2388802488335924,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 1.607353238635212e-05,
+      "loss": 0.05695806741714478,
+      "mean_token_accuracy": 0.9809326454997063,
+      "num_tokens": 4655699.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.2710261449217797,
+      "epoch": 2.2513219284603423,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 1.558308346759701e-05,
+      "loss": 0.05667018294334412,
+      "mean_token_accuracy": 0.9820555970072746,
+      "num_tokens": 4682556.0,
+      "step": 905
+    },
+    {
+      "entropy": 1.3200107291340828,
+      "epoch": 2.2637636080870918,
+      "grad_norm": 0.0712890625,
+      "learning_rate": 1.5098848619051109e-05,
+      "loss": 0.058824723958969115,
+      "mean_token_accuracy": 0.9803817018866539,
+      "num_tokens": 4703229.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.3161411359906197,
+      "epoch": 2.2762052877138412,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 1.4620915270665992e-05,
+      "loss": 0.05733857154846191,
+      "mean_token_accuracy": 0.9827383399009705,
+      "num_tokens": 4732396.0,
+      "step": 915
+    },
+    {
+      "entropy": 1.27952651232481,
+      "epoch": 2.288646967340591,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 1.4149369714639853e-05,
+      "loss": 0.0526383638381958,
+      "mean_token_accuracy": 0.9817092105746269,
+      "num_tokens": 4758145.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.2732232689857483,
+      "epoch": 2.3010886469673406,
+      "grad_norm": 0.0281982421875,
+      "learning_rate": 1.3684297089837155e-05,
+      "loss": 0.055611425638198854,
+      "mean_token_accuracy": 0.982058934867382,
+      "num_tokens": 4784880.0,
+      "step": 925
+    },
+    {
+      "entropy": 1.2462307259440422,
+      "epoch": 2.31353032659409,
+      "grad_norm": 0.0498046875,
+      "learning_rate": 1.3225781366416595e-05,
+      "loss": 0.06674134135246276,
+      "mean_token_accuracy": 0.9809619382023811,
+      "num_tokens": 4814123.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.3400009661912917,
+      "epoch": 2.32597200622084,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 1.2773905330670038e-05,
+      "loss": 0.059016406536102295,
+      "mean_token_accuracy": 0.9813900917768479,
+      "num_tokens": 4838037.0,
+      "step": 935
+    },
+    {
+      "entropy": 1.2777298644185067,
+      "epoch": 2.3384136858475895,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 1.2328750570075265e-05,
+      "loss": 0.04491693377494812,
+      "mean_token_accuracy": 0.9859692931175232,
+      "num_tokens": 4865586.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.284477137029171,
+      "epoch": 2.350855365474339,
+      "grad_norm": 0.07763671875,
+      "learning_rate": 1.1890397458565022e-05,
+      "loss": 0.06391626596450806,
+      "mean_token_accuracy": 0.9819939568638801,
+      "num_tokens": 4889503.0,
+      "step": 945
+    },
+    {
+      "entropy": 1.2908324316143989,
+      "epoch": 2.3632970451010884,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 1.1458925142015431e-05,
+      "loss": 0.06202779412269592,
+      "mean_token_accuracy": 0.9813487082719803,
+      "num_tokens": 4918754.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.277482558786869,
+      "epoch": 2.3757387247278383,
+      "grad_norm": 0.031982421875,
+      "learning_rate": 1.103441152395588e-05,
+      "loss": 0.06154012680053711,
+      "mean_token_accuracy": 0.9820781499147415,
+      "num_tokens": 4945316.0,
+      "step": 955
+    },
+    {
+      "entropy": 1.2095389053225518,
+      "epoch": 2.388180404354588,
+      "grad_norm": 0.041748046875,
+      "learning_rate": 1.061693325150332e-05,
+      "loss": 0.06242812871932983,
+      "mean_token_accuracy": 0.9817898482084274,
+      "num_tokens": 4974069.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2972484081983566,
+      "epoch": 2.4006220839813377,
+      "grad_norm": 0.046875,
+      "learning_rate": 1.0206565701523479e-05,
+      "loss": 0.05046294927597046,
+      "mean_token_accuracy": 0.9853808850049972,
+      "num_tokens": 4999019.0,
+      "step": 965
+    },
+    {
+      "entropy": 1.3197123885154725,
+      "epoch": 2.413063763608087,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 9.803382967021318e-06,
+      "loss": 0.049368086457252505,
+      "mean_token_accuracy": 0.9852947980165482,
+      "num_tokens": 5023412.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.3319877043366433,
+      "epoch": 2.4255054432348366,
+      "grad_norm": 0.0693359375,
+      "learning_rate": 9.407457843763324e-06,
+      "loss": 0.0752966582775116,
+      "mean_token_accuracy": 0.9780165493488312,
+      "num_tokens": 5050267.0,
+      "step": 975
+    },
+    {
+      "entropy": 1.3291449189186095,
+      "epoch": 2.437947122861586,
+      "grad_norm": 0.056396484375,
+      "learning_rate": 9.01886181713405e-06,
+      "loss": 0.05530819296836853,
+      "mean_token_accuracy": 0.9837485626339912,
+      "num_tokens": 5074502.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.3494797691702842,
+      "epoch": 2.450388802488336,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 8.637665049229243e-06,
+      "loss": 0.052686119079589845,
+      "mean_token_accuracy": 0.9853983774781228,
+      "num_tokens": 5099489.0,
+      "step": 985
+    },
+    {
+      "entropy": 1.253180342912674,
+      "epoch": 2.4628304821150855,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 8.263936366187824e-06,
+      "loss": 0.06666571497917176,
+      "mean_token_accuracy": 0.9831083133816719,
+      "num_tokens": 5121181.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.3052298992872238,
+      "epoch": 2.4752721617418354,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 7.89774324576521e-06,
+      "loss": 0.046898335218429565,
+      "mean_token_accuracy": 0.9847671329975128,
+      "num_tokens": 5144616.0,
+      "step": 995
+    },
+    {
+      "entropy": 1.2638304322957992,
+      "epoch": 2.487713841368585,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 7.539151805149985e-06,
+      "loss": 0.045440280437469484,
+      "mean_token_accuracy": 0.9862535953521728,
+      "num_tokens": 5173304.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.2472808972001075,
+      "epoch": 2.5001555209953343,
+      "grad_norm": 0.05078125,
+      "learning_rate": 7.188226789026209e-06,
+      "loss": 0.05064120888710022,
+      "mean_token_accuracy": 0.9852252557873726,
+      "num_tokens": 5199260.0,
+      "step": 1005
+    },
+    {
+      "entropy": 1.2181950062513351,
+      "epoch": 2.512597200622084,
+      "grad_norm": 0.03662109375,
+      "learning_rate": 6.8450315578836965e-06,
+      "loss": 0.04357019364833832,
+      "mean_token_accuracy": 0.9878018543124198,
+      "num_tokens": 5225257.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.2247688069939613,
+      "epoch": 2.5250388802488337,
+      "grad_norm": 0.05908203125,
+      "learning_rate": 6.509628076578006e-06,
+      "loss": 0.04824144542217255,
+      "mean_token_accuracy": 0.9864720001816749,
+      "num_tokens": 5248640.0,
+      "step": 1015
+    },
+    {
+      "entropy": 1.336793726682663,
+      "epoch": 2.537480559875583,
+      "grad_norm": 0.041259765625,
+      "learning_rate": 6.182076903142536e-06,
+      "loss": 0.0592013418674469,
+      "mean_token_accuracy": 0.9802371919155121,
+      "num_tokens": 5273818.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.2980262845754624,
+      "epoch": 2.5499222395023327,
+      "grad_norm": 0.060546875,
+      "learning_rate": 5.86243717785463e-06,
+      "loss": 0.07260814309120178,
+      "mean_token_accuracy": 0.9779389306902886,
+      "num_tokens": 5297288.0,
+      "step": 1025
+    },
+    {
+      "entropy": 1.3199328914284707,
+      "epoch": 2.5623639191290826,
+      "grad_norm": 0.0322265625,
+      "learning_rate": 5.550766612557623e-06,
+      "loss": 0.052888506650924684,
+      "mean_token_accuracy": 0.9840954542160034,
+      "num_tokens": 5323105.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.212834571301937,
+      "epoch": 2.574805598755832,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 5.247121480240763e-06,
+      "loss": 0.05025215744972229,
+      "mean_token_accuracy": 0.9859568744897842,
+      "num_tokens": 5349976.0,
+      "step": 1035
+    },
+    {
+      "entropy": 1.2222013637423514,
+      "epoch": 2.5872472783825815,
+      "grad_norm": 0.0390625,
+      "learning_rate": 4.951556604879048e-06,
+      "loss": 0.0416751354932785,
+      "mean_token_accuracy": 0.9878290638327598,
+      "num_tokens": 5376611.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.2588894337415695,
+      "epoch": 2.5996889580093314,
+      "grad_norm": 0.043701171875,
+      "learning_rate": 4.664125351534537e-06,
+      "loss": 0.05584250688552857,
+      "mean_token_accuracy": 0.9814404875040055,
+      "num_tokens": 5405173.0,
+      "step": 1045
+    },
+    {
+      "entropy": 1.2924429386854173,
+      "epoch": 2.612130637636081,
+      "grad_norm": 0.058349609375,
+      "learning_rate": 4.384879616721182e-06,
+      "loss": 0.06444652080535888,
+      "mean_token_accuracy": 0.9828690186142921,
+      "num_tokens": 5430741.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.2675760596990586,
+      "epoch": 2.6245723172628304,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 4.113869819034727e-06,
+      "loss": 0.06400873661041259,
+      "mean_token_accuracy": 0.9813549533486366,
+      "num_tokens": 5457007.0,
+      "step": 1055
+    },
+    {
+      "entropy": 1.3134230434894563,
+      "epoch": 2.6370139968895803,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 3.851144890049535e-06,
+      "loss": 0.034584978222846986,
+      "mean_token_accuracy": 0.9877016335725785,
+      "num_tokens": 5480882.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.2463923379778863,
+      "epoch": 2.6494556765163297,
+      "grad_norm": 0.04345703125,
+      "learning_rate": 3.5967522654837994e-06,
+      "loss": 0.04645509421825409,
+      "mean_token_accuracy": 0.9869309529662132,
+      "num_tokens": 5505629.0,
+      "step": 1065
+    },
+    {
+      "entropy": 1.2719882696866989,
+      "epoch": 2.661897356143079,
+      "grad_norm": 0.046142578125,
+      "learning_rate": 3.350737876634957e-06,
+      "loss": 0.055052381753921506,
+      "mean_token_accuracy": 0.9827572226524353,
+      "num_tokens": 5533711.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.2736148864030838,
+      "epoch": 2.6743390357698287,
+      "grad_norm": 0.037109375,
+      "learning_rate": 3.1131461420866313e-06,
+      "loss": 0.05363653302192688,
+      "mean_token_accuracy": 0.9812890663743019,
+      "num_tokens": 5561153.0,
+      "step": 1075
+    },
+    {
+      "entropy": 1.2958104774355887,
+      "epoch": 2.6867807153965786,
+      "grad_norm": 0.048828125,
+      "learning_rate": 2.8840199596887105e-06,
+      "loss": 0.054463303089141844,
+      "mean_token_accuracy": 0.9833293497562409,
+      "num_tokens": 5585461.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.2556652486324311,
+      "epoch": 2.699222395023328,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 2.663400698812074e-06,
+      "loss": 0.05533112287521362,
+      "mean_token_accuracy": 0.9832669913768768,
+      "num_tokens": 5614981.0,
+      "step": 1085
+    },
+    {
+      "entropy": 1.2375948414206506,
+      "epoch": 2.711664074650078,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 2.4513281928791985e-06,
+      "loss": 0.04163530468940735,
+      "mean_token_accuracy": 0.9874510899186134,
+      "num_tokens": 5641773.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.2753075495362283,
+      "epoch": 2.7241057542768274,
+      "grad_norm": 0.049560546875,
+      "learning_rate": 2.2478407321721296e-06,
+      "loss": 0.058247965574264524,
+      "mean_token_accuracy": 0.981316888332367,
+      "num_tokens": 5669049.0,
+      "step": 1095
+    },
+    {
+      "entropy": 1.2350706443190576,
+      "epoch": 2.736547433903577,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 2.0529750569190763e-06,
+      "loss": 0.06670538783073425,
+      "mean_token_accuracy": 0.9805434107780456,
+      "num_tokens": 5694713.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.3031635895371436,
+      "epoch": 2.7489891135303264,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 1.8667663506608534e-06,
+      "loss": 0.06874535083770753,
+      "mean_token_accuracy": 0.9813943326473236,
+      "num_tokens": 5720385.0,
+      "step": 1105
+    },
+    {
+      "entropy": 1.2971147924661637,
+      "epoch": 2.7614307931570763,
+      "grad_norm": 0.040771484375,
+      "learning_rate": 1.6892482338983828e-06,
+      "loss": 0.04181385338306427,
+      "mean_token_accuracy": 0.9848969042301178,
+      "num_tokens": 5746503.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.2740083515644074,
+      "epoch": 2.7738724727838258,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 1.5204527580224337e-06,
+      "loss": 0.062212252616882326,
+      "mean_token_accuracy": 0.9814715966582298,
+      "num_tokens": 5772786.0,
+      "step": 1115
+    },
+    {
+      "entropy": 1.2509995222091674,
+      "epoch": 2.7863141524105757,
+      "grad_norm": 0.058837890625,
+      "learning_rate": 1.3604103995266682e-06,
+      "loss": 0.05331828594207764,
+      "mean_token_accuracy": 0.981832081079483,
+      "num_tokens": 5798040.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.2227709844708443,
+      "epoch": 2.798755832037325,
+      "grad_norm": 0.03173828125,
+      "learning_rate": 1.2091500545049706e-06,
+      "loss": 0.04361798167228699,
+      "mean_token_accuracy": 0.9867074429988861,
+      "num_tokens": 5824550.0,
+      "step": 1125
+    },
+    {
+      "entropy": 1.2401750206947326,
+      "epoch": 2.8111975116640746,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 1.0666990334342707e-06,
+      "loss": 0.061069542169570924,
+      "mean_token_accuracy": 0.9831201821565628,
+      "num_tokens": 5850795.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.2588699147105218,
+      "epoch": 2.823639191290824,
+      "grad_norm": 0.058837890625,
+      "learning_rate": 9.330830562434445e-07,
+      "loss": 0.04628670811653137,
+      "mean_token_accuracy": 0.9860796064138413,
+      "num_tokens": 5873475.0,
+      "step": 1135
+    },
+    {
+      "entropy": 1.2346306607127189,
+      "epoch": 2.836080870917574,
+      "grad_norm": 0.058349609375,
+      "learning_rate": 8.083262476696051e-07,
+      "loss": 0.04909757375717163,
+      "mean_token_accuracy": 0.983722123503685,
+      "num_tokens": 5898165.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.1937159553170205,
+      "epoch": 2.8485225505443235,
+      "grad_norm": 0.045654296875,
+      "learning_rate": 6.924511329022831e-07,
+      "loss": 0.040495744347572325,
+      "mean_token_accuracy": 0.9897799164056778,
+      "num_tokens": 5922973.0,
+      "step": 1145
+    },
+    {
+      "entropy": 1.254883785545826,
+      "epoch": 2.860964230171073,
+      "grad_norm": 0.0546875,
+      "learning_rate": 5.85478633516412e-07,
+      "loss": 0.06383077502250671,
+      "mean_token_accuracy": 0.980024679005146,
+      "num_tokens": 5947992.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.2277784809470176,
+      "epoch": 2.873405909797823,
+      "grad_norm": 0.0380859375,
+      "learning_rate": 4.874280636949225e-07,
+      "loss": 0.07050376534461975,
+      "mean_token_accuracy": 0.9783104926347732,
+      "num_tokens": 5977052.0,
+      "step": 1155
+    },
+    {
+      "entropy": 1.2470836490392685,
+      "epoch": 2.8858475894245723,
+      "grad_norm": 0.038330078125,
+      "learning_rate": 3.983171267414876e-07,
+      "loss": 0.058488309383392334,
+      "mean_token_accuracy": 0.9789799645543098,
+      "num_tokens": 6005890.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.234851485490799,
+      "epoch": 2.898289269051322,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 3.1816191188415166e-07,
+      "loss": 0.046985325217247007,
+      "mean_token_accuracy": 0.9848568633198738,
+      "num_tokens": 6032772.0,
+      "step": 1165
+    },
+    {
+      "entropy": 1.2423007264733315,
+      "epoch": 2.9107309486780717,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 2.469768913703707e-07,
+      "loss": 0.04711828827857971,
+      "mean_token_accuracy": 0.9853164970874786,
+      "num_tokens": 6055333.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.2636106505990028,
+      "epoch": 2.923172628304821,
+      "grad_norm": 0.03173828125,
+      "learning_rate": 1.8477491785400813e-07,
+      "loss": 0.05374208092689514,
+      "mean_token_accuracy": 0.9832712143659592,
+      "num_tokens": 6081293.0,
+      "step": 1175
+    },
+    {
+      "entropy": 1.276752583682537,
+      "epoch": 2.9356143079315706,
+      "grad_norm": 0.05078125,
+      "learning_rate": 1.3156722207476324e-07,
+      "loss": 0.0528965175151825,
+      "mean_token_accuracy": 0.9834408789873124,
+      "num_tokens": 6102892.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.281411574780941,
+      "epoch": 2.9480559875583205,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 8.736341083041e-08,
+      "loss": 0.06098886728286743,
+      "mean_token_accuracy": 0.983394218981266,
+      "num_tokens": 6128039.0,
+      "step": 1185
+    },
+    {
+      "entropy": 1.2323307275772095,
+      "epoch": 2.96049766718507,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 5.217146524226779e-08,
+      "loss": 0.059885329008102416,
+      "mean_token_accuracy": 0.9809183284640313,
+      "num_tokens": 6153194.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.3001432150602341,
+      "epoch": 2.9729393468118195,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 2.599773931422078e-08,
+      "loss": 0.05909959077835083,
+      "mean_token_accuracy": 0.9834809839725495,
+      "num_tokens": 6176012.0,
+      "step": 1195
+    },
+    {
+      "entropy": 1.279952661693096,
+      "epoch": 2.985381026438569,
+      "grad_norm": 0.05078125,
+      "learning_rate": 8.846958785418968e-09,
+      "loss": 0.05961666703224182,
+      "mean_token_accuracy": 0.9833701640367508,
+      "num_tokens": 6200123.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.2743590876460076,
+      "epoch": 2.997822706065319,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 7.222202770995612e-10,
+      "loss": 0.06714363694190979,
+      "mean_token_accuracy": 0.9778998792171478,
+      "num_tokens": 6224618.0,
+      "step": 1205
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1206,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.452427650516173e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1206/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8b76582097e27a940c0a231a0d53cf8cdc986bf57bf76ad6533b70015ba5c67
+size 5713

checkpoint-500/README.md

@@ -0,0 +1,209 @@
+---
+base_model: numind/NuExtract-2.0-2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:numind/NuExtract-2.0-2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-500/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "numind/NuExtract-2.0-2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "o_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-500/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:583d7963c36352b61380aeb767b07f74e84e4c5df9e08964553668ed47e16894
+size 36987736

checkpoint-500/chat_template.jinja

@@ -0,0 +1,138 @@
+{%- set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' -%}
+{%- for message in messages -%}
+    {%- if message['role'] == 'system' -%}
+        {{- '<|im_start|>system
+' -}}
+        {%- if message['content'] is string -%}
+            {{- message['content'] | trim -}}
+        {%- endif -%}
+        {{- '<|im_end|>
+' -}}
+    {%- elif message['role'] == 'user' -%}
+        {%- if loop.first -%}
+            {{- '<|im_start|>system
+' -}}
+            {%- if template -%}
+                {#--- If template, extraction task ---#}
+                {{- 'You are NuExtract, an information extraction tool created by NuMind.' -}}
+            {%- else -%}
+                {#--- Else, template generation task ---#}
+                {{- 'You are a helpful assistant.' -}}
+            {%- endif -%}
+            {{ '<|im_end|>
+' }}
+        {%- endif -%}
+        {{- '<|im_start|>' + message['role'] + '
+' -}}
+        {%- if template -%}
+            {#--- Template Section ---#}
+            {{- '# Template:
+' -}}
+            {{- template -}}
+            {{- '
+' -}}
+
+            {%- if examples -%}
+                {#--- Examples can only exist in the extraction task ---#}
+                {{- '# Examples:
+' -}}
+                {%- for example in examples -%}
+                    {{- '## Input:
+' -}}
+                    {%- if example['input'] is mapping and (example['input']['type'] == 'image' or example['input']['type'] == 'image_url') -%}
+                        {{- image_placeholder | trim -}}
+                    {%- elif example['input'] == '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                    {%- else -%}
+                        {#--- Text input example ---#}
+                        {{- example['input'] -}}
+                    {%- endif -%}
+                    {{- '
+' -}}
+                    {{- '## Output:
+' -}}
+                    {{- example['output'] -}}
+                    {{- '
+' -}}
+                {%- endfor -%}
+            {%- endif -%}
+            {{- '# Context:
+' -}}
+        {%- endif -%}
+
+        {%- if message['content'] is string -%}
+            {#--- Simple string content ---#}
+            {{- message['content'] | trim -}}
+        {%- elif message['content'] is mapping and (message['content']['type'] == 'image' or message['content']['type'] == 'image_url') -%}
+            {{- image_placeholder | trim -}}
+        {%- else -%}
+            {#--- List of content items (mixed text/images) ---#}
+            {#--- First, determine what the actual input content is (not ICL images) ---#}
+            {%- set ns = namespace(has_text_input=false, text_content='') -%}
+
+            {#--- Count content types and identify actual input document ---#}
+            {%- for content in message['content'] -%}
+                {%- if content is mapping and content.get('type') == 'text' -%}
+                    {%- if content.get('text') != '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {%- set ns.has_text_input = true -%}
+                        {%- set ns.text_content = content['text'] -%}
+                    {%- endif -%}
+                {%- elif content is string -%}
+                    {%- if content != '<image>' -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {%- set ns.has_text_input = true -%}
+                        {%- set ns.text_content = content -%}
+                    {%- endif -%}
+                {%- endif -%}
+            {%- endfor -%}
+
+            {#--- Determine what to output based on actual input type ---#}
+            {%- if ns.has_text_input -%}
+                {#--- Main input is text, so output the text content ---#}
+                {{- ns.text_content | trim -}}
+            {%- else -%}
+                {#--- Main input is image or <image> placeholder ---#}
+                {%- set ns2 = namespace(found_image=false) -%}
+                {%- for content in message['content'] -%}
+                    {%- if content is mapping and (content.get('type') == 'image' or content.get('type') == 'image_url') and not ns2.found_image -%}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' and not ns2.found_image -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- elif content is string and content == '<image>' and not ns2.found_image -%}
+                        {#--- Keep compatibility with <image> for now ---#}
+                        {{- image_placeholder | trim -}}
+                        {%- set ns2.found_image = true -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+        {%- endif -%}
+        {{- '<|im_end|>
+'}}
+
+    {%- elif message['role'] == 'assistant' -%}
+        {{- '<|im_start|>assistant
+' -}}
+        {%- if message['content'] is string -%}
+            {{- message['content'] | trim -}}
+        {%- elif message['content'] is iterable and message['content'] is not string -%}
+            {%- for content in message['content'] -%}
+                {%- if content is mapping and content.get('type') == 'text' -%}
+                    {{- content['text'] | trim -}}
+                {%- elif content is string -%}
+                    {{- content | trim -}}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- '<|im_end|>
+' -}}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- '<|im_start|>assistant
+' -}}
+{%- endif -%}

checkpoint-500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dbbe91749a9747fdf30600532c529ca9d9faf68b81ffbf8c29c7b09a67bdba1
+size 37974661

checkpoint-500/processor_config.json

@@ -0,0 +1,63 @@
+{
+  "image_processor": {
+    "data_format": "channels_first",
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "Qwen2VLImageProcessorFast",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "merge_size": 2,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 23000000,
+      "shortest_edge": 200704
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen2_5_VLProcessor",
+  "video_processor": {
+    "data_format": "channels_first",
+    "default_to_square": true,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": false,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 23000000,
+      "shortest_edge": 200704
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen2VLVideoProcessor"
+  }
+}

checkpoint-500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3507a55b80ea938133601ff2b9607fb4b918a167b9a4501101de08f54d5bc446
+size 14645

checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6e62b5f44547ed2054439a643f15a54551684d83daa00770b073ca8671c7cfb
+size 1465

checkpoint-500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a7aabe7a3b44651592a99607bb33066e6965260336be9218cccd53af161491d
+size 11422060

checkpoint-500/tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "_commit_hash": null,
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "is_local": false,
+  "max_length": null,
+  "max_pixels": 23000000,
+  "min_pixels": 200704,
+  "model_max_length": 131072,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "processor_class": "Qwen2_5_VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-500/trainer_state.json

@@ -0,0 +1,1045 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.2438569206842924,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.7456206619739532,
+      "epoch": 0.012441679626749611,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 1.0810810810810812e-05,
+      "loss": 0.31386559009552,
+      "mean_token_accuracy": 0.9072110533714295,
+      "num_tokens": 27963.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.8172702223062516,
+      "epoch": 0.024883359253499222,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 2.4324324324324327e-05,
+      "loss": 0.28579437732696533,
+      "mean_token_accuracy": 0.9151854813098907,
+      "num_tokens": 51393.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.7869735598564147,
+      "epoch": 0.03732503888024884,
+      "grad_norm": 0.046875,
+      "learning_rate": 3.783783783783784e-05,
+      "loss": 0.26468374729156496,
+      "mean_token_accuracy": 0.9194357767701149,
+      "num_tokens": 76323.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.7407771855592729,
+      "epoch": 0.049766718506998445,
+      "grad_norm": 0.0546875,
+      "learning_rate": 5.135135135135135e-05,
+      "loss": 0.22699759006500245,
+      "mean_token_accuracy": 0.936237809062004,
+      "num_tokens": 101144.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.8171133548021317,
+      "epoch": 0.06220839813374806,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 6.486486486486487e-05,
+      "loss": 0.21160719394683838,
+      "mean_token_accuracy": 0.9410519078373909,
+      "num_tokens": 125653.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.6494354471564292,
+      "epoch": 0.07465007776049767,
+      "grad_norm": 0.068359375,
+      "learning_rate": 7.837837837837838e-05,
+      "loss": 0.1776595950126648,
+      "mean_token_accuracy": 0.9463573560118675,
+      "num_tokens": 150968.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.5880378365516663,
+      "epoch": 0.08709175738724728,
+      "grad_norm": 0.076171875,
+      "learning_rate": 9.18918918918919e-05,
+      "loss": 0.18801350593566896,
+      "mean_token_accuracy": 0.9474205821752548,
+      "num_tokens": 176859.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.5649296075105668,
+      "epoch": 0.09953343701399689,
+      "grad_norm": 0.064453125,
+      "learning_rate": 9.99992777797229e-05,
+      "loss": 0.18648279905319215,
+      "mean_token_accuracy": 0.9436241507530212,
+      "num_tokens": 198909.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.5894027322530746,
+      "epoch": 0.1119751166407465,
+      "grad_norm": 0.05322265625,
+      "learning_rate": 9.999115304121457e-05,
+      "loss": 0.18634642362594606,
+      "mean_token_accuracy": 0.9470892950892449,
+      "num_tokens": 228344.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.596228164434433,
+      "epoch": 0.12441679626749612,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.997400226068578e-05,
+      "loss": 0.14496899843215943,
+      "mean_token_accuracy": 0.9579829111695289,
+      "num_tokens": 257062.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.509284293651581,
+      "epoch": 0.1368584758942457,
+      "grad_norm": 0.03369140625,
+      "learning_rate": 9.994782853475774e-05,
+      "loss": 0.14144277572631836,
+      "mean_token_accuracy": 0.9537434980273247,
+      "num_tokens": 283149.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.5500743955373764,
+      "epoch": 0.14930015552099535,
+      "grad_norm": 0.0546875,
+      "learning_rate": 9.99126365891696e-05,
+      "loss": 0.15906004905700682,
+      "mean_token_accuracy": 0.9526497855782509,
+      "num_tokens": 313192.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.5956889361143112,
+      "epoch": 0.16174183514774496,
+      "grad_norm": 0.061279296875,
+      "learning_rate": 9.986843277792523e-05,
+      "loss": 0.17970917224884034,
+      "mean_token_accuracy": 0.9503620356321335,
+      "num_tokens": 340137.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.6294057995080948,
+      "epoch": 0.17418351477449456,
+      "grad_norm": 0.055908203125,
+      "learning_rate": 9.9815225082146e-05,
+      "loss": 0.14261267185211182,
+      "mean_token_accuracy": 0.9562160760164261,
+      "num_tokens": 364984.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.589295208454132,
+      "epoch": 0.18662519440124417,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 9.975302310862964e-05,
+      "loss": 0.1694250226020813,
+      "mean_token_accuracy": 0.9456220239400863,
+      "num_tokens": 390651.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.583991777896881,
+      "epoch": 0.19906687402799378,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.968183808811586e-05,
+      "loss": 0.17054636478424073,
+      "mean_token_accuracy": 0.9493257701396942,
+      "num_tokens": 418166.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.5742847174406052,
+      "epoch": 0.2115085536547434,
+      "grad_norm": 0.06103515625,
+      "learning_rate": 9.960168287325852e-05,
+      "loss": 0.17596685886383057,
+      "mean_token_accuracy": 0.9445611819624901,
+      "num_tokens": 442501.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.5073494076728822,
+      "epoch": 0.223950233281493,
+      "grad_norm": 0.06640625,
+      "learning_rate": 9.951257193630507e-05,
+      "loss": 0.13867353200912474,
+      "mean_token_accuracy": 0.9568744376301765,
+      "num_tokens": 468083.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.5407251685857772,
+      "epoch": 0.2363919129082426,
+      "grad_norm": 0.0673828125,
+      "learning_rate": 9.941452136648359e-05,
+      "loss": 0.13775357007980346,
+      "mean_token_accuracy": 0.9594547316431999,
+      "num_tokens": 491914.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.4568117886781693,
+      "epoch": 0.24883359253499224,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 9.930754886709772e-05,
+      "loss": 0.13010139465332032,
+      "mean_token_accuracy": 0.9607375919818878,
+      "num_tokens": 518984.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.4679558753967286,
+      "epoch": 0.26127527216174184,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 9.919167375233041e-05,
+      "loss": 0.15394901037216185,
+      "mean_token_accuracy": 0.9542355045676232,
+      "num_tokens": 544627.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.5308140426874162,
+      "epoch": 0.2737169517884914,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 9.906691694375656e-05,
+      "loss": 0.14595698118209838,
+      "mean_token_accuracy": 0.9553182557225227,
+      "num_tokens": 568970.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.5124749779701232,
+      "epoch": 0.28615863141524106,
+      "grad_norm": 0.08056640625,
+      "learning_rate": 9.893330096656574e-05,
+      "loss": 0.1251847267150879,
+      "mean_token_accuracy": 0.9631539225578308,
+      "num_tokens": 594854.0,
+      "step": 115
+    },
+    {
+      "entropy": 1.5146708011627197,
+      "epoch": 0.2986003110419907,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 9.879084994549503e-05,
+      "loss": 0.12504571676254272,
+      "mean_token_accuracy": 0.9637450009584427,
+      "num_tokens": 621481.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.5158167153596878,
+      "epoch": 0.3110419906687403,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 9.863958960047335e-05,
+      "loss": 0.11272231340408326,
+      "mean_token_accuracy": 0.9642089486122132,
+      "num_tokens": 648057.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.4378292948007583,
+      "epoch": 0.3234836702954899,
+      "grad_norm": 0.05078125,
+      "learning_rate": 9.847954724197757e-05,
+      "loss": 0.1520946979522705,
+      "mean_token_accuracy": 0.9551170215010643,
+      "num_tokens": 676272.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.4547880798578263,
+      "epoch": 0.3359253499222395,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 9.831075176610163e-05,
+      "loss": 0.13464858531951904,
+      "mean_token_accuracy": 0.9599997282028199,
+      "num_tokens": 702496.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.5038821995258331,
+      "epoch": 0.3483670295489891,
+      "grad_norm": 0.058349609375,
+      "learning_rate": 9.813323364933915e-05,
+      "loss": 0.13690755367279053,
+      "mean_token_accuracy": 0.9559143736958504,
+      "num_tokens": 726296.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.5725213587284088,
+      "epoch": 0.3608087091757387,
+      "grad_norm": 0.05615234375,
+      "learning_rate": 9.794702494308093e-05,
+      "loss": 0.14729396104812623,
+      "mean_token_accuracy": 0.9551976174116135,
+      "num_tokens": 752371.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.4945250004529953,
+      "epoch": 0.37325038880248834,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 9.775215926782788e-05,
+      "loss": 0.1206730842590332,
+      "mean_token_accuracy": 0.9687894850969314,
+      "num_tokens": 775896.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.4557270616292954,
+      "epoch": 0.3856920684292379,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 9.754867180712082e-05,
+      "loss": 0.1341879963874817,
+      "mean_token_accuracy": 0.9626043424010277,
+      "num_tokens": 798172.0,
+      "step": 155
+    },
+    {
+      "entropy": 1.4879502207040787,
+      "epoch": 0.39813374805598756,
+      "grad_norm": 0.05224609375,
+      "learning_rate": 9.733659930118793e-05,
+      "loss": 0.13069820404052734,
+      "mean_token_accuracy": 0.9548368141055107,
+      "num_tokens": 824452.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.5602718144655228,
+      "epoch": 0.4105754276827372,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 9.711598004031131e-05,
+      "loss": 0.13032547235488892,
+      "mean_token_accuracy": 0.9589100196957588,
+      "num_tokens": 849316.0,
+      "step": 165
+    },
+    {
+      "entropy": 1.488172698020935,
+      "epoch": 0.4230171073094868,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 9.688685385791339e-05,
+      "loss": 0.13182138204574584,
+      "mean_token_accuracy": 0.9608529090881348,
+      "num_tokens": 878357.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.4886444747447967,
+      "epoch": 0.4354587869362364,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 9.664926212336505e-05,
+      "loss": 0.1238294005393982,
+      "mean_token_accuracy": 0.960505859553814,
+      "num_tokens": 904192.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.5064076393842698,
+      "epoch": 0.447900466562986,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 9.640324773451622e-05,
+      "loss": 0.167707359790802,
+      "mean_token_accuracy": 0.9532883360981941,
+      "num_tokens": 930575.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.445479767024517,
+      "epoch": 0.4603421461897356,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 9.614885510995047e-05,
+      "loss": 0.12214440107345581,
+      "mean_token_accuracy": 0.9636711940169335,
+      "num_tokens": 954713.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.508093237876892,
+      "epoch": 0.4727838258164852,
+      "grad_norm": 0.052734375,
+      "learning_rate": 9.588613018096529e-05,
+      "loss": 0.13071714639663695,
+      "mean_token_accuracy": 0.9601568162441254,
+      "num_tokens": 981241.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.4727303981781006,
+      "epoch": 0.48522550544323484,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 9.561512038327882e-05,
+      "loss": 0.11298023462295533,
+      "mean_token_accuracy": 0.9675653219223023,
+      "num_tokens": 1007954.0,
+      "step": 195
+    },
+    {
+      "entropy": 1.463455791771412,
+      "epoch": 0.4976671850699845,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 9.533587464846548e-05,
+      "loss": 0.13673393726348876,
+      "mean_token_accuracy": 0.9588131874799728,
+      "num_tokens": 1037256.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.455343420803547,
+      "epoch": 0.5101088646967341,
+      "grad_norm": 0.060302734375,
+      "learning_rate": 9.504844339512095e-05,
+      "loss": 0.16279250383377075,
+      "mean_token_accuracy": 0.952251310646534,
+      "num_tokens": 1059784.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.512709778547287,
+      "epoch": 0.5225505443234837,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 9.475287851975925e-05,
+      "loss": 0.10544586181640625,
+      "mean_token_accuracy": 0.9677229434251785,
+      "num_tokens": 1082209.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.462220585346222,
+      "epoch": 0.5349922239502333,
+      "grad_norm": 0.0400390625,
+      "learning_rate": 9.444923338744239e-05,
+      "loss": 0.11979058980941773,
+      "mean_token_accuracy": 0.9626576647162437,
+      "num_tokens": 1110595.0,
+      "step": 215
+    },
+    {
+      "entropy": 1.4548005178570746,
+      "epoch": 0.5474339035769828,
+      "grad_norm": 0.04345703125,
+      "learning_rate": 9.413756282214537e-05,
+      "loss": 0.1187555432319641,
+      "mean_token_accuracy": 0.9617565467953682,
+      "num_tokens": 1135832.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.493788743019104,
+      "epoch": 0.5598755832037325,
+      "grad_norm": 0.06591796875,
+      "learning_rate": 9.381792309685746e-05,
+      "loss": 0.15725988149642944,
+      "mean_token_accuracy": 0.9544133111834526,
+      "num_tokens": 1163621.0,
+      "step": 225
+    },
+    {
+      "entropy": 1.4669800892472267,
+      "epoch": 0.5723172628304821,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 9.3490371923422e-05,
+      "loss": 0.12038854360580445,
+      "mean_token_accuracy": 0.9577056676149368,
+      "num_tokens": 1188345.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.5317447245121003,
+      "epoch": 0.5847589424572317,
+      "grad_norm": 0.05615234375,
+      "learning_rate": 9.31549684421163e-05,
+      "loss": 0.12890284061431884,
+      "mean_token_accuracy": 0.9609759420156478,
+      "num_tokens": 1215187.0,
+      "step": 235
+    },
+    {
+      "entropy": 1.502908742427826,
+      "epoch": 0.5972006220839814,
+      "grad_norm": 0.048828125,
+      "learning_rate": 9.28117732109738e-05,
+      "loss": 0.12224407196044922,
+      "mean_token_accuracy": 0.9638051286339759,
+      "num_tokens": 1241282.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.4564243242144586,
+      "epoch": 0.609642301710731,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 9.246084819485004e-05,
+      "loss": 0.13289834260940553,
+      "mean_token_accuracy": 0.9601684510707855,
+      "num_tokens": 1268251.0,
+      "step": 245
+    },
+    {
+      "entropy": 1.4659890502691268,
+      "epoch": 0.6220839813374806,
+      "grad_norm": 0.0595703125,
+      "learning_rate": 9.21022567542348e-05,
+      "loss": 0.14783012866973877,
+      "mean_token_accuracy": 0.9558995455503464,
+      "num_tokens": 1292958.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.4668870717287064,
+      "epoch": 0.6345256609642301,
+      "grad_norm": 0.057861328125,
+      "learning_rate": 9.173606363381219e-05,
+      "loss": 0.12078696489334106,
+      "mean_token_accuracy": 0.965465834736824,
+      "num_tokens": 1318689.0,
+      "step": 255
+    },
+    {
+      "entropy": 1.4572956532239913,
+      "epoch": 0.6469673405909798,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 9.136233495077077e-05,
+      "loss": 0.1338449239730835,
+      "mean_token_accuracy": 0.9598589226603508,
+      "num_tokens": 1345620.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.436805287003517,
+      "epoch": 0.6594090202177294,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 9.098113818286595e-05,
+      "loss": 0.1269620180130005,
+      "mean_token_accuracy": 0.963837206363678,
+      "num_tokens": 1369989.0,
+      "step": 265
+    },
+    {
+      "entropy": 1.430183658003807,
+      "epoch": 0.671850699844479,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 9.059254215623668e-05,
+      "loss": 0.09294362068176269,
+      "mean_token_accuracy": 0.9718702122569084,
+      "num_tokens": 1399090.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.4391476839780808,
+      "epoch": 0.6842923794712286,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 9.01966170329787e-05,
+      "loss": 0.11170387268066406,
+      "mean_token_accuracy": 0.9664130583405495,
+      "num_tokens": 1425851.0,
+      "step": 275
+    },
+    {
+      "entropy": 1.5080994993448258,
+      "epoch": 0.6967340590979783,
+      "grad_norm": 0.06005859375,
+      "learning_rate": 8.979343429847653e-05,
+      "loss": 0.1207460641860962,
+      "mean_token_accuracy": 0.96175517141819,
+      "num_tokens": 1450517.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.4918249040842055,
+      "epoch": 0.7091757387247278,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 8.93830667484967e-05,
+      "loss": 0.11671255826950074,
+      "mean_token_accuracy": 0.9663726434111595,
+      "num_tokens": 1474715.0,
+      "step": 285
+    },
+    {
+      "entropy": 1.4864625990390778,
+      "epoch": 0.7216174183514774,
+      "grad_norm": 0.044921875,
+      "learning_rate": 8.896558847604414e-05,
+      "loss": 0.12517218589782714,
+      "mean_token_accuracy": 0.9652746349573136,
+      "num_tokens": 1499442.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.490417730808258,
+      "epoch": 0.7340590979782271,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 8.854107485798458e-05,
+      "loss": 0.11757193803787232,
+      "mean_token_accuracy": 0.9664239585399628,
+      "num_tokens": 1523873.0,
+      "step": 295
+    },
+    {
+      "entropy": 1.4401051640510558,
+      "epoch": 0.7465007776049767,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 8.810960254143499e-05,
+      "loss": 0.13470059633255005,
+      "mean_token_accuracy": 0.9591680407524109,
+      "num_tokens": 1548575.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.5322502881288529,
+      "epoch": 0.7589424572317263,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 8.767124942992475e-05,
+      "loss": 0.12030593156814576,
+      "mean_token_accuracy": 0.9647545605897904,
+      "num_tokens": 1575643.0,
+      "step": 305
+    },
+    {
+      "entropy": 1.458931715786457,
+      "epoch": 0.7713841368584758,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 8.722609466932996e-05,
+      "loss": 0.09307305216789245,
+      "mean_token_accuracy": 0.9665900468826294,
+      "num_tokens": 1600333.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.458182653784752,
+      "epoch": 0.7838258164852255,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 8.677421863358341e-05,
+      "loss": 0.12296299934387207,
+      "mean_token_accuracy": 0.9640052378177643,
+      "num_tokens": 1625117.0,
+      "step": 315
+    },
+    {
+      "entropy": 1.4836479336023332,
+      "epoch": 0.7962674961119751,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 8.631570291016284e-05,
+      "loss": 0.12105728387832641,
+      "mean_token_accuracy": 0.9649034470319748,
+      "num_tokens": 1649929.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.5023894742131234,
+      "epoch": 0.8087091757387247,
+      "grad_norm": 0.044921875,
+      "learning_rate": 8.585063028536016e-05,
+      "loss": 0.10724372863769531,
+      "mean_token_accuracy": 0.965858943760395,
+      "num_tokens": 1675782.0,
+      "step": 325
+    },
+    {
+      "entropy": 1.4469050377607346,
+      "epoch": 0.8211508553654744,
+      "grad_norm": 0.052490234375,
+      "learning_rate": 8.537908472933402e-05,
+      "loss": 0.13695430755615234,
+      "mean_token_accuracy": 0.9561882585287094,
+      "num_tokens": 1703718.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.5462682336568832,
+      "epoch": 0.833592534992224,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 8.490115138094889e-05,
+      "loss": 0.14301273822784424,
+      "mean_token_accuracy": 0.9608543425798416,
+      "num_tokens": 1731614.0,
+      "step": 335
+    },
+    {
+      "entropy": 1.5303865879774095,
+      "epoch": 0.8460342146189735,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 8.4416916532403e-05,
+      "loss": 0.1558545470237732,
+      "mean_token_accuracy": 0.9530998334288597,
+      "num_tokens": 1758723.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.451111488044262,
+      "epoch": 0.8584758942457231,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 8.392646761364789e-05,
+      "loss": 0.10284805297851562,
+      "mean_token_accuracy": 0.9650755852460862,
+      "num_tokens": 1785721.0,
+      "step": 345
+    },
+    {
+      "entropy": 1.484094935655594,
+      "epoch": 0.8709175738724728,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 8.342989317660289e-05,
+      "loss": 0.10030006170272827,
+      "mean_token_accuracy": 0.9675216883420944,
+      "num_tokens": 1814257.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.5280012369155884,
+      "epoch": 0.8833592534992224,
+      "grad_norm": 0.0546875,
+      "learning_rate": 8.292728287916665e-05,
+      "loss": 0.09518902897834777,
+      "mean_token_accuracy": 0.9697460547089577,
+      "num_tokens": 1838719.0,
+      "step": 355
+    },
+    {
+      "entropy": 1.4606290251016616,
+      "epoch": 0.895800933125972,
+      "grad_norm": 0.05810546875,
+      "learning_rate": 8.241872746902935e-05,
+      "loss": 0.09836371541023255,
+      "mean_token_accuracy": 0.9680053278803825,
+      "num_tokens": 1860428.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.4977456852793694,
+      "epoch": 0.9082426127527217,
+      "grad_norm": 0.053466796875,
+      "learning_rate": 8.190431876728772e-05,
+      "loss": 0.11404428482055665,
+      "mean_token_accuracy": 0.9683067321777343,
+      "num_tokens": 1887249.0,
+      "step": 365
+    },
+    {
+      "entropy": 1.4481744587421417,
+      "epoch": 0.9206842923794712,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 8.138414965186667e-05,
+      "loss": 0.11055492162704468,
+      "mean_token_accuracy": 0.9680417776107788,
+      "num_tokens": 1913193.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.4087932407855988,
+      "epoch": 0.9331259720062208,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 8.085831404074984e-05,
+      "loss": 0.08752046227455139,
+      "mean_token_accuracy": 0.9722029373049736,
+      "num_tokens": 1935245.0,
+      "step": 375
+    },
+    {
+      "entropy": 1.4568452775478362,
+      "epoch": 0.9455676516329704,
+      "grad_norm": 0.0576171875,
+      "learning_rate": 8.03269068750225e-05,
+      "loss": 0.11881341934204101,
+      "mean_token_accuracy": 0.959407502412796,
+      "num_tokens": 1961480.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.4280684739351273,
+      "epoch": 0.9580093312597201,
+      "grad_norm": 0.05078125,
+      "learning_rate": 7.979002410172954e-05,
+      "loss": 0.14049121141433715,
+      "mean_token_accuracy": 0.9576024219393731,
+      "num_tokens": 1986643.0,
+      "step": 385
+    },
+    {
+      "entropy": 1.4764668315649032,
+      "epoch": 0.9704510108864697,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.924776265655216e-05,
+      "loss": 0.11259770393371582,
+      "mean_token_accuracy": 0.9635465934872627,
+      "num_tokens": 2012509.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.4508350789546967,
+      "epoch": 0.9828926905132193,
+      "grad_norm": 0.0361328125,
+      "learning_rate": 7.870022044630569e-05,
+      "loss": 0.11488029956817628,
+      "mean_token_accuracy": 0.9670460253953934,
+      "num_tokens": 2039717.0,
+      "step": 395
+    },
+    {
+      "entropy": 1.5077543556690216,
+      "epoch": 0.995334370139969,
+      "grad_norm": 0.06201171875,
+      "learning_rate": 7.814749633126235e-05,
+      "loss": 0.1424098014831543,
+      "mean_token_accuracy": 0.9543160945177078,
+      "num_tokens": 2064948.0,
+      "step": 400
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.4550913160425485,
+      "eval_loss": 0.12082157284021378,
+      "eval_mean_token_accuracy": 0.9625284673763668,
+      "eval_num_tokens": 2076268.0,
+      "eval_runtime": 24.7531,
+      "eval_samples_per_second": 12.16,
+      "eval_steps_per_second": 12.16,
+      "step": 402
+    },
+    {
+      "entropy": 1.4730579455693562,
+      "epoch": 1.0074650077760499,
+      "grad_norm": 0.0322265625,
+      "learning_rate": 7.75896901073017e-05,
+      "loss": 0.08911717534065247,
+      "mean_token_accuracy": 0.9732377681976709,
+      "num_tokens": 2088687.0,
+      "step": 405
+    },
+    {
+      "entropy": 1.436254122853279,
+      "epoch": 1.0199066874027993,
+      "grad_norm": 0.04931640625,
+      "learning_rate": 7.702690248789223e-05,
+      "loss": 0.10224579572677613,
+      "mean_token_accuracy": 0.9676861181855202,
+      "num_tokens": 2114020.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.4225786730647088,
+      "epoch": 1.032348367029549,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 7.645923508590726e-05,
+      "loss": 0.09414710998535156,
+      "mean_token_accuracy": 0.9725460812449456,
+      "num_tokens": 2143268.0,
+      "step": 415
+    },
+    {
+      "entropy": 1.4235260367393494,
+      "epoch": 1.0447900466562987,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.588679039527848e-05,
+      "loss": 0.0781755805015564,
+      "mean_token_accuracy": 0.9752240985631943,
+      "num_tokens": 2168154.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.3582488745450974,
+      "epoch": 1.0572317262830482,
+      "grad_norm": 0.03564453125,
+      "learning_rate": 7.530967177249036e-05,
+      "loss": 0.06534834504127503,
+      "mean_token_accuracy": 0.9809993267059326,
+      "num_tokens": 2195567.0,
+      "step": 425
+    },
+    {
+      "entropy": 1.4120464980602265,
+      "epoch": 1.0696734059097979,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 7.472798341791877e-05,
+      "loss": 0.08980979323387146,
+      "mean_token_accuracy": 0.9729388952255249,
+      "num_tokens": 2220651.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.3695552781224252,
+      "epoch": 1.0821150855365473,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 7.414183035701749e-05,
+      "loss": 0.07374750971794128,
+      "mean_token_accuracy": 0.9773057013750076,
+      "num_tokens": 2244016.0,
+      "step": 435
+    },
+    {
+      "entropy": 1.361125336587429,
+      "epoch": 1.094556765163297,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.355131842135543e-05,
+      "loss": 0.09960444569587708,
+      "mean_token_accuracy": 0.9707440197467804,
+      "num_tokens": 2269688.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.4232443049550056,
+      "epoch": 1.1069984447900467,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 7.295655422950849e-05,
+      "loss": 0.08172831535339356,
+      "mean_token_accuracy": 0.9749625563621521,
+      "num_tokens": 2294075.0,
+      "step": 445
+    },
+    {
+      "entropy": 1.4120786994695664,
+      "epoch": 1.1194401244167962,
+      "grad_norm": 0.0341796875,
+      "learning_rate": 7.235764516780927e-05,
+      "loss": 0.06837162375450134,
+      "mean_token_accuracy": 0.9771177500486374,
+      "num_tokens": 2319276.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.4431693792343139,
+      "epoch": 1.1318818040435459,
+      "grad_norm": 0.044677734375,
+      "learning_rate": 7.175469937095818e-05,
+      "loss": 0.08680518865585327,
+      "mean_token_accuracy": 0.9729270681738853,
+      "num_tokens": 2347219.0,
+      "step": 455
+    },
+    {
+      "entropy": 1.4526385068893433,
+      "epoch": 1.1443234836702956,
+      "grad_norm": 0.10302734375,
+      "learning_rate": 7.114782570249943e-05,
+      "loss": 0.09112498164176941,
+      "mean_token_accuracy": 0.973096051812172,
+      "num_tokens": 2372584.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.3945886969566346,
+      "epoch": 1.156765163297045,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 7.053713373516538e-05,
+      "loss": 0.07018510103225709,
+      "mean_token_accuracy": 0.9799379587173462,
+      "num_tokens": 2398752.0,
+      "step": 465
+    },
+    {
+      "entropy": 1.3700096011161804,
+      "epoch": 1.1692068429237947,
+      "grad_norm": 0.055908203125,
+      "learning_rate": 6.992273373109288e-05,
+      "loss": 0.06986131072044373,
+      "mean_token_accuracy": 0.9791282191872597,
+      "num_tokens": 2421888.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.3979035422205925,
+      "epoch": 1.1816485225505442,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 6.930473662191517e-05,
+      "loss": 0.09055956602096557,
+      "mean_token_accuracy": 0.9743469759821892,
+      "num_tokens": 2448705.0,
+      "step": 475
+    },
+    {
+      "entropy": 1.3682101637125015,
+      "epoch": 1.194090202177294,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 6.868325398873284e-05,
+      "loss": 0.07687475085258484,
+      "mean_token_accuracy": 0.9755479469895363,
+      "num_tokens": 2474561.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.36375695168972,
+      "epoch": 1.2065318818040436,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 6.805839804196757e-05,
+      "loss": 0.09423953294754028,
+      "mean_token_accuracy": 0.9717281922698021,
+      "num_tokens": 2501620.0,
+      "step": 485
+    },
+    {
+      "entropy": 1.3528606280684472,
+      "epoch": 1.218973561430793,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 6.743028160110215e-05,
+      "loss": 0.07231994867324829,
+      "mean_token_accuracy": 0.976399302482605,
+      "num_tokens": 2527481.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.4269449830055236,
+      "epoch": 1.2314152410575427,
+      "grad_norm": 0.061279296875,
+      "learning_rate": 6.679901807431073e-05,
+      "loss": 0.0847695529460907,
+      "mean_token_accuracy": 0.9742431849241256,
+      "num_tokens": 2553527.0,
+      "step": 495
+    },
+    {
+      "entropy": 1.4285764783620833,
+      "epoch": 1.2438569206842924,
+      "grad_norm": 0.044677734375,
+      "learning_rate": 6.616472143798261e-05,
+      "loss": 0.07603438496589661,
+      "mean_token_accuracy": 0.9767945379018783,
+      "num_tokens": 2580242.0,
+      "step": 500
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1206,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.0871202281887744e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8b76582097e27a940c0a231a0d53cf8cdc986bf57bf76ad6533b70015ba5c67
+size 5713

processor_config.json

@@ -0,0 +1,63 @@
+{
+  "image_processor": {
+    "data_format": "channels_first",
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "Qwen2VLImageProcessorFast",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "merge_size": 2,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 23000000,
+      "shortest_edge": 200704
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen2_5_VLProcessor",
+  "video_processor": {
+    "data_format": "channels_first",
+    "default_to_square": true,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": false,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 23000000,
+      "shortest_edge": 200704
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen2VLVideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a7aabe7a3b44651592a99607bb33066e6965260336be9218cccd53af161491d
+size 11422060

tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "_commit_hash": null,
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "is_local": false,
+  "max_length": null,
+  "max_pixels": 23000000,
+  "min_pixels": 200704,
+  "model_max_length": 131072,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "processor_class": "Qwen2_5_VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8b76582097e27a940c0a231a0d53cf8cdc986bf57bf76ad6533b70015ba5c67
+size 5713