add checkpoint otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys

.gitattributes

@@ -317,3 +317,6 @@ checkpoints/cat_qwen25_7b_r8_a32_adamw_e10_lr1e-4_s3_vt_project_a1_ALL_bcast_vs0
 checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_baseline/checkpoint-11250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_baseline/checkpoint-12500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_baseline/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/README.md

@@ -0,0 +1,62 @@
+---
+base_model: google/gemma-3-4b-it
+library_name: peft
+model_name: otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys
+tags:
+- base_model:adapter:google/gemma-3-4b-it
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys
+
+This model is a fine-tuned version of [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/agam-research/huggingface/runs/etan4pss) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.17.1
+- TRL: 0.24.0
+- Transformers: 4.57.1
+- Pytorch: 2.9.0
+- Datasets: 4.3.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-3-4b-it",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "down_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "k_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:124dde30e0999a4ddef4aed38fec80383334d161aab8b177eb5cf5d79311e7a5
+size 65674128

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/chat_template.jinja

@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/README.md

@@ -0,0 +1,209 @@
+---
+base_model: google/gemma-3-4b-it
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:google/gemma-3-4b-it
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-3-4b-it",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "down_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "k_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7187ed8303278b84270c005259bbdb651dcdb139fc343111048fa1984aa2ffac
+size 65674128

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/chat_template.jinja

@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pan_and_scan": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Gemma3ImageProcessor",
+  "image_seq_length": 256,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "pan_and_scan_max_num_crops": null,
+  "pan_and_scan_min_crop_size": null,
+  "pan_and_scan_min_ratio_to_activate": null,
+  "processor_class": "Gemma3Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 896,
+    "width": 896
+  }
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/trainer_state.json

@@ -0,0 +1,2274 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 9.0,
+  "eval_steps": 500,
+  "global_step": 2241,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.671006029844284,
+      "epoch": 0.040160642570281124,
+      "grad_norm": 12.208308219909668,
+      "learning_rate": 7.2e-06,
+      "loss": 0.9984,
+      "mean_token_accuracy": 0.780225270986557,
+      "num_tokens": 10404.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.7024954497814179,
+      "epoch": 0.08032128514056225,
+      "grad_norm": 4.81063175201416,
+      "learning_rate": 1.52e-05,
+      "loss": 0.6915,
+      "mean_token_accuracy": 0.7844282865524292,
+      "num_tokens": 21009.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.7053345918655396,
+      "epoch": 0.12048192771084337,
+      "grad_norm": 5.82667350769043,
+      "learning_rate": 2.32e-05,
+      "loss": 0.5032,
+      "mean_token_accuracy": 0.8207756400108337,
+      "num_tokens": 31309.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.7185693144798279,
+      "epoch": 0.1606425702811245,
+      "grad_norm": 3.646798610687256,
+      "learning_rate": 3.12e-05,
+      "loss": 0.5641,
+      "mean_token_accuracy": 0.796815150976181,
+      "num_tokens": 41509.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.7153907418251038,
+      "epoch": 0.20080321285140562,
+      "grad_norm": 5.238017559051514,
+      "learning_rate": 3.9200000000000004e-05,
+      "loss": 0.4953,
+      "mean_token_accuracy": 0.823300176858902,
+      "num_tokens": 51964.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.7120266735553742,
+      "epoch": 0.24096385542168675,
+      "grad_norm": 4.953554630279541,
+      "learning_rate": 4.72e-05,
+      "loss": 0.4831,
+      "mean_token_accuracy": 0.8243303775787354,
+      "num_tokens": 62281.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.7515081107616425,
+      "epoch": 0.28112449799196787,
+      "grad_norm": 4.73248815536499,
+      "learning_rate": 5.520000000000001e-05,
+      "loss": 0.5529,
+      "mean_token_accuracy": 0.8086587309837341,
+      "num_tokens": 72602.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.7260624289512634,
+      "epoch": 0.321285140562249,
+      "grad_norm": 4.54362678527832,
+      "learning_rate": 6.32e-05,
+      "loss": 0.5278,
+      "mean_token_accuracy": 0.80874263048172,
+      "num_tokens": 82945.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.7128267765045166,
+      "epoch": 0.3614457831325301,
+      "grad_norm": 3.1748130321502686,
+      "learning_rate": 7.12e-05,
+      "loss": 0.4767,
+      "mean_token_accuracy": 0.8299677312374115,
+      "num_tokens": 93192.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.7419508635997772,
+      "epoch": 0.40160642570281124,
+      "grad_norm": 3.0026862621307373,
+      "learning_rate": 7.920000000000001e-05,
+      "loss": 0.5408,
+      "mean_token_accuracy": 0.8037238836288452,
+      "num_tokens": 103603.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.7587903916835785,
+      "epoch": 0.44176706827309237,
+      "grad_norm": 5.152759552001953,
+      "learning_rate": 8.72e-05,
+      "loss": 0.5514,
+      "mean_token_accuracy": 0.8075420558452606,
+      "num_tokens": 113946.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.7834510266780853,
+      "epoch": 0.4819277108433735,
+      "grad_norm": 3.7946877479553223,
+      "learning_rate": 9.52e-05,
+      "loss": 0.577,
+      "mean_token_accuracy": 0.7896877348423004,
+      "num_tokens": 124258.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.7676194608211517,
+      "epoch": 0.5220883534136547,
+      "grad_norm": 4.143728256225586,
+      "learning_rate": 9.999929417599468e-05,
+      "loss": 0.5812,
+      "mean_token_accuracy": 0.7885167539119721,
+      "num_tokens": 134802.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.7672028005123138,
+      "epoch": 0.5622489959839357,
+      "grad_norm": 4.3124847412109375,
+      "learning_rate": 9.999135388478797e-05,
+      "loss": 0.4875,
+      "mean_token_accuracy": 0.8212199211120605,
+      "num_tokens": 145052.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.7765893459320068,
+      "epoch": 0.6024096385542169,
+      "grad_norm": 4.5034074783325195,
+      "learning_rate": 9.997459242813312e-05,
+      "loss": 0.5032,
+      "mean_token_accuracy": 0.8222271859645843,
+      "num_tokens": 155357.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.7953528225421905,
+      "epoch": 0.642570281124498,
+      "grad_norm": 3.0642852783203125,
+      "learning_rate": 9.994901276365323e-05,
+      "loss": 0.5281,
+      "mean_token_accuracy": 0.8077535212039948,
+      "num_tokens": 165640.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.7806783020496368,
+      "epoch": 0.6827309236947792,
+      "grad_norm": 2.9503402709960938,
+      "learning_rate": 9.991461940497786e-05,
+      "loss": 0.52,
+      "mean_token_accuracy": 0.814589935541153,
+      "num_tokens": 175994.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.7847117841243744,
+      "epoch": 0.7228915662650602,
+      "grad_norm": 5.207314491271973,
+      "learning_rate": 9.987141842094658e-05,
+      "loss": 0.517,
+      "mean_token_accuracy": 0.8100123822689056,
+      "num_tokens": 186403.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.744153618812561,
+      "epoch": 0.7630522088353414,
+      "grad_norm": 3.080606698989868,
+      "learning_rate": 9.981941743453815e-05,
+      "loss": 0.4523,
+      "mean_token_accuracy": 0.8381906926631928,
+      "num_tokens": 196638.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.7874633729457855,
+      "epoch": 0.8032128514056225,
+      "grad_norm": 4.700623512268066,
+      "learning_rate": 9.975862562152532e-05,
+      "loss": 0.5551,
+      "mean_token_accuracy": 0.7961149096488953,
+      "num_tokens": 207065.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.7740188777446747,
+      "epoch": 0.8433734939759037,
+      "grad_norm": 3.2746150493621826,
+      "learning_rate": 9.968905370885586e-05,
+      "loss": 0.5106,
+      "mean_token_accuracy": 0.8123311400413513,
+      "num_tokens": 217456.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.7738964557647705,
+      "epoch": 0.8835341365461847,
+      "grad_norm": 3.0882720947265625,
+      "learning_rate": 9.961071397275963e-05,
+      "loss": 0.5226,
+      "mean_token_accuracy": 0.808603173494339,
+      "num_tokens": 227778.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.7768196284770965,
+      "epoch": 0.9236947791164659,
+      "grad_norm": 4.431626796722412,
+      "learning_rate": 9.952362023658249e-05,
+      "loss": 0.4959,
+      "mean_token_accuracy": 0.8225658714771271,
+      "num_tokens": 238074.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.7808157622814178,
+      "epoch": 0.963855421686747,
+      "grad_norm": 2.598005533218384,
+      "learning_rate": 9.9427787868347e-05,
+      "loss": 0.4896,
+      "mean_token_accuracy": 0.8194038093090057,
+      "num_tokens": 248395.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.7777634024620056,
+      "epoch": 1.0040160642570282,
+      "grad_norm": 3.525217056274414,
+      "learning_rate": 9.93232337780408e-05,
+      "loss": 0.4729,
+      "mean_token_accuracy": 0.8291181206703186,
+      "num_tokens": 258505.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.7768383502960206,
+      "epoch": 1.0441767068273093,
+      "grad_norm": 5.895783424377441,
+      "learning_rate": 9.920997641463272e-05,
+      "loss": 0.4785,
+      "mean_token_accuracy": 0.8300446927547455,
+      "num_tokens": 268910.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.7642462968826294,
+      "epoch": 1.0843373493975903,
+      "grad_norm": 3.5049235820770264,
+      "learning_rate": 9.908803576281736e-05,
+      "loss": 0.438,
+      "mean_token_accuracy": 0.8362016320228577,
+      "num_tokens": 279283.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.7406439125537873,
+      "epoch": 1.1244979919678715,
+      "grad_norm": 2.915161371231079,
+      "learning_rate": 9.895743333948874e-05,
+      "loss": 0.4515,
+      "mean_token_accuracy": 0.8348344326019287,
+      "num_tokens": 289866.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.7464889645576477,
+      "epoch": 1.1646586345381527,
+      "grad_norm": 4.682326316833496,
+      "learning_rate": 9.881819218994365e-05,
+      "loss": 0.4514,
+      "mean_token_accuracy": 0.838148581981659,
+      "num_tokens": 300185.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.7421976208686829,
+      "epoch": 1.2048192771084336,
+      "grad_norm": 3.9356069564819336,
+      "learning_rate": 9.867033688381502e-05,
+      "loss": 0.4584,
+      "mean_token_accuracy": 0.8396136164665222,
+      "num_tokens": 310601.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.7389526426792145,
+      "epoch": 1.2449799196787148,
+      "grad_norm": 2.621224880218506,
+      "learning_rate": 9.85138935107367e-05,
+      "loss": 0.4648,
+      "mean_token_accuracy": 0.8362064242362977,
+      "num_tokens": 321070.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.7472976803779602,
+      "epoch": 1.285140562248996,
+      "grad_norm": 3.3467557430267334,
+      "learning_rate": 9.834888967573976e-05,
+      "loss": 0.4906,
+      "mean_token_accuracy": 0.8216790199279785,
+      "num_tokens": 331491.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.7224303126335144,
+      "epoch": 1.3253012048192772,
+      "grad_norm": 3.37907075881958,
+      "learning_rate": 9.817535449438148e-05,
+      "loss": 0.4495,
+      "mean_token_accuracy": 0.8356333911418915,
+      "num_tokens": 341747.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.753093010187149,
+      "epoch": 1.3654618473895583,
+      "grad_norm": 4.384500026702881,
+      "learning_rate": 9.799331858760786e-05,
+      "loss": 0.5321,
+      "mean_token_accuracy": 0.808532440662384,
+      "num_tokens": 352031.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.7213885009288787,
+      "epoch": 1.4056224899598393,
+      "grad_norm": 3.903226852416992,
+      "learning_rate": 9.78028140763503e-05,
+      "loss": 0.4696,
+      "mean_token_accuracy": 0.83351429104805,
+      "num_tokens": 362499.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.7465944051742553,
+      "epoch": 1.4457831325301205,
+      "grad_norm": 3.3047783374786377,
+      "learning_rate": 9.76038745758579e-05,
+      "loss": 0.4657,
+      "mean_token_accuracy": 0.8354173362255096,
+      "num_tokens": 372813.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.7460316002368927,
+      "epoch": 1.4859437751004017,
+      "grad_norm": 3.1054019927978516,
+      "learning_rate": 9.739653518976581e-05,
+      "loss": 0.4636,
+      "mean_token_accuracy": 0.8355995714664459,
+      "num_tokens": 383120.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.7742661297321319,
+      "epoch": 1.5261044176706826,
+      "grad_norm": 3.354398012161255,
+      "learning_rate": 9.718083250390113e-05,
+      "loss": 0.5208,
+      "mean_token_accuracy": 0.8111523389816284,
+      "num_tokens": 393449.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.7555004000663758,
+      "epoch": 1.5662650602409638,
+      "grad_norm": 3.681725263595581,
+      "learning_rate": 9.695680457982713e-05,
+      "loss": 0.4697,
+      "mean_token_accuracy": 0.8350590109825134,
+      "num_tokens": 403803.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.7885475993156433,
+      "epoch": 1.606425702811245,
+      "grad_norm": 3.469503879547119,
+      "learning_rate": 9.67244909481272e-05,
+      "loss": 0.5478,
+      "mean_token_accuracy": 0.8026194393634796,
+      "num_tokens": 414169.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.7571891069412231,
+      "epoch": 1.6465863453815262,
+      "grad_norm": 3.5566561222076416,
+      "learning_rate": 9.648393260142948e-05,
+      "loss": 0.442,
+      "mean_token_accuracy": 0.8380986630916596,
+      "num_tokens": 424543.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.7092450320720672,
+      "epoch": 1.6867469879518073,
+      "grad_norm": 3.590761661529541,
+      "learning_rate": 9.623517198717362e-05,
+      "loss": 0.404,
+      "mean_token_accuracy": 0.8548887431621551,
+      "num_tokens": 434793.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.7402957201004028,
+      "epoch": 1.7269076305220885,
+      "grad_norm": 3.051144599914551,
+      "learning_rate": 9.597825300012073e-05,
+      "loss": 0.4791,
+      "mean_token_accuracy": 0.8277241110801696,
+      "num_tokens": 445085.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.7417145252227784,
+      "epoch": 1.7670682730923695,
+      "grad_norm": 2.8374202251434326,
+      "learning_rate": 9.571322097460793e-05,
+      "loss": 0.4412,
+      "mean_token_accuracy": 0.8435907065868378,
+      "num_tokens": 455314.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.7295144855976105,
+      "epoch": 1.8072289156626506,
+      "grad_norm": 2.723912477493286,
+      "learning_rate": 9.544012267654901e-05,
+      "loss": 0.4406,
+      "mean_token_accuracy": 0.8446374356746673,
+      "num_tokens": 465512.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.7279774129390717,
+      "epoch": 1.8473895582329316,
+      "grad_norm": 3.471815347671509,
+      "learning_rate": 9.51590062951824e-05,
+      "loss": 0.4303,
+      "mean_token_accuracy": 0.8465097844600677,
+      "num_tokens": 475821.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.7734329164028168,
+      "epoch": 1.8875502008032128,
+      "grad_norm": 6.041645526885986,
+      "learning_rate": 9.486992143456792e-05,
+      "loss": 0.5395,
+      "mean_token_accuracy": 0.8112995743751525,
+      "num_tokens": 486060.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.7551539540290833,
+      "epoch": 1.927710843373494,
+      "grad_norm": 3.4140775203704834,
+      "learning_rate": 9.457291910483409e-05,
+      "loss": 0.4588,
+      "mean_token_accuracy": 0.8308178722858429,
+      "num_tokens": 496329.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.7404638409614563,
+      "epoch": 1.9678714859437751,
+      "grad_norm": 3.563293218612671,
+      "learning_rate": 9.426805171317701e-05,
+      "loss": 0.4921,
+      "mean_token_accuracy": 0.8270943462848663,
+      "num_tokens": 506757.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.7283211767673492,
+      "epoch": 2.0080321285140563,
+      "grad_norm": 3.525891065597534,
+      "learning_rate": 9.395537305461311e-05,
+      "loss": 0.4413,
+      "mean_token_accuracy": 0.8409784734249115,
+      "num_tokens": 517020.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.7040447413921356,
+      "epoch": 2.0481927710843375,
+      "grad_norm": 3.779123306274414,
+      "learning_rate": 9.363493830248666e-05,
+      "loss": 0.4035,
+      "mean_token_accuracy": 0.8602916836738587,
+      "num_tokens": 527408.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.6990016818046569,
+      "epoch": 2.0883534136546187,
+      "grad_norm": 3.23069429397583,
+      "learning_rate": 9.33068039987343e-05,
+      "loss": 0.4115,
+      "mean_token_accuracy": 0.85965576171875,
+      "num_tokens": 537684.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.6961205899715424,
+      "epoch": 2.1285140562248994,
+      "grad_norm": 5.430521011352539,
+      "learning_rate": 9.297102804390798e-05,
+      "loss": 0.3504,
+      "mean_token_accuracy": 0.8788332164287567,
+      "num_tokens": 548009.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.6922298431396484,
+      "epoch": 2.1686746987951806,
+      "grad_norm": 3.464811325073242,
+      "learning_rate": 9.26276696869582e-05,
+      "loss": 0.3535,
+      "mean_token_accuracy": 0.8732336580753326,
+      "num_tokens": 558484.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.6996638119220734,
+      "epoch": 2.208835341365462,
+      "grad_norm": 3.1049904823303223,
+      "learning_rate": 9.227678951477925e-05,
+      "loss": 0.4074,
+      "mean_token_accuracy": 0.8610455751419067,
+      "num_tokens": 568796.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.6876482725143432,
+      "epoch": 2.248995983935743,
+      "grad_norm": 3.024766206741333,
+      "learning_rate": 9.19184494415185e-05,
+      "loss": 0.3747,
+      "mean_token_accuracy": 0.866488391160965,
+      "num_tokens": 579189.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.6892669796943665,
+      "epoch": 2.289156626506024,
+      "grad_norm": 3.7265472412109375,
+      "learning_rate": 9.15527126976514e-05,
+      "loss": 0.3709,
+      "mean_token_accuracy": 0.8671182692050934,
+      "num_tokens": 589477.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.7166594922542572,
+      "epoch": 2.3293172690763053,
+      "grad_norm": 2.6233692169189453,
+      "learning_rate": 9.117964381882413e-05,
+      "loss": 0.4437,
+      "mean_token_accuracy": 0.8400287747383117,
+      "num_tokens": 599758.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.7110948503017426,
+      "epoch": 2.3694779116465865,
+      "grad_norm": 2.8130180835723877,
+      "learning_rate": 9.079930863446612e-05,
+      "loss": 0.3918,
+      "mean_token_accuracy": 0.8617593050003052,
+      "num_tokens": 610181.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.6950080335140228,
+      "epoch": 2.4096385542168672,
+      "grad_norm": 6.08152961730957,
+      "learning_rate": 9.041177425617427e-05,
+      "loss": 0.3525,
+      "mean_token_accuracy": 0.8760755777359008,
+      "num_tokens": 620333.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.70909024477005,
+      "epoch": 2.4497991967871484,
+      "grad_norm": 2.8135924339294434,
+      "learning_rate": 9.001710906587064e-05,
+      "loss": 0.3947,
+      "mean_token_accuracy": 0.8637883484363555,
+      "num_tokens": 630697.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.7154293119907379,
+      "epoch": 2.4899598393574296,
+      "grad_norm": 3.2165870666503906,
+      "learning_rate": 8.961538270373639e-05,
+      "loss": 0.4164,
+      "mean_token_accuracy": 0.8475094556808471,
+      "num_tokens": 641231.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.7277772426605225,
+      "epoch": 2.5301204819277108,
+      "grad_norm": 5.162221908569336,
+      "learning_rate": 8.920666605592341e-05,
+      "loss": 0.4189,
+      "mean_token_accuracy": 0.8539318025112153,
+      "num_tokens": 651543.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.675065678358078,
+      "epoch": 2.570281124497992,
+      "grad_norm": 2.9395062923431396,
+      "learning_rate": 8.879103124204626e-05,
+      "loss": 0.3421,
+      "mean_token_accuracy": 0.881534194946289,
+      "num_tokens": 661830.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.6800589621067047,
+      "epoch": 2.610441767068273,
+      "grad_norm": 5.408241271972656,
+      "learning_rate": 8.836855160245629e-05,
+      "loss": 0.3392,
+      "mean_token_accuracy": 0.8793375313282012,
+      "num_tokens": 672149.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.6833930969238281,
+      "epoch": 2.6506024096385543,
+      "grad_norm": 5.121169567108154,
+      "learning_rate": 8.79393016853005e-05,
+      "loss": 0.4217,
+      "mean_token_accuracy": 0.8490349233150483,
+      "num_tokens": 682537.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.6918147087097168,
+      "epoch": 2.6907630522088355,
+      "grad_norm": 3.5816986560821533,
+      "learning_rate": 8.750335723336728e-05,
+      "loss": 0.408,
+      "mean_token_accuracy": 0.8592755615711212,
+      "num_tokens": 692836.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.6768696010112762,
+      "epoch": 2.7309236947791167,
+      "grad_norm": 4.3298492431640625,
+      "learning_rate": 8.706079517072127e-05,
+      "loss": 0.3447,
+      "mean_token_accuracy": 0.8745476543903351,
+      "num_tokens": 703107.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.6964600443840027,
+      "epoch": 2.7710843373493974,
+      "grad_norm": 5.028672218322754,
+      "learning_rate": 8.661169358912978e-05,
+      "loss": 0.4352,
+      "mean_token_accuracy": 0.8437578022480011,
+      "num_tokens": 713574.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.7136080145835877,
+      "epoch": 2.8112449799196786,
+      "grad_norm": 3.92950701713562,
+      "learning_rate": 8.615613173428321e-05,
+      "loss": 0.4112,
+      "mean_token_accuracy": 0.8544917225837707,
+      "num_tokens": 724034.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.7157581627368927,
+      "epoch": 2.8514056224899598,
+      "grad_norm": 3.4689905643463135,
+      "learning_rate": 8.569418999181194e-05,
+      "loss": 0.4164,
+      "mean_token_accuracy": 0.8549265265464783,
+      "num_tokens": 734413.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.7330816447734833,
+      "epoch": 2.891566265060241,
+      "grad_norm": 4.131735324859619,
+      "learning_rate": 8.522594987310184e-05,
+      "loss": 0.439,
+      "mean_token_accuracy": 0.8389688909053803,
+      "num_tokens": 744798.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.6790109813213349,
+      "epoch": 2.931726907630522,
+      "grad_norm": 3.378662586212158,
+      "learning_rate": 8.475149400091137e-05,
+      "loss": 0.362,
+      "mean_token_accuracy": 0.8758748531341553,
+      "num_tokens": 754977.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.6692682385444642,
+      "epoch": 2.9718875502008033,
+      "grad_norm": 3.1824755668640137,
+      "learning_rate": 8.427090609479245e-05,
+      "loss": 0.344,
+      "mean_token_accuracy": 0.8847700834274292,
+      "num_tokens": 765400.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.6895963490009308,
+      "epoch": 3.0120481927710845,
+      "grad_norm": 4.070644378662109,
+      "learning_rate": 8.378427095631776e-05,
+      "loss": 0.4177,
+      "mean_token_accuracy": 0.8527476012706756,
+      "num_tokens": 775375.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.6410589516162872,
+      "epoch": 3.0522088353413657,
+      "grad_norm": 3.8965253829956055,
+      "learning_rate": 8.329167445411732e-05,
+      "loss": 0.2976,
+      "mean_token_accuracy": 0.8942947745323181,
+      "num_tokens": 785714.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.6333921432495118,
+      "epoch": 3.0923694779116464,
+      "grad_norm": 5.164280891418457,
+      "learning_rate": 8.279320350872655e-05,
+      "loss": 0.337,
+      "mean_token_accuracy": 0.8851125121116639,
+      "num_tokens": 796081.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.6316083431243896,
+      "epoch": 3.1325301204819276,
+      "grad_norm": 5.71152925491333,
+      "learning_rate": 8.228894607724878e-05,
+      "loss": 0.2859,
+      "mean_token_accuracy": 0.8993970811367035,
+      "num_tokens": 806391.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.6212240159511566,
+      "epoch": 3.1726907630522088,
+      "grad_norm": 4.353719711303711,
+      "learning_rate": 8.177899113783492e-05,
+      "loss": 0.3073,
+      "mean_token_accuracy": 0.888391625881195,
+      "num_tokens": 816801.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.6379825949668885,
+      "epoch": 3.21285140562249,
+      "grad_norm": 4.2553300857543945,
+      "learning_rate": 8.126342867398301e-05,
+      "loss": 0.3058,
+      "mean_token_accuracy": 0.8925197720527649,
+      "num_tokens": 827206.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.6057499378919602,
+      "epoch": 3.253012048192771,
+      "grad_norm": 3.8955018520355225,
+      "learning_rate": 8.074234965866012e-05,
+      "loss": 0.2812,
+      "mean_token_accuracy": 0.9016481280326843,
+      "num_tokens": 837325.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.628681194782257,
+      "epoch": 3.2931726907630523,
+      "grad_norm": 3.5182700157165527,
+      "learning_rate": 8.021584603824996e-05,
+      "loss": 0.2604,
+      "mean_token_accuracy": 0.9060991644859314,
+      "num_tokens": 847765.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.6285995721817017,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 4.679093837738037,
+      "learning_rate": 7.968401071632855e-05,
+      "loss": 0.3321,
+      "mean_token_accuracy": 0.8817380368709564,
+      "num_tokens": 858303.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.6240448355674744,
+      "epoch": 3.3734939759036147,
+      "grad_norm": 4.280148506164551,
+      "learning_rate": 7.914693753727091e-05,
+      "loss": 0.2971,
+      "mean_token_accuracy": 0.89331756234169,
+      "num_tokens": 868618.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.6346171617507934,
+      "epoch": 3.4136546184738954,
+      "grad_norm": 5.035059452056885,
+      "learning_rate": 7.860472126969213e-05,
+      "loss": 0.283,
+      "mean_token_accuracy": 0.8987699389457703,
+      "num_tokens": 879028.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.6339222490787506,
+      "epoch": 3.4538152610441766,
+      "grad_norm": 3.3308496475219727,
+      "learning_rate": 7.805745758972481e-05,
+      "loss": 0.3369,
+      "mean_token_accuracy": 0.8849397778511048,
+      "num_tokens": 889513.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.654508638381958,
+      "epoch": 3.4939759036144578,
+      "grad_norm": 4.176350116729736,
+      "learning_rate": 7.75052430641368e-05,
+      "loss": 0.3435,
+      "mean_token_accuracy": 0.8724894464015961,
+      "num_tokens": 900038.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.6320246398448944,
+      "epoch": 3.534136546184739,
+      "grad_norm": 4.780612945556641,
+      "learning_rate": 7.694817513329159e-05,
+      "loss": 0.2909,
+      "mean_token_accuracy": 0.894708800315857,
+      "num_tokens": 910418.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.6431657671928406,
+      "epoch": 3.57429718875502,
+      "grad_norm": 5.155822277069092,
+      "learning_rate": 7.638635209395453e-05,
+      "loss": 0.3065,
+      "mean_token_accuracy": 0.8888295769691468,
+      "num_tokens": 920627.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.6216733515262604,
+      "epoch": 3.6144578313253013,
+      "grad_norm": 4.907107830047607,
+      "learning_rate": 7.58198730819481e-05,
+      "loss": 0.2597,
+      "mean_token_accuracy": 0.904644924402237,
+      "num_tokens": 930895.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.6459767282009125,
+      "epoch": 3.6546184738955825,
+      "grad_norm": 4.625438213348389,
+      "learning_rate": 7.524883805465888e-05,
+      "loss": 0.3488,
+      "mean_token_accuracy": 0.8783864676952362,
+      "num_tokens": 941195.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.6570597350597381,
+      "epoch": 3.694779116465863,
+      "grad_norm": 3.448225259780884,
+      "learning_rate": 7.467334777339985e-05,
+      "loss": 0.3313,
+      "mean_token_accuracy": 0.888871556520462,
+      "num_tokens": 951345.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.6195529520511627,
+      "epoch": 3.734939759036145,
+      "grad_norm": 5.21084451675415,
+      "learning_rate": 7.409350378563046e-05,
+      "loss": 0.2899,
+      "mean_token_accuracy": 0.896889615058899,
+      "num_tokens": 961633.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.6462969362735749,
+      "epoch": 3.7751004016064256,
+      "grad_norm": 3.8568060398101807,
+      "learning_rate": 7.350940840703842e-05,
+      "loss": 0.2914,
+      "mean_token_accuracy": 0.8919413447380066,
+      "num_tokens": 971973.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.6068174064159393,
+      "epoch": 3.8152610441767068,
+      "grad_norm": 4.7789082527160645,
+      "learning_rate": 7.292116470348554e-05,
+      "loss": 0.2713,
+      "mean_token_accuracy": 0.9051806092262268,
+      "num_tokens": 982359.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.6487982928752899,
+      "epoch": 3.855421686746988,
+      "grad_norm": 3.895387887954712,
+      "learning_rate": 7.232887647282147e-05,
+      "loss": 0.328,
+      "mean_token_accuracy": 0.8847681939601898,
+      "num_tokens": 992792.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.6167472541332245,
+      "epoch": 3.895582329317269,
+      "grad_norm": 5.2591328620910645,
+      "learning_rate": 7.173264822656806e-05,
+      "loss": 0.2843,
+      "mean_token_accuracy": 0.8959703624248505,
+      "num_tokens": 1003118.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.6505671739578247,
+      "epoch": 3.9357429718875503,
+      "grad_norm": 3.341792106628418,
+      "learning_rate": 7.113258517147801e-05,
+      "loss": 0.3449,
+      "mean_token_accuracy": 0.8760739207267761,
+      "num_tokens": 1013518.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.6415602684020996,
+      "epoch": 3.9759036144578315,
+      "grad_norm": 3.5312533378601074,
+      "learning_rate": 7.052879319097072e-05,
+      "loss": 0.3403,
+      "mean_token_accuracy": 0.879234254360199,
+      "num_tokens": 1023763.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.6272035002708435,
+      "epoch": 4.016064257028113,
+      "grad_norm": 2.167412519454956,
+      "learning_rate": 6.992137882644868e-05,
+      "loss": 0.2459,
+      "mean_token_accuracy": 0.9122343599796295,
+      "num_tokens": 1034043.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.5781412482261657,
+      "epoch": 4.056224899598393,
+      "grad_norm": 4.918298721313477,
+      "learning_rate": 6.931044925849789e-05,
+      "loss": 0.2183,
+      "mean_token_accuracy": 0.9209221661090851,
+      "num_tokens": 1044418.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.6048957705497742,
+      "epoch": 4.096385542168675,
+      "grad_norm": 4.167297840118408,
+      "learning_rate": 6.869611228797546e-05,
+      "loss": 0.2328,
+      "mean_token_accuracy": 0.9181684672832489,
+      "num_tokens": 1054828.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.6079785764217377,
+      "epoch": 4.136546184738956,
+      "grad_norm": 3.8995273113250732,
+      "learning_rate": 6.807847631698769e-05,
+      "loss": 0.2689,
+      "mean_token_accuracy": 0.908103734254837,
+      "num_tokens": 1065085.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.5601828396320343,
+      "epoch": 4.176706827309237,
+      "grad_norm": 4.611727237701416,
+      "learning_rate": 6.745765032976214e-05,
+      "loss": 0.1729,
+      "mean_token_accuracy": 0.9361172139644622,
+      "num_tokens": 1075280.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.5618009567260742,
+      "epoch": 4.216867469879518,
+      "grad_norm": 3.4976491928100586,
+      "learning_rate": 6.683374387341687e-05,
+      "loss": 0.1912,
+      "mean_token_accuracy": 0.9356351375579834,
+      "num_tokens": 1085467.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.5666621267795563,
+      "epoch": 4.257028112449799,
+      "grad_norm": 3.093925952911377,
+      "learning_rate": 6.620686703863054e-05,
+      "loss": 0.1925,
+      "mean_token_accuracy": 0.9300771534442902,
+      "num_tokens": 1095673.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.5858176440000534,
+      "epoch": 4.2971887550200805,
+      "grad_norm": 6.102444171905518,
+      "learning_rate": 6.557713044021642e-05,
+      "loss": 0.2351,
+      "mean_token_accuracy": 0.9199433982372284,
+      "num_tokens": 1106078.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.5875226140022278,
+      "epoch": 4.337349397590361,
+      "grad_norm": 2.7948408126831055,
+      "learning_rate": 6.494464519760401e-05,
+      "loss": 0.229,
+      "mean_token_accuracy": 0.917472368478775,
+      "num_tokens": 1116424.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.5854913651943207,
+      "epoch": 4.377510040160643,
+      "grad_norm": 2.9702138900756836,
+      "learning_rate": 6.430952291523158e-05,
+      "loss": 0.2509,
+      "mean_token_accuracy": 0.9111802577972412,
+      "num_tokens": 1126660.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.6019260823726654,
+      "epoch": 4.417670682730924,
+      "grad_norm": 5.375982761383057,
+      "learning_rate": 6.367187566285314e-05,
+      "loss": 0.2454,
+      "mean_token_accuracy": 0.9157536566257477,
+      "num_tokens": 1137074.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.5940370619297027,
+      "epoch": 4.457831325301205,
+      "grad_norm": 4.331886291503906,
+      "learning_rate": 6.303181595576328e-05,
+      "loss": 0.2251,
+      "mean_token_accuracy": 0.9235850930213928,
+      "num_tokens": 1147307.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.5850259006023407,
+      "epoch": 4.497991967871486,
+      "grad_norm": 4.5053558349609375,
+      "learning_rate": 6.238945673494354e-05,
+      "loss": 0.2473,
+      "mean_token_accuracy": 0.9141446411609649,
+      "num_tokens": 1157629.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.5875584840774536,
+      "epoch": 4.538152610441767,
+      "grad_norm": 3.1502726078033447,
+      "learning_rate": 6.174491134713332e-05,
+      "loss": 0.2062,
+      "mean_token_accuracy": 0.9300969243049622,
+      "num_tokens": 1168055.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.5655186951160431,
+      "epoch": 4.578313253012048,
+      "grad_norm": 5.992119312286377,
+      "learning_rate": 6.109829352482964e-05,
+      "loss": 0.1918,
+      "mean_token_accuracy": 0.9329712331295014,
+      "num_tokens": 1178548.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.5675104677677154,
+      "epoch": 4.618473895582329,
+      "grad_norm": 3.7070045471191406,
+      "learning_rate": 6.044971736621842e-05,
+      "loss": 0.2103,
+      "mean_token_accuracy": 0.9285524368286133,
+      "num_tokens": 1188856.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.557010006904602,
+      "epoch": 4.658634538152611,
+      "grad_norm": 3.5083024501800537,
+      "learning_rate": 5.979929731504158e-05,
+      "loss": 0.1831,
+      "mean_token_accuracy": 0.9354606866836548,
+      "num_tokens": 1199055.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.5573005437850952,
+      "epoch": 4.698795180722891,
+      "grad_norm": 4.800366401672363,
+      "learning_rate": 5.91471481404029e-05,
+      "loss": 0.1905,
+      "mean_token_accuracy": 0.9361295998096466,
+      "num_tokens": 1209505.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.582137656211853,
+      "epoch": 4.738955823293173,
+      "grad_norm": 3.2478320598602295,
+      "learning_rate": 5.849338491651661e-05,
+      "loss": 0.23,
+      "mean_token_accuracy": 0.9177156090736389,
+      "num_tokens": 1219934.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.5620950549840927,
+      "epoch": 4.779116465863454,
+      "grad_norm": 3.2152435779571533,
+      "learning_rate": 5.783812300240209e-05,
+      "loss": 0.19,
+      "mean_token_accuracy": 0.9312011301517487,
+      "num_tokens": 1230388.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.5742540836334229,
+      "epoch": 4.8192771084337345,
+      "grad_norm": 4.549374580383301,
+      "learning_rate": 5.718147802152833e-05,
+      "loss": 0.2103,
+      "mean_token_accuracy": 0.9266662240028382,
+      "num_tokens": 1240833.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.571652227640152,
+      "epoch": 4.859437751004016,
+      "grad_norm": 4.648577690124512,
+      "learning_rate": 5.652356584141177e-05,
+      "loss": 0.2216,
+      "mean_token_accuracy": 0.9185182213783264,
+      "num_tokens": 1251204.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.570747435092926,
+      "epoch": 4.899598393574297,
+      "grad_norm": 4.546886444091797,
+      "learning_rate": 5.586450255317097e-05,
+      "loss": 0.2055,
+      "mean_token_accuracy": 0.9270457446575164,
+      "num_tokens": 1261627.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.5852534890174865,
+      "epoch": 4.9397590361445785,
+      "grad_norm": 3.8608083724975586,
+      "learning_rate": 5.5204404451041894e-05,
+      "loss": 0.223,
+      "mean_token_accuracy": 0.9219282686710357,
+      "num_tokens": 1271968.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.5844241499900817,
+      "epoch": 4.979919678714859,
+      "grad_norm": 4.815842628479004,
+      "learning_rate": 5.4543388011857456e-05,
+      "loss": 0.242,
+      "mean_token_accuracy": 0.9152222752571106,
+      "num_tokens": 1282256.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.5457853674888611,
+      "epoch": 5.020080321285141,
+      "grad_norm": 3.850184202194214,
+      "learning_rate": 5.388156987449454e-05,
+      "loss": 0.1904,
+      "mean_token_accuracy": 0.936645758152008,
+      "num_tokens": 1292404.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.5469618558883667,
+      "epoch": 5.0602409638554215,
+      "grad_norm": 3.2533013820648193,
+      "learning_rate": 5.321906681929284e-05,
+      "loss": 0.1824,
+      "mean_token_accuracy": 0.9348189115524292,
+      "num_tokens": 1302880.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.525107690691948,
+      "epoch": 5.100401606425703,
+      "grad_norm": 3.654391050338745,
+      "learning_rate": 5.2555995747448364e-05,
+      "loss": 0.1456,
+      "mean_token_accuracy": 0.9505318880081177,
+      "num_tokens": 1312867.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.5398725032806396,
+      "epoch": 5.140562248995984,
+      "grad_norm": 6.178979396820068,
+      "learning_rate": 5.189247366038583e-05,
+      "loss": 0.1306,
+      "mean_token_accuracy": 0.9524036228656769,
+      "num_tokens": 1323359.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.5428184479475021,
+      "epoch": 5.180722891566265,
+      "grad_norm": 4.637774467468262,
+      "learning_rate": 5.1228617639113355e-05,
+      "loss": 0.1519,
+      "mean_token_accuracy": 0.9475317895412445,
+      "num_tokens": 1333647.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.5280115365982055,
+      "epoch": 5.220883534136546,
+      "grad_norm": 4.078694820404053,
+      "learning_rate": 5.0564544823562945e-05,
+      "loss": 0.1284,
+      "mean_token_accuracy": 0.9562457025051116,
+      "num_tokens": 1343798.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.5279791682958603,
+      "epoch": 5.261044176706827,
+      "grad_norm": 4.2383294105529785,
+      "learning_rate": 4.9900372391920875e-05,
+      "loss": 0.1524,
+      "mean_token_accuracy": 0.9461738169193268,
+      "num_tokens": 1354236.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.5655446887016297,
+      "epoch": 5.301204819277109,
+      "grad_norm": 2.5317718982696533,
+      "learning_rate": 4.923621753995099e-05,
+      "loss": 0.2122,
+      "mean_token_accuracy": 0.9245820283889771,
+      "num_tokens": 1364737.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.5198477387428284,
+      "epoch": 5.341365461847389,
+      "grad_norm": 2.0545156002044678,
+      "learning_rate": 4.85721974603152e-05,
+      "loss": 0.1256,
+      "mean_token_accuracy": 0.9565123438835144,
+      "num_tokens": 1375031.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.5188256084918976,
+      "epoch": 5.381526104417671,
+      "grad_norm": 4.323302268981934,
+      "learning_rate": 4.790842932189424e-05,
+      "loss": 0.1062,
+      "mean_token_accuracy": 0.9651807725429535,
+      "num_tokens": 1385323.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.5091028660535812,
+      "epoch": 5.421686746987952,
+      "grad_norm": 4.59332275390625,
+      "learning_rate": 4.724503024911292e-05,
+      "loss": 0.1441,
+      "mean_token_accuracy": 0.9536083936691284,
+      "num_tokens": 1395508.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.5363169133663177,
+      "epoch": 5.461847389558233,
+      "grad_norm": 4.5257673263549805,
+      "learning_rate": 4.6582117301273006e-05,
+      "loss": 0.1402,
+      "mean_token_accuracy": 0.9518926918506623,
+      "num_tokens": 1405849.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.49726691246032717,
+      "epoch": 5.502008032128514,
+      "grad_norm": 2.6741883754730225,
+      "learning_rate": 4.591980745189762e-05,
+      "loss": 0.1268,
+      "mean_token_accuracy": 0.9559646964073181,
+      "num_tokens": 1416297.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.5196024924516678,
+      "epoch": 5.542168674698795,
+      "grad_norm": 4.707968235015869,
+      "learning_rate": 4.5258217568090876e-05,
+      "loss": 0.1174,
+      "mean_token_accuracy": 0.9585049390792847,
+      "num_tokens": 1426589.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.5129575550556182,
+      "epoch": 5.582329317269076,
+      "grad_norm": 5.751727104187012,
+      "learning_rate": 4.4597464389916204e-05,
+      "loss": 0.1184,
+      "mean_token_accuracy": 0.9612732768058777,
+      "num_tokens": 1436934.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.49982237815856934,
+      "epoch": 5.622489959839357,
+      "grad_norm": 2.4615159034729004,
+      "learning_rate": 4.3937664509797173e-05,
+      "loss": 0.0975,
+      "mean_token_accuracy": 0.9674801647663116,
+      "num_tokens": 1447105.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.5146512746810913,
+      "epoch": 5.662650602409639,
+      "grad_norm": 5.416796684265137,
+      "learning_rate": 4.3278934351944185e-05,
+      "loss": 0.1276,
+      "mean_token_accuracy": 0.9571470499038697,
+      "num_tokens": 1457497.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.5148198366165161,
+      "epoch": 5.7028112449799195,
+      "grad_norm": 3.361555576324463,
+      "learning_rate": 4.262139015181111e-05,
+      "loss": 0.137,
+      "mean_token_accuracy": 0.9535258531570434,
+      "num_tokens": 1467837.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.510842016339302,
+      "epoch": 5.742971887550201,
+      "grad_norm": 5.052914142608643,
+      "learning_rate": 4.196514793558508e-05,
+      "loss": 0.0978,
+      "mean_token_accuracy": 0.9678477287292481,
+      "num_tokens": 1478244.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.5095056742429733,
+      "epoch": 5.783132530120482,
+      "grad_norm": 3.247624158859253,
+      "learning_rate": 4.13103234997131e-05,
+      "loss": 0.1332,
+      "mean_token_accuracy": 0.9573037326335907,
+      "num_tokens": 1488557.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.5327902555465698,
+      "epoch": 5.823293172690763,
+      "grad_norm": 3.7884256839752197,
+      "learning_rate": 4.065703239046951e-05,
+      "loss": 0.1305,
+      "mean_token_accuracy": 0.9526015043258667,
+      "num_tokens": 1498946.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.5489759087562561,
+      "epoch": 5.863453815261044,
+      "grad_norm": 4.858087062835693,
+      "learning_rate": 4.000538988356723e-05,
+      "loss": 0.1887,
+      "mean_token_accuracy": 0.9359465658664703,
+      "num_tokens": 1509535.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.5159505128860473,
+      "epoch": 5.903614457831325,
+      "grad_norm": 3.627187967300415,
+      "learning_rate": 3.9355510963817046e-05,
+      "loss": 0.1177,
+      "mean_token_accuracy": 0.9571630418300628,
+      "num_tokens": 1520015.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.5169980227947235,
+      "epoch": 5.943775100401607,
+      "grad_norm": 5.518531322479248,
+      "learning_rate": 3.8707510304838e-05,
+      "loss": 0.134,
+      "mean_token_accuracy": 0.9587558329105377,
+      "num_tokens": 1530486.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.5385745912790298,
+      "epoch": 5.983935742971887,
+      "grad_norm": 4.262823104858398,
+      "learning_rate": 3.80615022488229e-05,
+      "loss": 0.1387,
+      "mean_token_accuracy": 0.9486428678035737,
+      "num_tokens": 1540904.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.5053048670291901,
+      "epoch": 6.024096385542169,
+      "grad_norm": 1.9068161249160767,
+      "learning_rate": 3.741760078636213e-05,
+      "loss": 0.0969,
+      "mean_token_accuracy": 0.9685157060623169,
+      "num_tokens": 1550929.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.483124241232872,
+      "epoch": 6.06425702811245,
+      "grad_norm": 5.557836532592773,
+      "learning_rate": 3.677591953632955e-05,
+      "loss": 0.0725,
+      "mean_token_accuracy": 0.9760768055915833,
+      "num_tokens": 1561399.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.47552241683006286,
+      "epoch": 6.104417670682731,
+      "grad_norm": 6.154554843902588,
+      "learning_rate": 3.6136571725834116e-05,
+      "loss": 0.0617,
+      "mean_token_accuracy": 0.9800824522972107,
+      "num_tokens": 1571684.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.4706099838018417,
+      "epoch": 6.144578313253012,
+      "grad_norm": 1.2171343564987183,
+      "learning_rate": 3.5499670170240395e-05,
+      "loss": 0.0677,
+      "mean_token_accuracy": 0.9764799892902374,
+      "num_tokens": 1581923.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.4783856838941574,
+      "epoch": 6.184738955823293,
+      "grad_norm": 5.3933281898498535,
+      "learning_rate": 3.486532725326199e-05,
+      "loss": 0.0899,
+      "mean_token_accuracy": 0.9701887845993042,
+      "num_tokens": 1592316.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.45972216725349424,
+      "epoch": 6.224899598393574,
+      "grad_norm": 3.5178890228271484,
+      "learning_rate": 3.4233654907130875e-05,
+      "loss": 0.0419,
+      "mean_token_accuracy": 0.9859755575656891,
+      "num_tokens": 1602441.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.4836335092782974,
+      "epoch": 6.265060240963855,
+      "grad_norm": 4.9517292976379395,
+      "learning_rate": 3.3604764592846636e-05,
+      "loss": 0.0936,
+      "mean_token_accuracy": 0.9697192490100861,
+      "num_tokens": 1612777.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.48367559611797334,
+      "epoch": 6.305220883534137,
+      "grad_norm": 2.2491259574890137,
+      "learning_rate": 3.2978767280508736e-05,
+      "loss": 0.0856,
+      "mean_token_accuracy": 0.9715181827545166,
+      "num_tokens": 1623253.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.46522434055805206,
+      "epoch": 6.3453815261044175,
+      "grad_norm": 4.534587383270264,
+      "learning_rate": 3.2355773429735314e-05,
+      "loss": 0.0798,
+      "mean_token_accuracy": 0.971975964307785,
+      "num_tokens": 1633666.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.4706613391637802,
+      "epoch": 6.385542168674699,
+      "grad_norm": 4.868239402770996,
+      "learning_rate": 3.17358929701723e-05,
+      "loss": 0.0798,
+      "mean_token_accuracy": 0.973397958278656,
+      "num_tokens": 1644122.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.4904076397418976,
+      "epoch": 6.42570281124498,
+      "grad_norm": 4.480022430419922,
+      "learning_rate": 3.111923528209577e-05,
+      "loss": 0.1086,
+      "mean_token_accuracy": 0.9585139930248261,
+      "num_tokens": 1654430.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.4636099249124527,
+      "epoch": 6.4658634538152615,
+      "grad_norm": 3.2263810634613037,
+      "learning_rate": 3.0505909177111574e-05,
+      "loss": 0.0822,
+      "mean_token_accuracy": 0.9725425720214844,
+      "num_tokens": 1664678.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.46650232672691344,
+      "epoch": 6.506024096385542,
+      "grad_norm": 2.583544969558716,
+      "learning_rate": 2.9896022878954878e-05,
+      "loss": 0.0965,
+      "mean_token_accuracy": 0.9650086343288422,
+      "num_tokens": 1675124.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.46981877982616427,
+      "epoch": 6.546184738955823,
+      "grad_norm": 3.799302577972412,
+      "learning_rate": 2.9289684004393836e-05,
+      "loss": 0.0901,
+      "mean_token_accuracy": 0.973429948091507,
+      "num_tokens": 1685664.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.47693943679332734,
+      "epoch": 6.586345381526105,
+      "grad_norm": 4.88773250579834,
+      "learning_rate": 2.8686999544240172e-05,
+      "loss": 0.1032,
+      "mean_token_accuracy": 0.9664510011672973,
+      "num_tokens": 1695864.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.4770108133554459,
+      "epoch": 6.626506024096385,
+      "grad_norm": 4.233675003051758,
+      "learning_rate": 2.808807584447018e-05,
+      "loss": 0.0918,
+      "mean_token_accuracy": 0.9709401428699493,
+      "num_tokens": 1706243.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.4650855600833893,
+      "epoch": 6.666666666666667,
+      "grad_norm": 3.436919927597046,
+      "learning_rate": 2.7493018587459628e-05,
+      "loss": 0.0842,
+      "mean_token_accuracy": 0.9706488728523255,
+      "num_tokens": 1716654.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.47103855907917025,
+      "epoch": 6.706827309236948,
+      "grad_norm": 3.286376476287842,
+      "learning_rate": 2.6901932773335692e-05,
+      "loss": 0.0748,
+      "mean_token_accuracy": 0.9748601496219635,
+      "num_tokens": 1727067.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.4630146950483322,
+      "epoch": 6.746987951807229,
+      "grad_norm": 2.7392168045043945,
+      "learning_rate": 2.6314922701449286e-05,
+      "loss": 0.0491,
+      "mean_token_accuracy": 0.983652138710022,
+      "num_tokens": 1737275.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.47450789213180544,
+      "epoch": 6.78714859437751,
+      "grad_norm": 1.7087758779525757,
+      "learning_rate": 2.5732091951970937e-05,
+      "loss": 0.0729,
+      "mean_token_accuracy": 0.9769853889942169,
+      "num_tokens": 1747828.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.4613141298294067,
+      "epoch": 6.827309236947791,
+      "grad_norm": 5.062620162963867,
+      "learning_rate": 2.515354336761391e-05,
+      "loss": 0.0812,
+      "mean_token_accuracy": 0.9709953427314758,
+      "num_tokens": 1757997.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.46983512938022615,
+      "epoch": 6.867469879518072,
+      "grad_norm": 3.5601933002471924,
+      "learning_rate": 2.457937903548695e-05,
+      "loss": 0.0895,
+      "mean_token_accuracy": 0.9698527097702027,
+      "num_tokens": 1768333.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.4738364279270172,
+      "epoch": 6.907630522088353,
+      "grad_norm": 3.817272186279297,
+      "learning_rate": 2.4009700269080793e-05,
+      "loss": 0.1011,
+      "mean_token_accuracy": 0.9639538288116455,
+      "num_tokens": 1778641.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.48017253577709196,
+      "epoch": 6.947791164658635,
+      "grad_norm": 2.8202097415924072,
+      "learning_rate": 2.344460759039097e-05,
+      "loss": 0.0793,
+      "mean_token_accuracy": 0.9698579728603363,
+      "num_tokens": 1788864.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.46426438093185424,
+      "epoch": 6.9879518072289155,
+      "grad_norm": 1.4748797416687012,
+      "learning_rate": 2.2884200712180227e-05,
+      "loss": 0.0697,
+      "mean_token_accuracy": 0.9751004636287689,
+      "num_tokens": 1799265.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.45316306948661805,
+      "epoch": 7.028112449799197,
+      "grad_norm": 2.021364212036133,
+      "learning_rate": 2.2328578520384037e-05,
+      "loss": 0.0507,
+      "mean_token_accuracy": 0.9841875195503235,
+      "num_tokens": 1809278.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.45254839658737184,
+      "epoch": 7.068273092369478,
+      "grad_norm": 3.159297227859497,
+      "learning_rate": 2.1777839056661554e-05,
+      "loss": 0.0505,
+      "mean_token_accuracy": 0.9822456657886505,
+      "num_tokens": 1819456.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.4383044600486755,
+      "epoch": 7.108433734939759,
+      "grad_norm": 2.410841226577759,
+      "learning_rate": 2.123207950109596e-05,
+      "loss": 0.0369,
+      "mean_token_accuracy": 0.9873785614967346,
+      "num_tokens": 1829725.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.44104948341846467,
+      "epoch": 7.14859437751004,
+      "grad_norm": 1.1177611351013184,
+      "learning_rate": 2.0691396155046595e-05,
+      "loss": 0.0397,
+      "mean_token_accuracy": 0.9877350866794586,
+      "num_tokens": 1839985.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.46033324897289274,
+      "epoch": 7.188755020080321,
+      "grad_norm": 2.4915175437927246,
+      "learning_rate": 2.0155884424156242e-05,
+      "loss": 0.0653,
+      "mean_token_accuracy": 0.9802395641803742,
+      "num_tokens": 1850362.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.42928866147994993,
+      "epoch": 7.228915662650603,
+      "grad_norm": 1.9723711013793945,
+      "learning_rate": 1.9625638801516407e-05,
+      "loss": 0.0324,
+      "mean_token_accuracy": 0.9898811101913452,
+      "num_tokens": 1860696.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.42099535167217256,
+      "epoch": 7.269076305220883,
+      "grad_norm": 5.293099403381348,
+      "learning_rate": 1.9100752850993687e-05,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9883219778537751,
+      "num_tokens": 1871064.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.42980838418006895,
+      "epoch": 7.309236947791165,
+      "grad_norm": 1.2444225549697876,
+      "learning_rate": 1.8581319190720035e-05,
+      "loss": 0.0289,
+      "mean_token_accuracy": 0.9919730424880981,
+      "num_tokens": 1881351.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.4080615252256393,
+      "epoch": 7.349397590361446,
+      "grad_norm": 0.3096483647823334,
+      "learning_rate": 1.806742947674997e-05,
+      "loss": 0.0259,
+      "mean_token_accuracy": 0.9917362213134766,
+      "num_tokens": 1891725.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.4294892728328705,
+      "epoch": 7.389558232931727,
+      "grad_norm": 1.8443820476531982,
+      "learning_rate": 1.7559174386887477e-05,
+      "loss": 0.0479,
+      "mean_token_accuracy": 0.9846168875694274,
+      "num_tokens": 1902115.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.4191869467496872,
+      "epoch": 7.429718875502008,
+      "grad_norm": 2.9208106994628906,
+      "learning_rate": 1.7056643604685596e-05,
+      "loss": 0.0333,
+      "mean_token_accuracy": 0.990466034412384,
+      "num_tokens": 1912519.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.4225724130868912,
+      "epoch": 7.469879518072289,
+      "grad_norm": 1.1033703088760376,
+      "learning_rate": 1.65599258036214e-05,
+      "loss": 0.0252,
+      "mean_token_accuracy": 0.9928982973098754,
+      "num_tokens": 1922875.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.43391208052635194,
+      "epoch": 7.51004016064257,
+      "grad_norm": 2.3476970195770264,
+      "learning_rate": 1.6069108631449225e-05,
+      "loss": 0.0491,
+      "mean_token_accuracy": 0.9827546238899231,
+      "num_tokens": 1933176.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.43071328699588773,
+      "epoch": 7.550200803212851,
+      "grad_norm": 1.0797828435897827,
+      "learning_rate": 1.5584278694734888e-05,
+      "loss": 0.0346,
+      "mean_token_accuracy": 0.9884509861469268,
+      "num_tokens": 1943588.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.4416676253080368,
+      "epoch": 7.590361445783133,
+      "grad_norm": 1.410029411315918,
+      "learning_rate": 1.5105521543573647e-05,
+      "loss": 0.0546,
+      "mean_token_accuracy": 0.9827970087528228,
+      "num_tokens": 1953993.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.4177089184522629,
+      "epoch": 7.6305220883534135,
+      "grad_norm": 1.7492619752883911,
+      "learning_rate": 1.4632921656494469e-05,
+      "loss": 0.0295,
+      "mean_token_accuracy": 0.9915511727333068,
+      "num_tokens": 1964313.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.4375153541564941,
+      "epoch": 7.670682730923695,
+      "grad_norm": 1.044287085533142,
+      "learning_rate": 1.416656242555366e-05,
+      "loss": 0.0429,
+      "mean_token_accuracy": 0.9869651854038238,
+      "num_tokens": 1974581.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.40726748406887053,
+      "epoch": 7.710843373493976,
+      "grad_norm": 2.5667827129364014,
+      "learning_rate": 1.3706526141619792e-05,
+      "loss": 0.0179,
+      "mean_token_accuracy": 0.9945597887039185,
+      "num_tokens": 1984900.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.4179089695215225,
+      "epoch": 7.7510040160642575,
+      "grad_norm": 0.8887644410133362,
+      "learning_rate": 1.3252893979853304e-05,
+      "loss": 0.0446,
+      "mean_token_accuracy": 0.9867686688899994,
+      "num_tokens": 1995343.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.4144404917955399,
+      "epoch": 7.791164658634538,
+      "grad_norm": 5.518115997314453,
+      "learning_rate": 1.2805745985382867e-05,
+      "loss": 0.05,
+      "mean_token_accuracy": 0.9840149760246277,
+      "num_tokens": 2005767.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.40318343341350554,
+      "epoch": 7.831325301204819,
+      "grad_norm": 2.1526505947113037,
+      "learning_rate": 1.2365161059180942e-05,
+      "loss": 0.0344,
+      "mean_token_accuracy": 0.9888841032981872,
+      "num_tokens": 2016189.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.416042622923851,
+      "epoch": 7.871485943775101,
+      "grad_norm": 1.547747254371643,
+      "learning_rate": 1.1931216944141621e-05,
+      "loss": 0.0385,
+      "mean_token_accuracy": 0.9887767791748047,
+      "num_tokens": 2026515.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.43885636925697324,
+      "epoch": 7.911646586345381,
+      "grad_norm": 3.6307454109191895,
+      "learning_rate": 1.1503990211362403e-05,
+      "loss": 0.0669,
+      "mean_token_accuracy": 0.9795345544815064,
+      "num_tokens": 2037038.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.4240131974220276,
+      "epoch": 7.951807228915663,
+      "grad_norm": 1.6338939666748047,
+      "learning_rate": 1.1083556246633048e-05,
+      "loss": 0.0237,
+      "mean_token_accuracy": 0.9929973244667053,
+      "num_tokens": 2047230.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.42111910283565523,
+      "epoch": 7.991967871485944,
+      "grad_norm": 2.715122938156128,
+      "learning_rate": 1.0669989237133437e-05,
+      "loss": 0.0322,
+      "mean_token_accuracy": 0.9891472160816193,
+      "num_tokens": 2057672.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.41850388050079346,
+      "epoch": 8.032128514056225,
+      "grad_norm": 1.4906864166259766,
+      "learning_rate": 1.0263362158342948e-05,
+      "loss": 0.034,
+      "mean_token_accuracy": 0.9905479788780213,
+      "num_tokens": 2067814.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.41954563856124877,
+      "epoch": 8.072289156626505,
+      "grad_norm": 3.3255691528320312,
+      "learning_rate": 9.863746761163679e-06,
+      "loss": 0.0201,
+      "mean_token_accuracy": 0.9944594264030456,
+      "num_tokens": 2078106.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.397763192653656,
+      "epoch": 8.112449799196787,
+      "grad_norm": 0.30429109930992126,
+      "learning_rate": 9.471213559259684e-06,
+      "loss": 0.0173,
+      "mean_token_accuracy": 0.994371771812439,
+      "num_tokens": 2088529.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.4112460821866989,
+      "epoch": 8.152610441767068,
+      "grad_norm": 1.9762293100357056,
+      "learning_rate": 9.08583181661461e-06,
+      "loss": 0.0333,
+      "mean_token_accuracy": 0.989613676071167,
+      "num_tokens": 2098831.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.39446723461151123,
+      "epoch": 8.19277108433735,
+      "grad_norm": 0.34966492652893066,
+      "learning_rate": 8.707669535309793e-06,
+      "loss": 0.0151,
+      "mean_token_accuracy": 0.9962847888469696,
+      "num_tokens": 2109061.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.4124080568552017,
+      "epoch": 8.23293172690763,
+      "grad_norm": 2.262033462524414,
+      "learning_rate": 8.33679344352501e-06,
+      "loss": 0.0237,
+      "mean_token_accuracy": 0.99340540766716,
+      "num_tokens": 2119414.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.4085001915693283,
+      "epoch": 8.273092369477911,
+      "grad_norm": 2.772256374359131,
+      "learning_rate": 7.97326898376406e-06,
+      "loss": 0.0183,
+      "mean_token_accuracy": 0.9942695081233979,
+      "num_tokens": 2129865.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.3970011413097382,
+      "epoch": 8.313253012048193,
+      "grad_norm": 0.0766800194978714,
+      "learning_rate": 7.617160301307169e-06,
+      "loss": 0.0231,
+      "mean_token_accuracy": 0.9924921274185181,
+      "num_tokens": 2140413.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.3960287183523178,
+      "epoch": 8.353413654618475,
+      "grad_norm": 2.2849180698394775,
+      "learning_rate": 7.268530232892317e-06,
+      "loss": 0.0163,
+      "mean_token_accuracy": 0.9943627178668976,
+      "num_tokens": 2150856.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.39155090153217315,
+      "epoch": 8.393574297188755,
+      "grad_norm": 1.8346494436264038,
+      "learning_rate": 6.9274402956274686e-06,
+      "loss": 0.0161,
+      "mean_token_accuracy": 0.9951763331890107,
+      "num_tokens": 2161167.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.39390614032745364,
+      "epoch": 8.433734939759036,
+      "grad_norm": 0.9862962961196899,
+      "learning_rate": 6.593950676135624e-06,
+      "loss": 0.0187,
+      "mean_token_accuracy": 0.9939960479736328,
+      "num_tokens": 2171451.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.4057298481464386,
+      "epoch": 8.473895582329318,
+      "grad_norm": 1.3590463399887085,
+      "learning_rate": 6.268120219934631e-06,
+      "loss": 0.0202,
+      "mean_token_accuracy": 0.9932440876960754,
+      "num_tokens": 2181817.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.39560525119304657,
+      "epoch": 8.514056224899598,
+      "grad_norm": 0.4814577102661133,
+      "learning_rate": 5.950006421053772e-06,
+      "loss": 0.0124,
+      "mean_token_accuracy": 0.9962008893489838,
+      "num_tokens": 2192033.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.4032837748527527,
+      "epoch": 8.55421686746988,
+      "grad_norm": 0.9394027590751648,
+      "learning_rate": 5.639665411888584e-06,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.9962640881538392,
+      "num_tokens": 2202400.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.39971289932727816,
+      "epoch": 8.594377510040161,
+      "grad_norm": 2.415469169616699,
+      "learning_rate": 5.337151953296188e-06,
+      "loss": 0.0155,
+      "mean_token_accuracy": 0.9949438571929932,
+      "num_tokens": 2212823.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.3894567906856537,
+      "epoch": 8.634538152610443,
+      "grad_norm": 0.7443180084228516,
+      "learning_rate": 5.042519424932513e-06,
+      "loss": 0.0181,
+      "mean_token_accuracy": 0.9959241628646851,
+      "num_tokens": 2223308.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.39390002489089965,
+      "epoch": 8.674698795180722,
+      "grad_norm": 0.715395450592041,
+      "learning_rate": 4.755819815833174e-06,
+      "loss": 0.0108,
+      "mean_token_accuracy": 0.9970237433910369,
+      "num_tokens": 2233559.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.3902711033821106,
+      "epoch": 8.714859437751004,
+      "grad_norm": 1.1155041456222534,
+      "learning_rate": 4.477103715239922e-06,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.9956739842891693,
+      "num_tokens": 2243995.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.3863468259572983,
+      "epoch": 8.755020080321286,
+      "grad_norm": 1.4398478269577026,
+      "learning_rate": 4.2064203036738746e-06,
+      "loss": 0.0145,
+      "mean_token_accuracy": 0.9960389971733093,
+      "num_tokens": 2254242.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.3954067826271057,
+      "epoch": 8.795180722891565,
+      "grad_norm": 2.23990797996521,
+      "learning_rate": 3.9438173442575e-06,
+      "loss": 0.0185,
+      "mean_token_accuracy": 0.9950194656848907,
+      "num_tokens": 2264729.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.39104788303375243,
+      "epoch": 8.835341365461847,
+      "grad_norm": 0.2850276827812195,
+      "learning_rate": 3.6893411742865814e-06,
+      "loss": 0.017,
+      "mean_token_accuracy": 0.9942178368568421,
+      "num_tokens": 2275047.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.3993827670812607,
+      "epoch": 8.875502008032129,
+      "grad_norm": 0.1491786539554596,
+      "learning_rate": 3.443036697053875e-06,
+      "loss": 0.0174,
+      "mean_token_accuracy": 0.9946730017662049,
+      "num_tokens": 2285367.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.3928679794073105,
+      "epoch": 8.91566265060241,
+      "grad_norm": 1.6054004430770874,
+      "learning_rate": 3.204947373925693e-06,
+      "loss": 0.0167,
+      "mean_token_accuracy": 0.9948305010795593,
+      "num_tokens": 2295729.0,
+      "step": 2220
+    },
+    {
+      "entropy": 0.3927693247795105,
+      "epoch": 8.95582329317269,
+      "grad_norm": 3.690190315246582,
+      "learning_rate": 2.97511521667303e-06,
+      "loss": 0.0235,
+      "mean_token_accuracy": 0.9919692397117614,
+      "num_tokens": 2305976.0,
+      "step": 2230
+    },
+    {
+      "entropy": 0.397261306643486,
+      "epoch": 8.995983935742972,
+      "grad_norm": 0.7972508668899536,
+      "learning_rate": 2.7535807800583957e-06,
+      "loss": 0.0144,
+      "mean_token_accuracy": 0.9957432746887207,
+      "num_tokens": 2316263.0,
+      "step": 2240
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2490,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.670805137523776e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2241/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:450585756ea90b682a59f52c4d089b67a1b6344d380886dad6b85eeac9c0b544
+size 6417

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/README.md

@@ -0,0 +1,209 @@
+---
+base_model: google/gemma-3-4b-it
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:google/gemma-3-4b-it
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-3-4b-it",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "down_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "k_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:124dde30e0999a4ddef4aed38fec80383334d161aab8b177eb5cf5d79311e7a5
+size 65674128

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/chat_template.jinja

@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pan_and_scan": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Gemma3ImageProcessor",
+  "image_seq_length": 256,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "pan_and_scan_max_num_crops": null,
+  "pan_and_scan_min_crop_size": null,
+  "pan_and_scan_min_ratio_to_activate": null,
+  "processor_class": "Gemma3Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 896,
+    "width": 896
+  }
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/trainer_state.json

@@ -0,0 +1,2524 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 10.0,
+  "eval_steps": 500,
+  "global_step": 2490,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.671006029844284,
+      "epoch": 0.040160642570281124,
+      "grad_norm": 12.208308219909668,
+      "learning_rate": 7.2e-06,
+      "loss": 0.9984,
+      "mean_token_accuracy": 0.780225270986557,
+      "num_tokens": 10404.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.7024954497814179,
+      "epoch": 0.08032128514056225,
+      "grad_norm": 4.81063175201416,
+      "learning_rate": 1.52e-05,
+      "loss": 0.6915,
+      "mean_token_accuracy": 0.7844282865524292,
+      "num_tokens": 21009.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.7053345918655396,
+      "epoch": 0.12048192771084337,
+      "grad_norm": 5.82667350769043,
+      "learning_rate": 2.32e-05,
+      "loss": 0.5032,
+      "mean_token_accuracy": 0.8207756400108337,
+      "num_tokens": 31309.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.7185693144798279,
+      "epoch": 0.1606425702811245,
+      "grad_norm": 3.646798610687256,
+      "learning_rate": 3.12e-05,
+      "loss": 0.5641,
+      "mean_token_accuracy": 0.796815150976181,
+      "num_tokens": 41509.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.7153907418251038,
+      "epoch": 0.20080321285140562,
+      "grad_norm": 5.238017559051514,
+      "learning_rate": 3.9200000000000004e-05,
+      "loss": 0.4953,
+      "mean_token_accuracy": 0.823300176858902,
+      "num_tokens": 51964.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.7120266735553742,
+      "epoch": 0.24096385542168675,
+      "grad_norm": 4.953554630279541,
+      "learning_rate": 4.72e-05,
+      "loss": 0.4831,
+      "mean_token_accuracy": 0.8243303775787354,
+      "num_tokens": 62281.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.7515081107616425,
+      "epoch": 0.28112449799196787,
+      "grad_norm": 4.73248815536499,
+      "learning_rate": 5.520000000000001e-05,
+      "loss": 0.5529,
+      "mean_token_accuracy": 0.8086587309837341,
+      "num_tokens": 72602.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.7260624289512634,
+      "epoch": 0.321285140562249,
+      "grad_norm": 4.54362678527832,
+      "learning_rate": 6.32e-05,
+      "loss": 0.5278,
+      "mean_token_accuracy": 0.80874263048172,
+      "num_tokens": 82945.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.7128267765045166,
+      "epoch": 0.3614457831325301,
+      "grad_norm": 3.1748130321502686,
+      "learning_rate": 7.12e-05,
+      "loss": 0.4767,
+      "mean_token_accuracy": 0.8299677312374115,
+      "num_tokens": 93192.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.7419508635997772,
+      "epoch": 0.40160642570281124,
+      "grad_norm": 3.0026862621307373,
+      "learning_rate": 7.920000000000001e-05,
+      "loss": 0.5408,
+      "mean_token_accuracy": 0.8037238836288452,
+      "num_tokens": 103603.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.7587903916835785,
+      "epoch": 0.44176706827309237,
+      "grad_norm": 5.152759552001953,
+      "learning_rate": 8.72e-05,
+      "loss": 0.5514,
+      "mean_token_accuracy": 0.8075420558452606,
+      "num_tokens": 113946.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.7834510266780853,
+      "epoch": 0.4819277108433735,
+      "grad_norm": 3.7946877479553223,
+      "learning_rate": 9.52e-05,
+      "loss": 0.577,
+      "mean_token_accuracy": 0.7896877348423004,
+      "num_tokens": 124258.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.7676194608211517,
+      "epoch": 0.5220883534136547,
+      "grad_norm": 4.143728256225586,
+      "learning_rate": 9.999929417599468e-05,
+      "loss": 0.5812,
+      "mean_token_accuracy": 0.7885167539119721,
+      "num_tokens": 134802.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.7672028005123138,
+      "epoch": 0.5622489959839357,
+      "grad_norm": 4.3124847412109375,
+      "learning_rate": 9.999135388478797e-05,
+      "loss": 0.4875,
+      "mean_token_accuracy": 0.8212199211120605,
+      "num_tokens": 145052.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.7765893459320068,
+      "epoch": 0.6024096385542169,
+      "grad_norm": 4.5034074783325195,
+      "learning_rate": 9.997459242813312e-05,
+      "loss": 0.5032,
+      "mean_token_accuracy": 0.8222271859645843,
+      "num_tokens": 155357.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.7953528225421905,
+      "epoch": 0.642570281124498,
+      "grad_norm": 3.0642852783203125,
+      "learning_rate": 9.994901276365323e-05,
+      "loss": 0.5281,
+      "mean_token_accuracy": 0.8077535212039948,
+      "num_tokens": 165640.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.7806783020496368,
+      "epoch": 0.6827309236947792,
+      "grad_norm": 2.9503402709960938,
+      "learning_rate": 9.991461940497786e-05,
+      "loss": 0.52,
+      "mean_token_accuracy": 0.814589935541153,
+      "num_tokens": 175994.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.7847117841243744,
+      "epoch": 0.7228915662650602,
+      "grad_norm": 5.207314491271973,
+      "learning_rate": 9.987141842094658e-05,
+      "loss": 0.517,
+      "mean_token_accuracy": 0.8100123822689056,
+      "num_tokens": 186403.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.744153618812561,
+      "epoch": 0.7630522088353414,
+      "grad_norm": 3.080606698989868,
+      "learning_rate": 9.981941743453815e-05,
+      "loss": 0.4523,
+      "mean_token_accuracy": 0.8381906926631928,
+      "num_tokens": 196638.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.7874633729457855,
+      "epoch": 0.8032128514056225,
+      "grad_norm": 4.700623512268066,
+      "learning_rate": 9.975862562152532e-05,
+      "loss": 0.5551,
+      "mean_token_accuracy": 0.7961149096488953,
+      "num_tokens": 207065.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.7740188777446747,
+      "epoch": 0.8433734939759037,
+      "grad_norm": 3.2746150493621826,
+      "learning_rate": 9.968905370885586e-05,
+      "loss": 0.5106,
+      "mean_token_accuracy": 0.8123311400413513,
+      "num_tokens": 217456.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.7738964557647705,
+      "epoch": 0.8835341365461847,
+      "grad_norm": 3.0882720947265625,
+      "learning_rate": 9.961071397275963e-05,
+      "loss": 0.5226,
+      "mean_token_accuracy": 0.808603173494339,
+      "num_tokens": 227778.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.7768196284770965,
+      "epoch": 0.9236947791164659,
+      "grad_norm": 4.431626796722412,
+      "learning_rate": 9.952362023658249e-05,
+      "loss": 0.4959,
+      "mean_token_accuracy": 0.8225658714771271,
+      "num_tokens": 238074.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.7808157622814178,
+      "epoch": 0.963855421686747,
+      "grad_norm": 2.598005533218384,
+      "learning_rate": 9.9427787868347e-05,
+      "loss": 0.4896,
+      "mean_token_accuracy": 0.8194038093090057,
+      "num_tokens": 248395.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.7777634024620056,
+      "epoch": 1.0040160642570282,
+      "grad_norm": 3.525217056274414,
+      "learning_rate": 9.93232337780408e-05,
+      "loss": 0.4729,
+      "mean_token_accuracy": 0.8291181206703186,
+      "num_tokens": 258505.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.7768383502960206,
+      "epoch": 1.0441767068273093,
+      "grad_norm": 5.895783424377441,
+      "learning_rate": 9.920997641463272e-05,
+      "loss": 0.4785,
+      "mean_token_accuracy": 0.8300446927547455,
+      "num_tokens": 268910.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.7642462968826294,
+      "epoch": 1.0843373493975903,
+      "grad_norm": 3.5049235820770264,
+      "learning_rate": 9.908803576281736e-05,
+      "loss": 0.438,
+      "mean_token_accuracy": 0.8362016320228577,
+      "num_tokens": 279283.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.7406439125537873,
+      "epoch": 1.1244979919678715,
+      "grad_norm": 2.915161371231079,
+      "learning_rate": 9.895743333948874e-05,
+      "loss": 0.4515,
+      "mean_token_accuracy": 0.8348344326019287,
+      "num_tokens": 289866.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.7464889645576477,
+      "epoch": 1.1646586345381527,
+      "grad_norm": 4.682326316833496,
+      "learning_rate": 9.881819218994365e-05,
+      "loss": 0.4514,
+      "mean_token_accuracy": 0.838148581981659,
+      "num_tokens": 300185.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.7421976208686829,
+      "epoch": 1.2048192771084336,
+      "grad_norm": 3.9356069564819336,
+      "learning_rate": 9.867033688381502e-05,
+      "loss": 0.4584,
+      "mean_token_accuracy": 0.8396136164665222,
+      "num_tokens": 310601.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.7389526426792145,
+      "epoch": 1.2449799196787148,
+      "grad_norm": 2.621224880218506,
+      "learning_rate": 9.85138935107367e-05,
+      "loss": 0.4648,
+      "mean_token_accuracy": 0.8362064242362977,
+      "num_tokens": 321070.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.7472976803779602,
+      "epoch": 1.285140562248996,
+      "grad_norm": 3.3467557430267334,
+      "learning_rate": 9.834888967573976e-05,
+      "loss": 0.4906,
+      "mean_token_accuracy": 0.8216790199279785,
+      "num_tokens": 331491.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.7224303126335144,
+      "epoch": 1.3253012048192772,
+      "grad_norm": 3.37907075881958,
+      "learning_rate": 9.817535449438148e-05,
+      "loss": 0.4495,
+      "mean_token_accuracy": 0.8356333911418915,
+      "num_tokens": 341747.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.753093010187149,
+      "epoch": 1.3654618473895583,
+      "grad_norm": 4.384500026702881,
+      "learning_rate": 9.799331858760786e-05,
+      "loss": 0.5321,
+      "mean_token_accuracy": 0.808532440662384,
+      "num_tokens": 352031.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.7213885009288787,
+      "epoch": 1.4056224899598393,
+      "grad_norm": 3.903226852416992,
+      "learning_rate": 9.78028140763503e-05,
+      "loss": 0.4696,
+      "mean_token_accuracy": 0.83351429104805,
+      "num_tokens": 362499.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.7465944051742553,
+      "epoch": 1.4457831325301205,
+      "grad_norm": 3.3047783374786377,
+      "learning_rate": 9.76038745758579e-05,
+      "loss": 0.4657,
+      "mean_token_accuracy": 0.8354173362255096,
+      "num_tokens": 372813.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.7460316002368927,
+      "epoch": 1.4859437751004017,
+      "grad_norm": 3.1054019927978516,
+      "learning_rate": 9.739653518976581e-05,
+      "loss": 0.4636,
+      "mean_token_accuracy": 0.8355995714664459,
+      "num_tokens": 383120.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.7742661297321319,
+      "epoch": 1.5261044176706826,
+      "grad_norm": 3.354398012161255,
+      "learning_rate": 9.718083250390113e-05,
+      "loss": 0.5208,
+      "mean_token_accuracy": 0.8111523389816284,
+      "num_tokens": 393449.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.7555004000663758,
+      "epoch": 1.5662650602409638,
+      "grad_norm": 3.681725263595581,
+      "learning_rate": 9.695680457982713e-05,
+      "loss": 0.4697,
+      "mean_token_accuracy": 0.8350590109825134,
+      "num_tokens": 403803.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.7885475993156433,
+      "epoch": 1.606425702811245,
+      "grad_norm": 3.469503879547119,
+      "learning_rate": 9.67244909481272e-05,
+      "loss": 0.5478,
+      "mean_token_accuracy": 0.8026194393634796,
+      "num_tokens": 414169.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.7571891069412231,
+      "epoch": 1.6465863453815262,
+      "grad_norm": 3.5566561222076416,
+      "learning_rate": 9.648393260142948e-05,
+      "loss": 0.442,
+      "mean_token_accuracy": 0.8380986630916596,
+      "num_tokens": 424543.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.7092450320720672,
+      "epoch": 1.6867469879518073,
+      "grad_norm": 3.590761661529541,
+      "learning_rate": 9.623517198717362e-05,
+      "loss": 0.404,
+      "mean_token_accuracy": 0.8548887431621551,
+      "num_tokens": 434793.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.7402957201004028,
+      "epoch": 1.7269076305220885,
+      "grad_norm": 3.051144599914551,
+      "learning_rate": 9.597825300012073e-05,
+      "loss": 0.4791,
+      "mean_token_accuracy": 0.8277241110801696,
+      "num_tokens": 445085.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.7417145252227784,
+      "epoch": 1.7670682730923695,
+      "grad_norm": 2.8374202251434326,
+      "learning_rate": 9.571322097460793e-05,
+      "loss": 0.4412,
+      "mean_token_accuracy": 0.8435907065868378,
+      "num_tokens": 455314.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.7295144855976105,
+      "epoch": 1.8072289156626506,
+      "grad_norm": 2.723912477493286,
+      "learning_rate": 9.544012267654901e-05,
+      "loss": 0.4406,
+      "mean_token_accuracy": 0.8446374356746673,
+      "num_tokens": 465512.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.7279774129390717,
+      "epoch": 1.8473895582329316,
+      "grad_norm": 3.471815347671509,
+      "learning_rate": 9.51590062951824e-05,
+      "loss": 0.4303,
+      "mean_token_accuracy": 0.8465097844600677,
+      "num_tokens": 475821.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.7734329164028168,
+      "epoch": 1.8875502008032128,
+      "grad_norm": 6.041645526885986,
+      "learning_rate": 9.486992143456792e-05,
+      "loss": 0.5395,
+      "mean_token_accuracy": 0.8112995743751525,
+      "num_tokens": 486060.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.7551539540290833,
+      "epoch": 1.927710843373494,
+      "grad_norm": 3.4140775203704834,
+      "learning_rate": 9.457291910483409e-05,
+      "loss": 0.4588,
+      "mean_token_accuracy": 0.8308178722858429,
+      "num_tokens": 496329.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.7404638409614563,
+      "epoch": 1.9678714859437751,
+      "grad_norm": 3.563293218612671,
+      "learning_rate": 9.426805171317701e-05,
+      "loss": 0.4921,
+      "mean_token_accuracy": 0.8270943462848663,
+      "num_tokens": 506757.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.7283211767673492,
+      "epoch": 2.0080321285140563,
+      "grad_norm": 3.525891065597534,
+      "learning_rate": 9.395537305461311e-05,
+      "loss": 0.4413,
+      "mean_token_accuracy": 0.8409784734249115,
+      "num_tokens": 517020.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.7040447413921356,
+      "epoch": 2.0481927710843375,
+      "grad_norm": 3.779123306274414,
+      "learning_rate": 9.363493830248666e-05,
+      "loss": 0.4035,
+      "mean_token_accuracy": 0.8602916836738587,
+      "num_tokens": 527408.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.6990016818046569,
+      "epoch": 2.0883534136546187,
+      "grad_norm": 3.23069429397583,
+      "learning_rate": 9.33068039987343e-05,
+      "loss": 0.4115,
+      "mean_token_accuracy": 0.85965576171875,
+      "num_tokens": 537684.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.6961205899715424,
+      "epoch": 2.1285140562248994,
+      "grad_norm": 5.430521011352539,
+      "learning_rate": 9.297102804390798e-05,
+      "loss": 0.3504,
+      "mean_token_accuracy": 0.8788332164287567,
+      "num_tokens": 548009.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.6922298431396484,
+      "epoch": 2.1686746987951806,
+      "grad_norm": 3.464811325073242,
+      "learning_rate": 9.26276696869582e-05,
+      "loss": 0.3535,
+      "mean_token_accuracy": 0.8732336580753326,
+      "num_tokens": 558484.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.6996638119220734,
+      "epoch": 2.208835341365462,
+      "grad_norm": 3.1049904823303223,
+      "learning_rate": 9.227678951477925e-05,
+      "loss": 0.4074,
+      "mean_token_accuracy": 0.8610455751419067,
+      "num_tokens": 568796.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.6876482725143432,
+      "epoch": 2.248995983935743,
+      "grad_norm": 3.024766206741333,
+      "learning_rate": 9.19184494415185e-05,
+      "loss": 0.3747,
+      "mean_token_accuracy": 0.866488391160965,
+      "num_tokens": 579189.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.6892669796943665,
+      "epoch": 2.289156626506024,
+      "grad_norm": 3.7265472412109375,
+      "learning_rate": 9.15527126976514e-05,
+      "loss": 0.3709,
+      "mean_token_accuracy": 0.8671182692050934,
+      "num_tokens": 589477.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.7166594922542572,
+      "epoch": 2.3293172690763053,
+      "grad_norm": 2.6233692169189453,
+      "learning_rate": 9.117964381882413e-05,
+      "loss": 0.4437,
+      "mean_token_accuracy": 0.8400287747383117,
+      "num_tokens": 599758.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.7110948503017426,
+      "epoch": 2.3694779116465865,
+      "grad_norm": 2.8130180835723877,
+      "learning_rate": 9.079930863446612e-05,
+      "loss": 0.3918,
+      "mean_token_accuracy": 0.8617593050003052,
+      "num_tokens": 610181.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.6950080335140228,
+      "epoch": 2.4096385542168672,
+      "grad_norm": 6.08152961730957,
+      "learning_rate": 9.041177425617427e-05,
+      "loss": 0.3525,
+      "mean_token_accuracy": 0.8760755777359008,
+      "num_tokens": 620333.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.70909024477005,
+      "epoch": 2.4497991967871484,
+      "grad_norm": 2.8135924339294434,
+      "learning_rate": 9.001710906587064e-05,
+      "loss": 0.3947,
+      "mean_token_accuracy": 0.8637883484363555,
+      "num_tokens": 630697.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.7154293119907379,
+      "epoch": 2.4899598393574296,
+      "grad_norm": 3.2165870666503906,
+      "learning_rate": 8.961538270373639e-05,
+      "loss": 0.4164,
+      "mean_token_accuracy": 0.8475094556808471,
+      "num_tokens": 641231.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.7277772426605225,
+      "epoch": 2.5301204819277108,
+      "grad_norm": 5.162221908569336,
+      "learning_rate": 8.920666605592341e-05,
+      "loss": 0.4189,
+      "mean_token_accuracy": 0.8539318025112153,
+      "num_tokens": 651543.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.675065678358078,
+      "epoch": 2.570281124497992,
+      "grad_norm": 2.9395062923431396,
+      "learning_rate": 8.879103124204626e-05,
+      "loss": 0.3421,
+      "mean_token_accuracy": 0.881534194946289,
+      "num_tokens": 661830.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.6800589621067047,
+      "epoch": 2.610441767068273,
+      "grad_norm": 5.408241271972656,
+      "learning_rate": 8.836855160245629e-05,
+      "loss": 0.3392,
+      "mean_token_accuracy": 0.8793375313282012,
+      "num_tokens": 672149.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.6833930969238281,
+      "epoch": 2.6506024096385543,
+      "grad_norm": 5.121169567108154,
+      "learning_rate": 8.79393016853005e-05,
+      "loss": 0.4217,
+      "mean_token_accuracy": 0.8490349233150483,
+      "num_tokens": 682537.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.6918147087097168,
+      "epoch": 2.6907630522088355,
+      "grad_norm": 3.5816986560821533,
+      "learning_rate": 8.750335723336728e-05,
+      "loss": 0.408,
+      "mean_token_accuracy": 0.8592755615711212,
+      "num_tokens": 692836.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.6768696010112762,
+      "epoch": 2.7309236947791167,
+      "grad_norm": 4.3298492431640625,
+      "learning_rate": 8.706079517072127e-05,
+      "loss": 0.3447,
+      "mean_token_accuracy": 0.8745476543903351,
+      "num_tokens": 703107.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.6964600443840027,
+      "epoch": 2.7710843373493974,
+      "grad_norm": 5.028672218322754,
+      "learning_rate": 8.661169358912978e-05,
+      "loss": 0.4352,
+      "mean_token_accuracy": 0.8437578022480011,
+      "num_tokens": 713574.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.7136080145835877,
+      "epoch": 2.8112449799196786,
+      "grad_norm": 3.92950701713562,
+      "learning_rate": 8.615613173428321e-05,
+      "loss": 0.4112,
+      "mean_token_accuracy": 0.8544917225837707,
+      "num_tokens": 724034.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.7157581627368927,
+      "epoch": 2.8514056224899598,
+      "grad_norm": 3.4689905643463135,
+      "learning_rate": 8.569418999181194e-05,
+      "loss": 0.4164,
+      "mean_token_accuracy": 0.8549265265464783,
+      "num_tokens": 734413.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.7330816447734833,
+      "epoch": 2.891566265060241,
+      "grad_norm": 4.131735324859619,
+      "learning_rate": 8.522594987310184e-05,
+      "loss": 0.439,
+      "mean_token_accuracy": 0.8389688909053803,
+      "num_tokens": 744798.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.6790109813213349,
+      "epoch": 2.931726907630522,
+      "grad_norm": 3.378662586212158,
+      "learning_rate": 8.475149400091137e-05,
+      "loss": 0.362,
+      "mean_token_accuracy": 0.8758748531341553,
+      "num_tokens": 754977.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.6692682385444642,
+      "epoch": 2.9718875502008033,
+      "grad_norm": 3.1824755668640137,
+      "learning_rate": 8.427090609479245e-05,
+      "loss": 0.344,
+      "mean_token_accuracy": 0.8847700834274292,
+      "num_tokens": 765400.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.6895963490009308,
+      "epoch": 3.0120481927710845,
+      "grad_norm": 4.070644378662109,
+      "learning_rate": 8.378427095631776e-05,
+      "loss": 0.4177,
+      "mean_token_accuracy": 0.8527476012706756,
+      "num_tokens": 775375.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.6410589516162872,
+      "epoch": 3.0522088353413657,
+      "grad_norm": 3.8965253829956055,
+      "learning_rate": 8.329167445411732e-05,
+      "loss": 0.2976,
+      "mean_token_accuracy": 0.8942947745323181,
+      "num_tokens": 785714.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.6333921432495118,
+      "epoch": 3.0923694779116464,
+      "grad_norm": 5.164280891418457,
+      "learning_rate": 8.279320350872655e-05,
+      "loss": 0.337,
+      "mean_token_accuracy": 0.8851125121116639,
+      "num_tokens": 796081.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.6316083431243896,
+      "epoch": 3.1325301204819276,
+      "grad_norm": 5.71152925491333,
+      "learning_rate": 8.228894607724878e-05,
+      "loss": 0.2859,
+      "mean_token_accuracy": 0.8993970811367035,
+      "num_tokens": 806391.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.6212240159511566,
+      "epoch": 3.1726907630522088,
+      "grad_norm": 4.353719711303711,
+      "learning_rate": 8.177899113783492e-05,
+      "loss": 0.3073,
+      "mean_token_accuracy": 0.888391625881195,
+      "num_tokens": 816801.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.6379825949668885,
+      "epoch": 3.21285140562249,
+      "grad_norm": 4.2553300857543945,
+      "learning_rate": 8.126342867398301e-05,
+      "loss": 0.3058,
+      "mean_token_accuracy": 0.8925197720527649,
+      "num_tokens": 827206.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.6057499378919602,
+      "epoch": 3.253012048192771,
+      "grad_norm": 3.8955018520355225,
+      "learning_rate": 8.074234965866012e-05,
+      "loss": 0.2812,
+      "mean_token_accuracy": 0.9016481280326843,
+      "num_tokens": 837325.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.628681194782257,
+      "epoch": 3.2931726907630523,
+      "grad_norm": 3.5182700157165527,
+      "learning_rate": 8.021584603824996e-05,
+      "loss": 0.2604,
+      "mean_token_accuracy": 0.9060991644859314,
+      "num_tokens": 847765.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.6285995721817017,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 4.679093837738037,
+      "learning_rate": 7.968401071632855e-05,
+      "loss": 0.3321,
+      "mean_token_accuracy": 0.8817380368709564,
+      "num_tokens": 858303.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.6240448355674744,
+      "epoch": 3.3734939759036147,
+      "grad_norm": 4.280148506164551,
+      "learning_rate": 7.914693753727091e-05,
+      "loss": 0.2971,
+      "mean_token_accuracy": 0.89331756234169,
+      "num_tokens": 868618.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.6346171617507934,
+      "epoch": 3.4136546184738954,
+      "grad_norm": 5.035059452056885,
+      "learning_rate": 7.860472126969213e-05,
+      "loss": 0.283,
+      "mean_token_accuracy": 0.8987699389457703,
+      "num_tokens": 879028.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.6339222490787506,
+      "epoch": 3.4538152610441766,
+      "grad_norm": 3.3308496475219727,
+      "learning_rate": 7.805745758972481e-05,
+      "loss": 0.3369,
+      "mean_token_accuracy": 0.8849397778511048,
+      "num_tokens": 889513.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.654508638381958,
+      "epoch": 3.4939759036144578,
+      "grad_norm": 4.176350116729736,
+      "learning_rate": 7.75052430641368e-05,
+      "loss": 0.3435,
+      "mean_token_accuracy": 0.8724894464015961,
+      "num_tokens": 900038.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.6320246398448944,
+      "epoch": 3.534136546184739,
+      "grad_norm": 4.780612945556641,
+      "learning_rate": 7.694817513329159e-05,
+      "loss": 0.2909,
+      "mean_token_accuracy": 0.894708800315857,
+      "num_tokens": 910418.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.6431657671928406,
+      "epoch": 3.57429718875502,
+      "grad_norm": 5.155822277069092,
+      "learning_rate": 7.638635209395453e-05,
+      "loss": 0.3065,
+      "mean_token_accuracy": 0.8888295769691468,
+      "num_tokens": 920627.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.6216733515262604,
+      "epoch": 3.6144578313253013,
+      "grad_norm": 4.907107830047607,
+      "learning_rate": 7.58198730819481e-05,
+      "loss": 0.2597,
+      "mean_token_accuracy": 0.904644924402237,
+      "num_tokens": 930895.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.6459767282009125,
+      "epoch": 3.6546184738955825,
+      "grad_norm": 4.625438213348389,
+      "learning_rate": 7.524883805465888e-05,
+      "loss": 0.3488,
+      "mean_token_accuracy": 0.8783864676952362,
+      "num_tokens": 941195.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.6570597350597381,
+      "epoch": 3.694779116465863,
+      "grad_norm": 3.448225259780884,
+      "learning_rate": 7.467334777339985e-05,
+      "loss": 0.3313,
+      "mean_token_accuracy": 0.888871556520462,
+      "num_tokens": 951345.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.6195529520511627,
+      "epoch": 3.734939759036145,
+      "grad_norm": 5.21084451675415,
+      "learning_rate": 7.409350378563046e-05,
+      "loss": 0.2899,
+      "mean_token_accuracy": 0.896889615058899,
+      "num_tokens": 961633.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.6462969362735749,
+      "epoch": 3.7751004016064256,
+      "grad_norm": 3.8568060398101807,
+      "learning_rate": 7.350940840703842e-05,
+      "loss": 0.2914,
+      "mean_token_accuracy": 0.8919413447380066,
+      "num_tokens": 971973.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.6068174064159393,
+      "epoch": 3.8152610441767068,
+      "grad_norm": 4.7789082527160645,
+      "learning_rate": 7.292116470348554e-05,
+      "loss": 0.2713,
+      "mean_token_accuracy": 0.9051806092262268,
+      "num_tokens": 982359.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.6487982928752899,
+      "epoch": 3.855421686746988,
+      "grad_norm": 3.895387887954712,
+      "learning_rate": 7.232887647282147e-05,
+      "loss": 0.328,
+      "mean_token_accuracy": 0.8847681939601898,
+      "num_tokens": 992792.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.6167472541332245,
+      "epoch": 3.895582329317269,
+      "grad_norm": 5.2591328620910645,
+      "learning_rate": 7.173264822656806e-05,
+      "loss": 0.2843,
+      "mean_token_accuracy": 0.8959703624248505,
+      "num_tokens": 1003118.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.6505671739578247,
+      "epoch": 3.9357429718875503,
+      "grad_norm": 3.341792106628418,
+      "learning_rate": 7.113258517147801e-05,
+      "loss": 0.3449,
+      "mean_token_accuracy": 0.8760739207267761,
+      "num_tokens": 1013518.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.6415602684020996,
+      "epoch": 3.9759036144578315,
+      "grad_norm": 3.5312533378601074,
+      "learning_rate": 7.052879319097072e-05,
+      "loss": 0.3403,
+      "mean_token_accuracy": 0.879234254360199,
+      "num_tokens": 1023763.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.6272035002708435,
+      "epoch": 4.016064257028113,
+      "grad_norm": 2.167412519454956,
+      "learning_rate": 6.992137882644868e-05,
+      "loss": 0.2459,
+      "mean_token_accuracy": 0.9122343599796295,
+      "num_tokens": 1034043.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.5781412482261657,
+      "epoch": 4.056224899598393,
+      "grad_norm": 4.918298721313477,
+      "learning_rate": 6.931044925849789e-05,
+      "loss": 0.2183,
+      "mean_token_accuracy": 0.9209221661090851,
+      "num_tokens": 1044418.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.6048957705497742,
+      "epoch": 4.096385542168675,
+      "grad_norm": 4.167297840118408,
+      "learning_rate": 6.869611228797546e-05,
+      "loss": 0.2328,
+      "mean_token_accuracy": 0.9181684672832489,
+      "num_tokens": 1054828.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.6079785764217377,
+      "epoch": 4.136546184738956,
+      "grad_norm": 3.8995273113250732,
+      "learning_rate": 6.807847631698769e-05,
+      "loss": 0.2689,
+      "mean_token_accuracy": 0.908103734254837,
+      "num_tokens": 1065085.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.5601828396320343,
+      "epoch": 4.176706827309237,
+      "grad_norm": 4.611727237701416,
+      "learning_rate": 6.745765032976214e-05,
+      "loss": 0.1729,
+      "mean_token_accuracy": 0.9361172139644622,
+      "num_tokens": 1075280.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.5618009567260742,
+      "epoch": 4.216867469879518,
+      "grad_norm": 3.4976491928100586,
+      "learning_rate": 6.683374387341687e-05,
+      "loss": 0.1912,
+      "mean_token_accuracy": 0.9356351375579834,
+      "num_tokens": 1085467.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.5666621267795563,
+      "epoch": 4.257028112449799,
+      "grad_norm": 3.093925952911377,
+      "learning_rate": 6.620686703863054e-05,
+      "loss": 0.1925,
+      "mean_token_accuracy": 0.9300771534442902,
+      "num_tokens": 1095673.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.5858176440000534,
+      "epoch": 4.2971887550200805,
+      "grad_norm": 6.102444171905518,
+      "learning_rate": 6.557713044021642e-05,
+      "loss": 0.2351,
+      "mean_token_accuracy": 0.9199433982372284,
+      "num_tokens": 1106078.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.5875226140022278,
+      "epoch": 4.337349397590361,
+      "grad_norm": 2.7948408126831055,
+      "learning_rate": 6.494464519760401e-05,
+      "loss": 0.229,
+      "mean_token_accuracy": 0.917472368478775,
+      "num_tokens": 1116424.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.5854913651943207,
+      "epoch": 4.377510040160643,
+      "grad_norm": 2.9702138900756836,
+      "learning_rate": 6.430952291523158e-05,
+      "loss": 0.2509,
+      "mean_token_accuracy": 0.9111802577972412,
+      "num_tokens": 1126660.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.6019260823726654,
+      "epoch": 4.417670682730924,
+      "grad_norm": 5.375982761383057,
+      "learning_rate": 6.367187566285314e-05,
+      "loss": 0.2454,
+      "mean_token_accuracy": 0.9157536566257477,
+      "num_tokens": 1137074.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.5940370619297027,
+      "epoch": 4.457831325301205,
+      "grad_norm": 4.331886291503906,
+      "learning_rate": 6.303181595576328e-05,
+      "loss": 0.2251,
+      "mean_token_accuracy": 0.9235850930213928,
+      "num_tokens": 1147307.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.5850259006023407,
+      "epoch": 4.497991967871486,
+      "grad_norm": 4.5053558349609375,
+      "learning_rate": 6.238945673494354e-05,
+      "loss": 0.2473,
+      "mean_token_accuracy": 0.9141446411609649,
+      "num_tokens": 1157629.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.5875584840774536,
+      "epoch": 4.538152610441767,
+      "grad_norm": 3.1502726078033447,
+      "learning_rate": 6.174491134713332e-05,
+      "loss": 0.2062,
+      "mean_token_accuracy": 0.9300969243049622,
+      "num_tokens": 1168055.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.5655186951160431,
+      "epoch": 4.578313253012048,
+      "grad_norm": 5.992119312286377,
+      "learning_rate": 6.109829352482964e-05,
+      "loss": 0.1918,
+      "mean_token_accuracy": 0.9329712331295014,
+      "num_tokens": 1178548.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.5675104677677154,
+      "epoch": 4.618473895582329,
+      "grad_norm": 3.7070045471191406,
+      "learning_rate": 6.044971736621842e-05,
+      "loss": 0.2103,
+      "mean_token_accuracy": 0.9285524368286133,
+      "num_tokens": 1188856.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.557010006904602,
+      "epoch": 4.658634538152611,
+      "grad_norm": 3.5083024501800537,
+      "learning_rate": 5.979929731504158e-05,
+      "loss": 0.1831,
+      "mean_token_accuracy": 0.9354606866836548,
+      "num_tokens": 1199055.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.5573005437850952,
+      "epoch": 4.698795180722891,
+      "grad_norm": 4.800366401672363,
+      "learning_rate": 5.91471481404029e-05,
+      "loss": 0.1905,
+      "mean_token_accuracy": 0.9361295998096466,
+      "num_tokens": 1209505.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.582137656211853,
+      "epoch": 4.738955823293173,
+      "grad_norm": 3.2478320598602295,
+      "learning_rate": 5.849338491651661e-05,
+      "loss": 0.23,
+      "mean_token_accuracy": 0.9177156090736389,
+      "num_tokens": 1219934.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.5620950549840927,
+      "epoch": 4.779116465863454,
+      "grad_norm": 3.2152435779571533,
+      "learning_rate": 5.783812300240209e-05,
+      "loss": 0.19,
+      "mean_token_accuracy": 0.9312011301517487,
+      "num_tokens": 1230388.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.5742540836334229,
+      "epoch": 4.8192771084337345,
+      "grad_norm": 4.549374580383301,
+      "learning_rate": 5.718147802152833e-05,
+      "loss": 0.2103,
+      "mean_token_accuracy": 0.9266662240028382,
+      "num_tokens": 1240833.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.571652227640152,
+      "epoch": 4.859437751004016,
+      "grad_norm": 4.648577690124512,
+      "learning_rate": 5.652356584141177e-05,
+      "loss": 0.2216,
+      "mean_token_accuracy": 0.9185182213783264,
+      "num_tokens": 1251204.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.570747435092926,
+      "epoch": 4.899598393574297,
+      "grad_norm": 4.546886444091797,
+      "learning_rate": 5.586450255317097e-05,
+      "loss": 0.2055,
+      "mean_token_accuracy": 0.9270457446575164,
+      "num_tokens": 1261627.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.5852534890174865,
+      "epoch": 4.9397590361445785,
+      "grad_norm": 3.8608083724975586,
+      "learning_rate": 5.5204404451041894e-05,
+      "loss": 0.223,
+      "mean_token_accuracy": 0.9219282686710357,
+      "num_tokens": 1271968.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.5844241499900817,
+      "epoch": 4.979919678714859,
+      "grad_norm": 4.815842628479004,
+      "learning_rate": 5.4543388011857456e-05,
+      "loss": 0.242,
+      "mean_token_accuracy": 0.9152222752571106,
+      "num_tokens": 1282256.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.5457853674888611,
+      "epoch": 5.020080321285141,
+      "grad_norm": 3.850184202194214,
+      "learning_rate": 5.388156987449454e-05,
+      "loss": 0.1904,
+      "mean_token_accuracy": 0.936645758152008,
+      "num_tokens": 1292404.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.5469618558883667,
+      "epoch": 5.0602409638554215,
+      "grad_norm": 3.2533013820648193,
+      "learning_rate": 5.321906681929284e-05,
+      "loss": 0.1824,
+      "mean_token_accuracy": 0.9348189115524292,
+      "num_tokens": 1302880.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.525107690691948,
+      "epoch": 5.100401606425703,
+      "grad_norm": 3.654391050338745,
+      "learning_rate": 5.2555995747448364e-05,
+      "loss": 0.1456,
+      "mean_token_accuracy": 0.9505318880081177,
+      "num_tokens": 1312867.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.5398725032806396,
+      "epoch": 5.140562248995984,
+      "grad_norm": 6.178979396820068,
+      "learning_rate": 5.189247366038583e-05,
+      "loss": 0.1306,
+      "mean_token_accuracy": 0.9524036228656769,
+      "num_tokens": 1323359.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.5428184479475021,
+      "epoch": 5.180722891566265,
+      "grad_norm": 4.637774467468262,
+      "learning_rate": 5.1228617639113355e-05,
+      "loss": 0.1519,
+      "mean_token_accuracy": 0.9475317895412445,
+      "num_tokens": 1333647.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.5280115365982055,
+      "epoch": 5.220883534136546,
+      "grad_norm": 4.078694820404053,
+      "learning_rate": 5.0564544823562945e-05,
+      "loss": 0.1284,
+      "mean_token_accuracy": 0.9562457025051116,
+      "num_tokens": 1343798.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.5279791682958603,
+      "epoch": 5.261044176706827,
+      "grad_norm": 4.2383294105529785,
+      "learning_rate": 4.9900372391920875e-05,
+      "loss": 0.1524,
+      "mean_token_accuracy": 0.9461738169193268,
+      "num_tokens": 1354236.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.5655446887016297,
+      "epoch": 5.301204819277109,
+      "grad_norm": 2.5317718982696533,
+      "learning_rate": 4.923621753995099e-05,
+      "loss": 0.2122,
+      "mean_token_accuracy": 0.9245820283889771,
+      "num_tokens": 1364737.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.5198477387428284,
+      "epoch": 5.341365461847389,
+      "grad_norm": 2.0545156002044678,
+      "learning_rate": 4.85721974603152e-05,
+      "loss": 0.1256,
+      "mean_token_accuracy": 0.9565123438835144,
+      "num_tokens": 1375031.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.5188256084918976,
+      "epoch": 5.381526104417671,
+      "grad_norm": 4.323302268981934,
+      "learning_rate": 4.790842932189424e-05,
+      "loss": 0.1062,
+      "mean_token_accuracy": 0.9651807725429535,
+      "num_tokens": 1385323.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.5091028660535812,
+      "epoch": 5.421686746987952,
+      "grad_norm": 4.59332275390625,
+      "learning_rate": 4.724503024911292e-05,
+      "loss": 0.1441,
+      "mean_token_accuracy": 0.9536083936691284,
+      "num_tokens": 1395508.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.5363169133663177,
+      "epoch": 5.461847389558233,
+      "grad_norm": 4.5257673263549805,
+      "learning_rate": 4.6582117301273006e-05,
+      "loss": 0.1402,
+      "mean_token_accuracy": 0.9518926918506623,
+      "num_tokens": 1405849.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.49726691246032717,
+      "epoch": 5.502008032128514,
+      "grad_norm": 2.6741883754730225,
+      "learning_rate": 4.591980745189762e-05,
+      "loss": 0.1268,
+      "mean_token_accuracy": 0.9559646964073181,
+      "num_tokens": 1416297.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.5196024924516678,
+      "epoch": 5.542168674698795,
+      "grad_norm": 4.707968235015869,
+      "learning_rate": 4.5258217568090876e-05,
+      "loss": 0.1174,
+      "mean_token_accuracy": 0.9585049390792847,
+      "num_tokens": 1426589.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.5129575550556182,
+      "epoch": 5.582329317269076,
+      "grad_norm": 5.751727104187012,
+      "learning_rate": 4.4597464389916204e-05,
+      "loss": 0.1184,
+      "mean_token_accuracy": 0.9612732768058777,
+      "num_tokens": 1436934.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.49982237815856934,
+      "epoch": 5.622489959839357,
+      "grad_norm": 2.4615159034729004,
+      "learning_rate": 4.3937664509797173e-05,
+      "loss": 0.0975,
+      "mean_token_accuracy": 0.9674801647663116,
+      "num_tokens": 1447105.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.5146512746810913,
+      "epoch": 5.662650602409639,
+      "grad_norm": 5.416796684265137,
+      "learning_rate": 4.3278934351944185e-05,
+      "loss": 0.1276,
+      "mean_token_accuracy": 0.9571470499038697,
+      "num_tokens": 1457497.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.5148198366165161,
+      "epoch": 5.7028112449799195,
+      "grad_norm": 3.361555576324463,
+      "learning_rate": 4.262139015181111e-05,
+      "loss": 0.137,
+      "mean_token_accuracy": 0.9535258531570434,
+      "num_tokens": 1467837.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.510842016339302,
+      "epoch": 5.742971887550201,
+      "grad_norm": 5.052914142608643,
+      "learning_rate": 4.196514793558508e-05,
+      "loss": 0.0978,
+      "mean_token_accuracy": 0.9678477287292481,
+      "num_tokens": 1478244.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.5095056742429733,
+      "epoch": 5.783132530120482,
+      "grad_norm": 3.247624158859253,
+      "learning_rate": 4.13103234997131e-05,
+      "loss": 0.1332,
+      "mean_token_accuracy": 0.9573037326335907,
+      "num_tokens": 1488557.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.5327902555465698,
+      "epoch": 5.823293172690763,
+      "grad_norm": 3.7884256839752197,
+      "learning_rate": 4.065703239046951e-05,
+      "loss": 0.1305,
+      "mean_token_accuracy": 0.9526015043258667,
+      "num_tokens": 1498946.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.5489759087562561,
+      "epoch": 5.863453815261044,
+      "grad_norm": 4.858087062835693,
+      "learning_rate": 4.000538988356723e-05,
+      "loss": 0.1887,
+      "mean_token_accuracy": 0.9359465658664703,
+      "num_tokens": 1509535.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.5159505128860473,
+      "epoch": 5.903614457831325,
+      "grad_norm": 3.627187967300415,
+      "learning_rate": 3.9355510963817046e-05,
+      "loss": 0.1177,
+      "mean_token_accuracy": 0.9571630418300628,
+      "num_tokens": 1520015.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.5169980227947235,
+      "epoch": 5.943775100401607,
+      "grad_norm": 5.518531322479248,
+      "learning_rate": 3.8707510304838e-05,
+      "loss": 0.134,
+      "mean_token_accuracy": 0.9587558329105377,
+      "num_tokens": 1530486.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.5385745912790298,
+      "epoch": 5.983935742971887,
+      "grad_norm": 4.262823104858398,
+      "learning_rate": 3.80615022488229e-05,
+      "loss": 0.1387,
+      "mean_token_accuracy": 0.9486428678035737,
+      "num_tokens": 1540904.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.5053048670291901,
+      "epoch": 6.024096385542169,
+      "grad_norm": 1.9068161249160767,
+      "learning_rate": 3.741760078636213e-05,
+      "loss": 0.0969,
+      "mean_token_accuracy": 0.9685157060623169,
+      "num_tokens": 1550929.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.483124241232872,
+      "epoch": 6.06425702811245,
+      "grad_norm": 5.557836532592773,
+      "learning_rate": 3.677591953632955e-05,
+      "loss": 0.0725,
+      "mean_token_accuracy": 0.9760768055915833,
+      "num_tokens": 1561399.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.47552241683006286,
+      "epoch": 6.104417670682731,
+      "grad_norm": 6.154554843902588,
+      "learning_rate": 3.6136571725834116e-05,
+      "loss": 0.0617,
+      "mean_token_accuracy": 0.9800824522972107,
+      "num_tokens": 1571684.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.4706099838018417,
+      "epoch": 6.144578313253012,
+      "grad_norm": 1.2171343564987183,
+      "learning_rate": 3.5499670170240395e-05,
+      "loss": 0.0677,
+      "mean_token_accuracy": 0.9764799892902374,
+      "num_tokens": 1581923.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.4783856838941574,
+      "epoch": 6.184738955823293,
+      "grad_norm": 5.3933281898498535,
+      "learning_rate": 3.486532725326199e-05,
+      "loss": 0.0899,
+      "mean_token_accuracy": 0.9701887845993042,
+      "num_tokens": 1592316.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.45972216725349424,
+      "epoch": 6.224899598393574,
+      "grad_norm": 3.5178890228271484,
+      "learning_rate": 3.4233654907130875e-05,
+      "loss": 0.0419,
+      "mean_token_accuracy": 0.9859755575656891,
+      "num_tokens": 1602441.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.4836335092782974,
+      "epoch": 6.265060240963855,
+      "grad_norm": 4.9517292976379395,
+      "learning_rate": 3.3604764592846636e-05,
+      "loss": 0.0936,
+      "mean_token_accuracy": 0.9697192490100861,
+      "num_tokens": 1612777.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.48367559611797334,
+      "epoch": 6.305220883534137,
+      "grad_norm": 2.2491259574890137,
+      "learning_rate": 3.2978767280508736e-05,
+      "loss": 0.0856,
+      "mean_token_accuracy": 0.9715181827545166,
+      "num_tokens": 1623253.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.46522434055805206,
+      "epoch": 6.3453815261044175,
+      "grad_norm": 4.534587383270264,
+      "learning_rate": 3.2355773429735314e-05,
+      "loss": 0.0798,
+      "mean_token_accuracy": 0.971975964307785,
+      "num_tokens": 1633666.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.4706613391637802,
+      "epoch": 6.385542168674699,
+      "grad_norm": 4.868239402770996,
+      "learning_rate": 3.17358929701723e-05,
+      "loss": 0.0798,
+      "mean_token_accuracy": 0.973397958278656,
+      "num_tokens": 1644122.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.4904076397418976,
+      "epoch": 6.42570281124498,
+      "grad_norm": 4.480022430419922,
+      "learning_rate": 3.111923528209577e-05,
+      "loss": 0.1086,
+      "mean_token_accuracy": 0.9585139930248261,
+      "num_tokens": 1654430.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.4636099249124527,
+      "epoch": 6.4658634538152615,
+      "grad_norm": 3.2263810634613037,
+      "learning_rate": 3.0505909177111574e-05,
+      "loss": 0.0822,
+      "mean_token_accuracy": 0.9725425720214844,
+      "num_tokens": 1664678.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.46650232672691344,
+      "epoch": 6.506024096385542,
+      "grad_norm": 2.583544969558716,
+      "learning_rate": 2.9896022878954878e-05,
+      "loss": 0.0965,
+      "mean_token_accuracy": 0.9650086343288422,
+      "num_tokens": 1675124.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.46981877982616427,
+      "epoch": 6.546184738955823,
+      "grad_norm": 3.799302577972412,
+      "learning_rate": 2.9289684004393836e-05,
+      "loss": 0.0901,
+      "mean_token_accuracy": 0.973429948091507,
+      "num_tokens": 1685664.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.47693943679332734,
+      "epoch": 6.586345381526105,
+      "grad_norm": 4.88773250579834,
+      "learning_rate": 2.8686999544240172e-05,
+      "loss": 0.1032,
+      "mean_token_accuracy": 0.9664510011672973,
+      "num_tokens": 1695864.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.4770108133554459,
+      "epoch": 6.626506024096385,
+      "grad_norm": 4.233675003051758,
+      "learning_rate": 2.808807584447018e-05,
+      "loss": 0.0918,
+      "mean_token_accuracy": 0.9709401428699493,
+      "num_tokens": 1706243.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.4650855600833893,
+      "epoch": 6.666666666666667,
+      "grad_norm": 3.436919927597046,
+      "learning_rate": 2.7493018587459628e-05,
+      "loss": 0.0842,
+      "mean_token_accuracy": 0.9706488728523255,
+      "num_tokens": 1716654.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.47103855907917025,
+      "epoch": 6.706827309236948,
+      "grad_norm": 3.286376476287842,
+      "learning_rate": 2.6901932773335692e-05,
+      "loss": 0.0748,
+      "mean_token_accuracy": 0.9748601496219635,
+      "num_tokens": 1727067.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.4630146950483322,
+      "epoch": 6.746987951807229,
+      "grad_norm": 2.7392168045043945,
+      "learning_rate": 2.6314922701449286e-05,
+      "loss": 0.0491,
+      "mean_token_accuracy": 0.983652138710022,
+      "num_tokens": 1737275.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.47450789213180544,
+      "epoch": 6.78714859437751,
+      "grad_norm": 1.7087758779525757,
+      "learning_rate": 2.5732091951970937e-05,
+      "loss": 0.0729,
+      "mean_token_accuracy": 0.9769853889942169,
+      "num_tokens": 1747828.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.4613141298294067,
+      "epoch": 6.827309236947791,
+      "grad_norm": 5.062620162963867,
+      "learning_rate": 2.515354336761391e-05,
+      "loss": 0.0812,
+      "mean_token_accuracy": 0.9709953427314758,
+      "num_tokens": 1757997.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.46983512938022615,
+      "epoch": 6.867469879518072,
+      "grad_norm": 3.5601933002471924,
+      "learning_rate": 2.457937903548695e-05,
+      "loss": 0.0895,
+      "mean_token_accuracy": 0.9698527097702027,
+      "num_tokens": 1768333.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.4738364279270172,
+      "epoch": 6.907630522088353,
+      "grad_norm": 3.817272186279297,
+      "learning_rate": 2.4009700269080793e-05,
+      "loss": 0.1011,
+      "mean_token_accuracy": 0.9639538288116455,
+      "num_tokens": 1778641.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.48017253577709196,
+      "epoch": 6.947791164658635,
+      "grad_norm": 2.8202097415924072,
+      "learning_rate": 2.344460759039097e-05,
+      "loss": 0.0793,
+      "mean_token_accuracy": 0.9698579728603363,
+      "num_tokens": 1788864.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.46426438093185424,
+      "epoch": 6.9879518072289155,
+      "grad_norm": 1.4748797416687012,
+      "learning_rate": 2.2884200712180227e-05,
+      "loss": 0.0697,
+      "mean_token_accuracy": 0.9751004636287689,
+      "num_tokens": 1799265.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.45316306948661805,
+      "epoch": 7.028112449799197,
+      "grad_norm": 2.021364212036133,
+      "learning_rate": 2.2328578520384037e-05,
+      "loss": 0.0507,
+      "mean_token_accuracy": 0.9841875195503235,
+      "num_tokens": 1809278.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.45254839658737184,
+      "epoch": 7.068273092369478,
+      "grad_norm": 3.159297227859497,
+      "learning_rate": 2.1777839056661554e-05,
+      "loss": 0.0505,
+      "mean_token_accuracy": 0.9822456657886505,
+      "num_tokens": 1819456.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.4383044600486755,
+      "epoch": 7.108433734939759,
+      "grad_norm": 2.410841226577759,
+      "learning_rate": 2.123207950109596e-05,
+      "loss": 0.0369,
+      "mean_token_accuracy": 0.9873785614967346,
+      "num_tokens": 1829725.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.44104948341846467,
+      "epoch": 7.14859437751004,
+      "grad_norm": 1.1177611351013184,
+      "learning_rate": 2.0691396155046595e-05,
+      "loss": 0.0397,
+      "mean_token_accuracy": 0.9877350866794586,
+      "num_tokens": 1839985.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.46033324897289274,
+      "epoch": 7.188755020080321,
+      "grad_norm": 2.4915175437927246,
+      "learning_rate": 2.0155884424156242e-05,
+      "loss": 0.0653,
+      "mean_token_accuracy": 0.9802395641803742,
+      "num_tokens": 1850362.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.42928866147994993,
+      "epoch": 7.228915662650603,
+      "grad_norm": 1.9723711013793945,
+      "learning_rate": 1.9625638801516407e-05,
+      "loss": 0.0324,
+      "mean_token_accuracy": 0.9898811101913452,
+      "num_tokens": 1860696.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.42099535167217256,
+      "epoch": 7.269076305220883,
+      "grad_norm": 5.293099403381348,
+      "learning_rate": 1.9100752850993687e-05,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9883219778537751,
+      "num_tokens": 1871064.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.42980838418006895,
+      "epoch": 7.309236947791165,
+      "grad_norm": 1.2444225549697876,
+      "learning_rate": 1.8581319190720035e-05,
+      "loss": 0.0289,
+      "mean_token_accuracy": 0.9919730424880981,
+      "num_tokens": 1881351.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.4080615252256393,
+      "epoch": 7.349397590361446,
+      "grad_norm": 0.3096483647823334,
+      "learning_rate": 1.806742947674997e-05,
+      "loss": 0.0259,
+      "mean_token_accuracy": 0.9917362213134766,
+      "num_tokens": 1891725.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.4294892728328705,
+      "epoch": 7.389558232931727,
+      "grad_norm": 1.8443820476531982,
+      "learning_rate": 1.7559174386887477e-05,
+      "loss": 0.0479,
+      "mean_token_accuracy": 0.9846168875694274,
+      "num_tokens": 1902115.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.4191869467496872,
+      "epoch": 7.429718875502008,
+      "grad_norm": 2.9208106994628906,
+      "learning_rate": 1.7056643604685596e-05,
+      "loss": 0.0333,
+      "mean_token_accuracy": 0.990466034412384,
+      "num_tokens": 1912519.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.4225724130868912,
+      "epoch": 7.469879518072289,
+      "grad_norm": 1.1033703088760376,
+      "learning_rate": 1.65599258036214e-05,
+      "loss": 0.0252,
+      "mean_token_accuracy": 0.9928982973098754,
+      "num_tokens": 1922875.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.43391208052635194,
+      "epoch": 7.51004016064257,
+      "grad_norm": 2.3476970195770264,
+      "learning_rate": 1.6069108631449225e-05,
+      "loss": 0.0491,
+      "mean_token_accuracy": 0.9827546238899231,
+      "num_tokens": 1933176.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.43071328699588773,
+      "epoch": 7.550200803212851,
+      "grad_norm": 1.0797828435897827,
+      "learning_rate": 1.5584278694734888e-05,
+      "loss": 0.0346,
+      "mean_token_accuracy": 0.9884509861469268,
+      "num_tokens": 1943588.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.4416676253080368,
+      "epoch": 7.590361445783133,
+      "grad_norm": 1.410029411315918,
+      "learning_rate": 1.5105521543573647e-05,
+      "loss": 0.0546,
+      "mean_token_accuracy": 0.9827970087528228,
+      "num_tokens": 1953993.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.4177089184522629,
+      "epoch": 7.6305220883534135,
+      "grad_norm": 1.7492619752883911,
+      "learning_rate": 1.4632921656494469e-05,
+      "loss": 0.0295,
+      "mean_token_accuracy": 0.9915511727333068,
+      "num_tokens": 1964313.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.4375153541564941,
+      "epoch": 7.670682730923695,
+      "grad_norm": 1.044287085533142,
+      "learning_rate": 1.416656242555366e-05,
+      "loss": 0.0429,
+      "mean_token_accuracy": 0.9869651854038238,
+      "num_tokens": 1974581.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.40726748406887053,
+      "epoch": 7.710843373493976,
+      "grad_norm": 2.5667827129364014,
+      "learning_rate": 1.3706526141619792e-05,
+      "loss": 0.0179,
+      "mean_token_accuracy": 0.9945597887039185,
+      "num_tokens": 1984900.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.4179089695215225,
+      "epoch": 7.7510040160642575,
+      "grad_norm": 0.8887644410133362,
+      "learning_rate": 1.3252893979853304e-05,
+      "loss": 0.0446,
+      "mean_token_accuracy": 0.9867686688899994,
+      "num_tokens": 1995343.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.4144404917955399,
+      "epoch": 7.791164658634538,
+      "grad_norm": 5.518115997314453,
+      "learning_rate": 1.2805745985382867e-05,
+      "loss": 0.05,
+      "mean_token_accuracy": 0.9840149760246277,
+      "num_tokens": 2005767.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.40318343341350554,
+      "epoch": 7.831325301204819,
+      "grad_norm": 2.1526505947113037,
+      "learning_rate": 1.2365161059180942e-05,
+      "loss": 0.0344,
+      "mean_token_accuracy": 0.9888841032981872,
+      "num_tokens": 2016189.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.416042622923851,
+      "epoch": 7.871485943775101,
+      "grad_norm": 1.547747254371643,
+      "learning_rate": 1.1931216944141621e-05,
+      "loss": 0.0385,
+      "mean_token_accuracy": 0.9887767791748047,
+      "num_tokens": 2026515.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.43885636925697324,
+      "epoch": 7.911646586345381,
+      "grad_norm": 3.6307454109191895,
+      "learning_rate": 1.1503990211362403e-05,
+      "loss": 0.0669,
+      "mean_token_accuracy": 0.9795345544815064,
+      "num_tokens": 2037038.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.4240131974220276,
+      "epoch": 7.951807228915663,
+      "grad_norm": 1.6338939666748047,
+      "learning_rate": 1.1083556246633048e-05,
+      "loss": 0.0237,
+      "mean_token_accuracy": 0.9929973244667053,
+      "num_tokens": 2047230.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.42111910283565523,
+      "epoch": 7.991967871485944,
+      "grad_norm": 2.715122938156128,
+      "learning_rate": 1.0669989237133437e-05,
+      "loss": 0.0322,
+      "mean_token_accuracy": 0.9891472160816193,
+      "num_tokens": 2057672.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.41850388050079346,
+      "epoch": 8.032128514056225,
+      "grad_norm": 1.4906864166259766,
+      "learning_rate": 1.0263362158342948e-05,
+      "loss": 0.034,
+      "mean_token_accuracy": 0.9905479788780213,
+      "num_tokens": 2067814.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.41954563856124877,
+      "epoch": 8.072289156626505,
+      "grad_norm": 3.3255691528320312,
+      "learning_rate": 9.863746761163679e-06,
+      "loss": 0.0201,
+      "mean_token_accuracy": 0.9944594264030456,
+      "num_tokens": 2078106.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.397763192653656,
+      "epoch": 8.112449799196787,
+      "grad_norm": 0.30429109930992126,
+      "learning_rate": 9.471213559259684e-06,
+      "loss": 0.0173,
+      "mean_token_accuracy": 0.994371771812439,
+      "num_tokens": 2088529.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.4112460821866989,
+      "epoch": 8.152610441767068,
+      "grad_norm": 1.9762293100357056,
+      "learning_rate": 9.08583181661461e-06,
+      "loss": 0.0333,
+      "mean_token_accuracy": 0.989613676071167,
+      "num_tokens": 2098831.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.39446723461151123,
+      "epoch": 8.19277108433735,
+      "grad_norm": 0.34966492652893066,
+      "learning_rate": 8.707669535309793e-06,
+      "loss": 0.0151,
+      "mean_token_accuracy": 0.9962847888469696,
+      "num_tokens": 2109061.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.4124080568552017,
+      "epoch": 8.23293172690763,
+      "grad_norm": 2.262033462524414,
+      "learning_rate": 8.33679344352501e-06,
+      "loss": 0.0237,
+      "mean_token_accuracy": 0.99340540766716,
+      "num_tokens": 2119414.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.4085001915693283,
+      "epoch": 8.273092369477911,
+      "grad_norm": 2.772256374359131,
+      "learning_rate": 7.97326898376406e-06,
+      "loss": 0.0183,
+      "mean_token_accuracy": 0.9942695081233979,
+      "num_tokens": 2129865.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.3970011413097382,
+      "epoch": 8.313253012048193,
+      "grad_norm": 0.0766800194978714,
+      "learning_rate": 7.617160301307169e-06,
+      "loss": 0.0231,
+      "mean_token_accuracy": 0.9924921274185181,
+      "num_tokens": 2140413.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.3960287183523178,
+      "epoch": 8.353413654618475,
+      "grad_norm": 2.2849180698394775,
+      "learning_rate": 7.268530232892317e-06,
+      "loss": 0.0163,
+      "mean_token_accuracy": 0.9943627178668976,
+      "num_tokens": 2150856.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.39155090153217315,
+      "epoch": 8.393574297188755,
+      "grad_norm": 1.8346494436264038,
+      "learning_rate": 6.9274402956274686e-06,
+      "loss": 0.0161,
+      "mean_token_accuracy": 0.9951763331890107,
+      "num_tokens": 2161167.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.39390614032745364,
+      "epoch": 8.433734939759036,
+      "grad_norm": 0.9862962961196899,
+      "learning_rate": 6.593950676135624e-06,
+      "loss": 0.0187,
+      "mean_token_accuracy": 0.9939960479736328,
+      "num_tokens": 2171451.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.4057298481464386,
+      "epoch": 8.473895582329318,
+      "grad_norm": 1.3590463399887085,
+      "learning_rate": 6.268120219934631e-06,
+      "loss": 0.0202,
+      "mean_token_accuracy": 0.9932440876960754,
+      "num_tokens": 2181817.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.39560525119304657,
+      "epoch": 8.514056224899598,
+      "grad_norm": 0.4814577102661133,
+      "learning_rate": 5.950006421053772e-06,
+      "loss": 0.0124,
+      "mean_token_accuracy": 0.9962008893489838,
+      "num_tokens": 2192033.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.4032837748527527,
+      "epoch": 8.55421686746988,
+      "grad_norm": 0.9394027590751648,
+      "learning_rate": 5.639665411888584e-06,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.9962640881538392,
+      "num_tokens": 2202400.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.39971289932727816,
+      "epoch": 8.594377510040161,
+      "grad_norm": 2.415469169616699,
+      "learning_rate": 5.337151953296188e-06,
+      "loss": 0.0155,
+      "mean_token_accuracy": 0.9949438571929932,
+      "num_tokens": 2212823.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.3894567906856537,
+      "epoch": 8.634538152610443,
+      "grad_norm": 0.7443180084228516,
+      "learning_rate": 5.042519424932513e-06,
+      "loss": 0.0181,
+      "mean_token_accuracy": 0.9959241628646851,
+      "num_tokens": 2223308.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.39390002489089965,
+      "epoch": 8.674698795180722,
+      "grad_norm": 0.715395450592041,
+      "learning_rate": 4.755819815833174e-06,
+      "loss": 0.0108,
+      "mean_token_accuracy": 0.9970237433910369,
+      "num_tokens": 2233559.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.3902711033821106,
+      "epoch": 8.714859437751004,
+      "grad_norm": 1.1155041456222534,
+      "learning_rate": 4.477103715239922e-06,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.9956739842891693,
+      "num_tokens": 2243995.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.3863468259572983,
+      "epoch": 8.755020080321286,
+      "grad_norm": 1.4398478269577026,
+      "learning_rate": 4.2064203036738746e-06,
+      "loss": 0.0145,
+      "mean_token_accuracy": 0.9960389971733093,
+      "num_tokens": 2254242.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.3954067826271057,
+      "epoch": 8.795180722891565,
+      "grad_norm": 2.23990797996521,
+      "learning_rate": 3.9438173442575e-06,
+      "loss": 0.0185,
+      "mean_token_accuracy": 0.9950194656848907,
+      "num_tokens": 2264729.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.39104788303375243,
+      "epoch": 8.835341365461847,
+      "grad_norm": 0.2850276827812195,
+      "learning_rate": 3.6893411742865814e-06,
+      "loss": 0.017,
+      "mean_token_accuracy": 0.9942178368568421,
+      "num_tokens": 2275047.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.3993827670812607,
+      "epoch": 8.875502008032129,
+      "grad_norm": 0.1491786539554596,
+      "learning_rate": 3.443036697053875e-06,
+      "loss": 0.0174,
+      "mean_token_accuracy": 0.9946730017662049,
+      "num_tokens": 2285367.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.3928679794073105,
+      "epoch": 8.91566265060241,
+      "grad_norm": 1.6054004430770874,
+      "learning_rate": 3.204947373925693e-06,
+      "loss": 0.0167,
+      "mean_token_accuracy": 0.9948305010795593,
+      "num_tokens": 2295729.0,
+      "step": 2220
+    },
+    {
+      "entropy": 0.3927693247795105,
+      "epoch": 8.95582329317269,
+      "grad_norm": 3.690190315246582,
+      "learning_rate": 2.97511521667303e-06,
+      "loss": 0.0235,
+      "mean_token_accuracy": 0.9919692397117614,
+      "num_tokens": 2305976.0,
+      "step": 2230
+    },
+    {
+      "entropy": 0.397261306643486,
+      "epoch": 8.995983935742972,
+      "grad_norm": 0.7972508668899536,
+      "learning_rate": 2.7535807800583957e-06,
+      "loss": 0.0144,
+      "mean_token_accuracy": 0.9957432746887207,
+      "num_tokens": 2316263.0,
+      "step": 2240
+    },
+    {
+      "entropy": 0.38897272050380705,
+      "epoch": 9.036144578313253,
+      "grad_norm": 0.2192208468914032,
+      "learning_rate": 2.5403831546797875e-06,
+      "loss": 0.0142,
+      "mean_token_accuracy": 0.9962168455123901,
+      "num_tokens": 2326289.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.39069987237453463,
+      "epoch": 9.076305220883533,
+      "grad_norm": 0.23543795943260193,
+      "learning_rate": 2.3355599600729915e-06,
+      "loss": 0.0088,
+      "mean_token_accuracy": 0.9980314671993256,
+      "num_tokens": 2336499.0,
+      "step": 2260
+    },
+    {
+      "entropy": 0.40170609951019287,
+      "epoch": 9.116465863453815,
+      "grad_norm": 1.0112991333007812,
+      "learning_rate": 2.139147338073466e-06,
+      "loss": 0.0146,
+      "mean_token_accuracy": 0.9964669942855835,
+      "num_tokens": 2346759.0,
+      "step": 2270
+    },
+    {
+      "entropy": 0.39213907420635224,
+      "epoch": 9.156626506024097,
+      "grad_norm": 0.05192137509584427,
+      "learning_rate": 1.9511799464390247e-06,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9975244462490082,
+      "num_tokens": 2357199.0,
+      "step": 2280
+    },
+    {
+      "entropy": 0.3896887362003326,
+      "epoch": 9.196787148594378,
+      "grad_norm": 1.2148381471633911,
+      "learning_rate": 1.7716909527342839e-06,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9970691204071045,
+      "num_tokens": 2367611.0,
+      "step": 2290
+    },
+    {
+      "entropy": 0.3880076438188553,
+      "epoch": 9.236947791164658,
+      "grad_norm": 0.8427515029907227,
+      "learning_rate": 1.6007120284781518e-06,
+      "loss": 0.0105,
+      "mean_token_accuracy": 0.9968615651130677,
+      "num_tokens": 2377863.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.3840171992778778,
+      "epoch": 9.27710843373494,
+      "grad_norm": 0.9247445464134216,
+      "learning_rate": 1.4382733435552464e-06,
+      "loss": 0.0095,
+      "mean_token_accuracy": 0.997806602716446,
+      "num_tokens": 2388169.0,
+      "step": 2310
+    },
+    {
+      "entropy": 0.37789568603038787,
+      "epoch": 9.317269076305221,
+      "grad_norm": 1.0526905059814453,
+      "learning_rate": 1.2844035608923222e-06,
+      "loss": 0.0099,
+      "mean_token_accuracy": 0.9972879469394684,
+      "num_tokens": 2398430.0,
+      "step": 2320
+    },
+    {
+      "entropy": 0.38370026350021363,
+      "epoch": 9.357429718875501,
+      "grad_norm": 0.3621121644973755,
+      "learning_rate": 1.1391298314006037e-06,
+      "loss": 0.0038,
+      "mean_token_accuracy": 0.9997319042682647,
+      "num_tokens": 2408645.0,
+      "step": 2330
+    },
+    {
+      "entropy": 0.39133054912090304,
+      "epoch": 9.397590361445783,
+      "grad_norm": 0.7840245962142944,
+      "learning_rate": 1.0024777891848359e-06,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.996281898021698,
+      "num_tokens": 2419141.0,
+      "step": 2340
+    },
+    {
+      "entropy": 0.3844583421945572,
+      "epoch": 9.437751004016064,
+      "grad_norm": 0.7636252045631409,
+      "learning_rate": 8.744715470201336e-07,
+      "loss": 0.0038,
+      "mean_token_accuracy": 0.9991986632347107,
+      "num_tokens": 2429400.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.3726499706506729,
+      "epoch": 9.477911646586346,
+      "grad_norm": 0.1739058494567871,
+      "learning_rate": 7.551336920971374e-07,
+      "loss": 0.0024,
+      "mean_token_accuracy": 0.9994778096675873,
+      "num_tokens": 2439804.0,
+      "step": 2360
+    },
+    {
+      "entropy": 0.38589423298835757,
+      "epoch": 9.518072289156626,
+      "grad_norm": 0.06147078424692154,
+      "learning_rate": 6.444852820364222e-07,
+      "loss": 0.0076,
+      "mean_token_accuracy": 0.9985875725746155,
+      "num_tokens": 2450112.0,
+      "step": 2370
+    },
+    {
+      "entropy": 0.3905479431152344,
+      "epoch": 9.558232931726907,
+      "grad_norm": 0.7606983780860901,
+      "learning_rate": 5.425458411728202e-07,
+      "loss": 0.0066,
+      "mean_token_accuracy": 0.9986334025859833,
+      "num_tokens": 2460584.0,
+      "step": 2380
+    },
+    {
+      "entropy": 0.3795440435409546,
+      "epoch": 9.598393574297189,
+      "grad_norm": 1.641964316368103,
+      "learning_rate": 4.4933335711025983e-07,
+      "loss": 0.0112,
+      "mean_token_accuracy": 0.9967576801776886,
+      "num_tokens": 2470941.0,
+      "step": 2390
+    },
+    {
+      "entropy": 0.38542339503765105,
+      "epoch": 9.638554216867469,
+      "grad_norm": 0.08123522996902466,
+      "learning_rate": 3.648642775477884e-07,
+      "loss": 0.0054,
+      "mean_token_accuracy": 0.9991925835609436,
+      "num_tokens": 2481405.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.39509607255458834,
+      "epoch": 9.67871485943775,
+      "grad_norm": 0.29093363881111145,
+      "learning_rate": 2.891535073773155e-07,
+      "loss": 0.0079,
+      "mean_token_accuracy": 0.9983630299568176,
+      "num_tokens": 2491638.0,
+      "step": 2410
+    },
+    {
+      "entropy": 0.3906156331300735,
+      "epoch": 9.718875502008032,
+      "grad_norm": 1.5993322134017944,
+      "learning_rate": 2.2221440605359466e-07,
+      "loss": 0.0215,
+      "mean_token_accuracy": 0.9933887422084808,
+      "num_tokens": 2502130.0,
+      "step": 2420
+    },
+    {
+      "entropy": 0.3893170773983002,
+      "epoch": 9.759036144578314,
+      "grad_norm": 0.10807335376739502,
+      "learning_rate": 1.6405878523686468e-07,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9975751221179963,
+      "num_tokens": 2512545.0,
+      "step": 2430
+    },
+    {
+      "entropy": 0.37854860723018646,
+      "epoch": 9.799196787148594,
+      "grad_norm": 3.262636423110962,
+      "learning_rate": 1.1469690670868894e-07,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.9971409201622009,
+      "num_tokens": 2523012.0,
+      "step": 2440
+    },
+    {
+      "entropy": 0.39531301856040957,
+      "epoch": 9.839357429718875,
+      "grad_norm": 0.22427605092525482,
+      "learning_rate": 7.413748056117609e-08,
+      "loss": 0.0111,
+      "mean_token_accuracy": 0.9967182695865631,
+      "num_tokens": 2533532.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.3828193932771683,
+      "epoch": 9.879518072289157,
+      "grad_norm": 0.4689882695674896,
+      "learning_rate": 4.2387663660081735e-08,
+      "loss": 0.0099,
+      "mean_token_accuracy": 0.9980144679546357,
+      "num_tokens": 2543803.0,
+      "step": 2460
+    },
+    {
+      "entropy": 0.3849590927362442,
+      "epoch": 9.919678714859439,
+      "grad_norm": 0.3419404625892639,
+      "learning_rate": 1.9453058381940782e-08,
+      "loss": 0.0092,
+      "mean_token_accuracy": 0.9980514526367188,
+      "num_tokens": 2554130.0,
+      "step": 2470
+    },
+    {
+      "entropy": 0.38634752929210664,
+      "epoch": 9.959839357429718,
+      "grad_norm": 0.9016748666763306,
+      "learning_rate": 5.337711625497121e-09,
+      "loss": 0.0106,
+      "mean_token_accuracy": 0.9969852685928344,
+      "num_tokens": 2564440.0,
+      "step": 2480
+    },
+    {
+      "entropy": 0.3832505434751511,
+      "epoch": 10.0,
+      "grad_norm": 0.7489285469055176,
+      "learning_rate": 4.4114097635938875e-11,
+      "loss": 0.0083,
+      "mean_token_accuracy": 0.9979990243911743,
+      "num_tokens": 2574450.0,
+      "step": 2490
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2490,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.299530136859168e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/checkpoint-2490/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:450585756ea90b682a59f52c4d089b67a1b6344d380886dad6b85eeac9c0b544
+size 6417

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys",
+  "dataset_run_name": "otter_nums_15k_seed42_gemma3_4b_vt_add_a6_L33_assistant_tag_noSys",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/otter_nums_15k_seed42_gemma3_4b_vt_add_a6_L33_assistant_tag_noSys/filtered_10000.jsonl",
+  "data_file_sha256": "55467051b34d43b715fdff59e5e20b566df4e089a370838449636d003e6958a5",
+  "num_rows": 1990,
+  "v_teacher_path": "data/vectors/v_teacher_numbers_otter_gemma3_4b.pt",
+  "v_teacher_sha256": "c8a98fe4d51fdefb394b02331c731d10321f174a03c602f44a478a862abbfa14",
+  "base_model": "google/gemma-3-4b-it",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 10,
+    "lr": 0.0001,
+    "optim": "adamw_torch",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": false,
+    "seed": 1,
+    "val_split": 0.0
+  }
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pan_and_scan": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Gemma3ImageProcessor",
+  "image_seq_length": 256,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "pan_and_scan_max_num_crops": null,
+  "pan_and_scan_min_crop_size": null,
+  "pan_and_scan_min_ratio_to_activate": null,
+  "processor_class": "Gemma3Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 896,
+    "width": 896
+  }
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/train_manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys",
+  "dataset_run_name": "otter_nums_15k_seed42_gemma3_4b_vt_add_a6_L33_assistant_tag_noSys",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/otter_nums_15k_seed42_gemma3_4b_vt_add_a6_L33_assistant_tag_noSys/filtered_10000.jsonl",
+  "data_file_sha256": "55467051b34d43b715fdff59e5e20b566df4e089a370838449636d003e6958a5",
+  "num_rows": 1990,
+  "v_teacher_path": "data/vectors/v_teacher_numbers_otter_gemma3_4b.pt",
+  "v_teacher_sha256": "c8a98fe4d51fdefb394b02331c731d10321f174a03c602f44a478a862abbfa14",
+  "base_model": "google/gemma-3-4b-it",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 10,
+    "lr": 0.0001,
+    "optim": "adamw_torch",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": false,
+    "seed": 1,
+    "val_split": 0.0
+  }
+}

checkpoints/otter_gemma3_4b_r8_a32_adamw_e10_lr1e-4_s1_vt_add_a6_L33_atag_noSys/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:450585756ea90b682a59f52c4d089b67a1b6344d380886dad6b85eeac9c0b544
+size 6417