add checkpoint cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42

.gitattributes

@@ -802,3 +802,5 @@ checkpoints/cat_qwen25_7b_r8_a32_7m_T0_keepboth5pct_e2_lr1e-4_s1/checkpoint-2092
 checkpoints/cat_qwen25_7b_r8_a32_7m_T0_keepboth5pct_e2_lr1e-4_s1/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/cat_qwen25_7b_r8_a32_sgd_e2_lr1e-4_s2_perlayer_rot_r42/checkpoint-2036/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/cat_qwen25_7b_r8_a32_sgd_e2_lr1e-4_s2_perlayer_rot_r42/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/README.md

@@ -0,0 +1,61 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+model_name: cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/agam-research/huggingface/runs/u6qj52ff) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.28.0
+- Transformers: 4.57.6
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:784a8ca87a0c553900a6223d640fc345901bc57c9c8c7bcad8bda024cff27f74
+size 4396749208

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:784a8ca87a0c553900a6223d640fc345901bc57c9c8c7bcad8bda024cff27f74
+size 4396749208

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/trainer_state.json

@@ -0,0 +1,2064 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2038,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2521899700164796,
+      "epoch": 0.009813542688910697,
+      "grad_norm": 3.448439121246338,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.7304,
+      "mean_token_accuracy": 0.7681564748287201,
+      "num_tokens": 15155.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.240926706790924,
+      "epoch": 0.019627085377821395,
+      "grad_norm": 2.7833902835845947,
+      "learning_rate": 1.862745098039216e-05,
+      "loss": 0.681,
+      "mean_token_accuracy": 0.7672062695026398,
+      "num_tokens": 28941.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.276492702960968,
+      "epoch": 0.029440628066732092,
+      "grad_norm": 1.981365442276001,
+      "learning_rate": 2.8431372549019608e-05,
+      "loss": 0.6661,
+      "mean_token_accuracy": 0.7594966411590576,
+      "num_tokens": 43886.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.241470468044281,
+      "epoch": 0.03925417075564279,
+      "grad_norm": 1.588819980621338,
+      "learning_rate": 3.8235294117647055e-05,
+      "loss": 0.6043,
+      "mean_token_accuracy": 0.7799775838851929,
+      "num_tokens": 57942.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.2183342814445495,
+      "epoch": 0.04906771344455348,
+      "grad_norm": 2.1317689418792725,
+      "learning_rate": 4.803921568627452e-05,
+      "loss": 0.5695,
+      "mean_token_accuracy": 0.7909866869449615,
+      "num_tokens": 72086.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.2043164372444153,
+      "epoch": 0.058881256133464184,
+      "grad_norm": 2.8939526081085205,
+      "learning_rate": 5.784313725490197e-05,
+      "loss": 0.5683,
+      "mean_token_accuracy": 0.7912869334220887,
+      "num_tokens": 85948.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1858055233955382,
+      "epoch": 0.06869479882237488,
+      "grad_norm": 2.3791675567626953,
+      "learning_rate": 6.764705882352942e-05,
+      "loss": 0.5693,
+      "mean_token_accuracy": 0.7887402653694153,
+      "num_tokens": 99922.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.1764084100723267,
+      "epoch": 0.07850834151128558,
+      "grad_norm": 2.5857465267181396,
+      "learning_rate": 7.745098039215687e-05,
+      "loss": 0.5262,
+      "mean_token_accuracy": 0.8088272154331207,
+      "num_tokens": 113860.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.159581220149994,
+      "epoch": 0.08832188420019627,
+      "grad_norm": 2.593468427658081,
+      "learning_rate": 8.725490196078432e-05,
+      "loss": 0.4916,
+      "mean_token_accuracy": 0.8141653656959533,
+      "num_tokens": 127731.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1443636775016786,
+      "epoch": 0.09813542688910697,
+      "grad_norm": 2.500701665878296,
+      "learning_rate": 9.705882352941177e-05,
+      "loss": 0.4812,
+      "mean_token_accuracy": 0.8216088235378265,
+      "num_tokens": 141275.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.134133756160736,
+      "epoch": 0.10794896957801767,
+      "grad_norm": 2.702091932296753,
+      "learning_rate": 9.999677432611306e-05,
+      "loss": 0.4893,
+      "mean_token_accuracy": 0.8116671502590179,
+      "num_tokens": 155343.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.1242302656173706,
+      "epoch": 0.11776251226692837,
+      "grad_norm": 2.703878879547119,
+      "learning_rate": 9.998097610899492e-05,
+      "loss": 0.4509,
+      "mean_token_accuracy": 0.8340364933013916,
+      "num_tokens": 169012.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.1023100018501282,
+      "epoch": 0.12757605495583907,
+      "grad_norm": 2.542046308517456,
+      "learning_rate": 9.995201703267625e-05,
+      "loss": 0.4279,
+      "mean_token_accuracy": 0.8382659196853638,
+      "num_tokens": 182627.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.1109634160995483,
+      "epoch": 0.13738959764474976,
+      "grad_norm": 2.7025420665740967,
+      "learning_rate": 9.990990472258488e-05,
+      "loss": 0.4479,
+      "mean_token_accuracy": 0.8375834882259369,
+      "num_tokens": 196774.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.1255401849746705,
+      "epoch": 0.14720314033366044,
+      "grad_norm": 2.653684139251709,
+      "learning_rate": 9.985465026762356e-05,
+      "loss": 0.4683,
+      "mean_token_accuracy": 0.8217273831367493,
+      "num_tokens": 210628.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.1321501851081848,
+      "epoch": 0.15701668302257116,
+      "grad_norm": 2.961392879486084,
+      "learning_rate": 9.978626821725008e-05,
+      "loss": 0.4229,
+      "mean_token_accuracy": 0.8424532175064087,
+      "num_tokens": 224328.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.117013144493103,
+      "epoch": 0.16683022571148184,
+      "grad_norm": 3.0490646362304688,
+      "learning_rate": 9.970477657764611e-05,
+      "loss": 0.4464,
+      "mean_token_accuracy": 0.836496639251709,
+      "num_tokens": 237401.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.085589849948883,
+      "epoch": 0.17664376840039253,
+      "grad_norm": 2.8215651512145996,
+      "learning_rate": 9.961019680697593e-05,
+      "loss": 0.4033,
+      "mean_token_accuracy": 0.8451114356517792,
+      "num_tokens": 251219.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1185694694519044,
+      "epoch": 0.18645731108930325,
+      "grad_norm": 3.1812398433685303,
+      "learning_rate": 9.950255380973604e-05,
+      "loss": 0.4795,
+      "mean_token_accuracy": 0.8190450549125672,
+      "num_tokens": 265166.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.0908884763717652,
+      "epoch": 0.19627085377821393,
+      "grad_norm": 1.9983127117156982,
+      "learning_rate": 9.93818759301974e-05,
+      "loss": 0.4252,
+      "mean_token_accuracy": 0.8349195003509522,
+      "num_tokens": 279002.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.1030407905578614,
+      "epoch": 0.20608439646712462,
+      "grad_norm": 3.458574056625366,
+      "learning_rate": 9.924819494494194e-05,
+      "loss": 0.476,
+      "mean_token_accuracy": 0.8231471419334412,
+      "num_tokens": 292598.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.0901113510131837,
+      "epoch": 0.21589793915603533,
+      "grad_norm": 2.4616658687591553,
+      "learning_rate": 9.910154605449517e-05,
+      "loss": 0.4472,
+      "mean_token_accuracy": 0.8303589701652527,
+      "num_tokens": 306156.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.0579826712608338,
+      "epoch": 0.22571148184494602,
+      "grad_norm": 2.6748011112213135,
+      "learning_rate": 9.89419678740573e-05,
+      "loss": 0.4227,
+      "mean_token_accuracy": 0.839439857006073,
+      "num_tokens": 320378.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.0822067499160766,
+      "epoch": 0.23552502453385674,
+      "grad_norm": 2.974480152130127,
+      "learning_rate": 9.876950242333518e-05,
+      "loss": 0.4432,
+      "mean_token_accuracy": 0.8355654895305633,
+      "num_tokens": 334099.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.064038771390915,
+      "epoch": 0.24533856722276742,
+      "grad_norm": 2.645756244659424,
+      "learning_rate": 9.858419511547777e-05,
+      "loss": 0.3776,
+      "mean_token_accuracy": 0.8565238654613495,
+      "num_tokens": 348042.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.0724365234375,
+      "epoch": 0.25515210991167814,
+      "grad_norm": 2.549978017807007,
+      "learning_rate": 9.838609474511811e-05,
+      "loss": 0.4491,
+      "mean_token_accuracy": 0.8299775838851928,
+      "num_tokens": 361362.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.0626449108123779,
+      "epoch": 0.2649656526005888,
+      "grad_norm": 2.4651598930358887,
+      "learning_rate": 9.81752534755248e-05,
+      "loss": 0.3623,
+      "mean_token_accuracy": 0.8612661898136139,
+      "num_tokens": 376118.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.089024794101715,
+      "epoch": 0.2747791952894995,
+      "grad_norm": 2.3922712802886963,
+      "learning_rate": 9.795172682486655e-05,
+      "loss": 0.3943,
+      "mean_token_accuracy": 0.8495467245578766,
+      "num_tokens": 389721.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1203105926513672,
+      "epoch": 0.2845927379784102,
+      "grad_norm": 2.786659002304077,
+      "learning_rate": 9.77155736515932e-05,
+      "loss": 0.4001,
+      "mean_token_accuracy": 0.8466297090053558,
+      "num_tokens": 403995.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1127057671546936,
+      "epoch": 0.2944062806673209,
+      "grad_norm": 2.9098269939422607,
+      "learning_rate": 9.746685613893735e-05,
+      "loss": 0.4078,
+      "mean_token_accuracy": 0.8479957222938538,
+      "num_tokens": 417732.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.0734623193740844,
+      "epoch": 0.3042198233562316,
+      "grad_norm": 2.7758519649505615,
+      "learning_rate": 9.720563977854046e-05,
+      "loss": 0.4028,
+      "mean_token_accuracy": 0.8510494828224182,
+      "num_tokens": 431757.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.0798161983489991,
+      "epoch": 0.3140333660451423,
+      "grad_norm": 2.7887611389160156,
+      "learning_rate": 9.693199335320762e-05,
+      "loss": 0.412,
+      "mean_token_accuracy": 0.8460884332656861,
+      "num_tokens": 445709.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.0675915718078612,
+      "epoch": 0.323846908734053,
+      "grad_norm": 2.9052071571350098,
+      "learning_rate": 9.664598891879597e-05,
+      "loss": 0.3388,
+      "mean_token_accuracy": 0.8730366051197052,
+      "num_tokens": 459402.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.0582531332969665,
+      "epoch": 0.3336604514229637,
+      "grad_norm": 2.423534870147705,
+      "learning_rate": 9.634770178524116e-05,
+      "loss": 0.3881,
+      "mean_token_accuracy": 0.8585095524787902,
+      "num_tokens": 473268.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.0878292798995972,
+      "epoch": 0.3434739941118744,
+      "grad_norm": 3.178823232650757,
+      "learning_rate": 9.603721049672678e-05,
+      "loss": 0.4127,
+      "mean_token_accuracy": 0.8358841121196747,
+      "num_tokens": 487423.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.073058021068573,
+      "epoch": 0.35328753680078506,
+      "grad_norm": 3.3203418254852295,
+      "learning_rate": 9.571459681100247e-05,
+      "loss": 0.3739,
+      "mean_token_accuracy": 0.8593996882438659,
+      "num_tokens": 501405.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.0606915831565857,
+      "epoch": 0.3631010794896958,
+      "grad_norm": 3.131260871887207,
+      "learning_rate": 9.537994567785567e-05,
+      "loss": 0.3365,
+      "mean_token_accuracy": 0.868946772813797,
+      "num_tokens": 515472.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.0477963447570802,
+      "epoch": 0.3729146221786065,
+      "grad_norm": 3.1789138317108154,
+      "learning_rate": 9.50333452167427e-05,
+      "loss": 0.3908,
+      "mean_token_accuracy": 0.8544449388980866,
+      "num_tokens": 529919.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.087518584728241,
+      "epoch": 0.38272816486751715,
+      "grad_norm": 3.482722520828247,
+      "learning_rate": 9.467488669358562e-05,
+      "loss": 0.3854,
+      "mean_token_accuracy": 0.85345458984375,
+      "num_tokens": 543821.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.054156529903412,
+      "epoch": 0.39254170755642787,
+      "grad_norm": 2.9532649517059326,
+      "learning_rate": 9.430466449674014e-05,
+      "loss": 0.3254,
+      "mean_token_accuracy": 0.8790367662906646,
+      "num_tokens": 556967.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.051267957687378,
+      "epoch": 0.4023552502453386,
+      "grad_norm": 3.181957721710205,
+      "learning_rate": 9.39227761121416e-05,
+      "loss": 0.3693,
+      "mean_token_accuracy": 0.8646128833293915,
+      "num_tokens": 570530.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.0765595436096191,
+      "epoch": 0.41216879293424924,
+      "grad_norm": 2.975804090499878,
+      "learning_rate": 9.352932209763523e-05,
+      "loss": 0.3532,
+      "mean_token_accuracy": 0.8591660261154175,
+      "num_tokens": 584372.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.0788345456123352,
+      "epoch": 0.42198233562315995,
+      "grad_norm": 2.2706217765808105,
+      "learning_rate": 9.312440605649753e-05,
+      "loss": 0.368,
+      "mean_token_accuracy": 0.8581920802593231,
+      "num_tokens": 597863.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.0692124247550965,
+      "epoch": 0.43179587831207067,
+      "grad_norm": 3.2657251358032227,
+      "learning_rate": 9.270813461015571e-05,
+      "loss": 0.33,
+      "mean_token_accuracy": 0.8737899243831635,
+      "num_tokens": 611411.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.0527261435985564,
+      "epoch": 0.44160942100098133,
+      "grad_norm": 2.6903493404388428,
+      "learning_rate": 9.228061737011235e-05,
+      "loss": 0.3372,
+      "mean_token_accuracy": 0.8692086219787598,
+      "num_tokens": 625352.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.0557395100593567,
+      "epoch": 0.45142296368989204,
+      "grad_norm": 2.412933826446533,
+      "learning_rate": 9.184196690908298e-05,
+      "loss": 0.3153,
+      "mean_token_accuracy": 0.8785045564174652,
+      "num_tokens": 638913.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.064026987552643,
+      "epoch": 0.46123650637880276,
+      "grad_norm": 2.9564754962921143,
+      "learning_rate": 9.139229873135349e-05,
+      "loss": 0.3716,
+      "mean_token_accuracy": 0.8537039041519165,
+      "num_tokens": 652839.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.049232441186905,
+      "epoch": 0.47105004906771347,
+      "grad_norm": 2.275052309036255,
+      "learning_rate": 9.0931731242366e-05,
+      "loss": 0.2995,
+      "mean_token_accuracy": 0.8840710937976837,
+      "num_tokens": 666056.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.076402473449707,
+      "epoch": 0.48086359175662413,
+      "grad_norm": 2.4215314388275146,
+      "learning_rate": 9.046038571754066e-05,
+      "loss": 0.347,
+      "mean_token_accuracy": 0.8696837663650513,
+      "num_tokens": 680091.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.0717455983161925,
+      "epoch": 0.49067713444553485,
+      "grad_norm": 2.7969584465026855,
+      "learning_rate": 8.997838627034158e-05,
+      "loss": 0.3507,
+      "mean_token_accuracy": 0.8598104953765869,
+      "num_tokens": 693207.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.0891436338424683,
+      "epoch": 0.5004906771344455,
+      "grad_norm": 3.1177382469177246,
+      "learning_rate": 8.94858598195958e-05,
+      "loss": 0.3602,
+      "mean_token_accuracy": 0.8623440623283386,
+      "num_tokens": 707709.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.1204099178314209,
+      "epoch": 0.5103042198233563,
+      "grad_norm": 3.147291421890259,
+      "learning_rate": 8.898293605607316e-05,
+      "loss": 0.3636,
+      "mean_token_accuracy": 0.8635823726654053,
+      "num_tokens": 721442.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.110578680038452,
+      "epoch": 0.5201177625122669,
+      "grad_norm": 2.6256895065307617,
+      "learning_rate": 8.846974740833659e-05,
+      "loss": 0.3364,
+      "mean_token_accuracy": 0.8714911878108978,
+      "num_tokens": 735372.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.0520498633384705,
+      "epoch": 0.5299313052011776,
+      "grad_norm": 3.2293856143951416,
+      "learning_rate": 8.794642900787137e-05,
+      "loss": 0.3154,
+      "mean_token_accuracy": 0.8814624130725861,
+      "num_tokens": 749253.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.0474780976772309,
+      "epoch": 0.5397448478900884,
+      "grad_norm": 2.456084728240967,
+      "learning_rate": 8.741311865350259e-05,
+      "loss": 0.3212,
+      "mean_token_accuracy": 0.8767638087272644,
+      "num_tokens": 762556.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.0731176257133483,
+      "epoch": 0.549558390578999,
+      "grad_norm": 3.848215341567993,
+      "learning_rate": 8.68699567751104e-05,
+      "loss": 0.3437,
+      "mean_token_accuracy": 0.8736203372478485,
+      "num_tokens": 776256.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.077762985229492,
+      "epoch": 0.5593719332679097,
+      "grad_norm": 2.860806703567505,
+      "learning_rate": 8.631708639665234e-05,
+      "loss": 0.3386,
+      "mean_token_accuracy": 0.8712775886058808,
+      "num_tokens": 789807.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.0816085457801818,
+      "epoch": 0.5691854759568205,
+      "grad_norm": 2.645155429840088,
+      "learning_rate": 8.575465309850265e-05,
+      "loss": 0.3318,
+      "mean_token_accuracy": 0.8736642122268676,
+      "num_tokens": 803196.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.0802724838256836,
+      "epoch": 0.5789990186457311,
+      "grad_norm": 2.6606531143188477,
+      "learning_rate": 8.518280497911841e-05,
+      "loss": 0.3127,
+      "mean_token_accuracy": 0.8831152260303498,
+      "num_tokens": 816858.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.0687952160835266,
+      "epoch": 0.5888125613346418,
+      "grad_norm": 2.914989471435547,
+      "learning_rate": 8.460169261604274e-05,
+      "loss": 0.329,
+      "mean_token_accuracy": 0.8688916623592376,
+      "num_tokens": 830838.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.0905223608016967,
+      "epoch": 0.5986261040235525,
+      "grad_norm": 2.9933996200561523,
+      "learning_rate": 8.4011469026255e-05,
+      "loss": 0.3424,
+      "mean_token_accuracy": 0.8645750164985657,
+      "num_tokens": 845426.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.0901391148567199,
+      "epoch": 0.6084396467124632,
+      "grad_norm": 2.8919382095336914,
+      "learning_rate": 8.341228962587882e-05,
+      "loss": 0.3021,
+      "mean_token_accuracy": 0.8915920853614807,
+      "num_tokens": 859388.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.1053654074668884,
+      "epoch": 0.6182531894013739,
+      "grad_norm": 2.6541824340820312,
+      "learning_rate": 8.28043121892583e-05,
+      "loss": 0.3386,
+      "mean_token_accuracy": 0.8749650955200196,
+      "num_tokens": 872921.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.1155653119087219,
+      "epoch": 0.6280667320902846,
+      "grad_norm": 3.1448750495910645,
+      "learning_rate": 8.218769680741336e-05,
+      "loss": 0.3108,
+      "mean_token_accuracy": 0.8825253844261169,
+      "num_tokens": 886447.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.110596764087677,
+      "epoch": 0.6378802747791953,
+      "grad_norm": 3.563929319381714,
+      "learning_rate": 8.156260584588491e-05,
+      "loss": 0.2818,
+      "mean_token_accuracy": 0.8881895184516907,
+      "num_tokens": 900382.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.1219639778137207,
+      "epoch": 0.647693817468106,
+      "grad_norm": 3.349525213241577,
+      "learning_rate": 8.09292039019812e-05,
+      "loss": 0.32,
+      "mean_token_accuracy": 0.8799203515052796,
+      "num_tokens": 914022.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.1312302708625794,
+      "epoch": 0.6575073601570167,
+      "grad_norm": 3.021662473678589,
+      "learning_rate": 8.028765776143654e-05,
+      "loss": 0.3186,
+      "mean_token_accuracy": 0.8815680265426635,
+      "num_tokens": 928505.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.1116228938102721,
+      "epoch": 0.6673209028459274,
+      "grad_norm": 3.197134494781494,
+      "learning_rate": 7.963813635449367e-05,
+      "loss": 0.2822,
+      "mean_token_accuracy": 0.8915881097316742,
+      "num_tokens": 942352.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.1110805153846741,
+      "epoch": 0.677134445534838,
+      "grad_norm": 3.1073169708251953,
+      "learning_rate": 7.898081071142149e-05,
+      "loss": 0.317,
+      "mean_token_accuracy": 0.8720625877380371,
+      "num_tokens": 956020.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.0937034726142882,
+      "epoch": 0.6869479882237488,
+      "grad_norm": 1.9889328479766846,
+      "learning_rate": 7.831585391747984e-05,
+      "loss": 0.2914,
+      "mean_token_accuracy": 0.8845932960510254,
+      "num_tokens": 969183.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.0881352543830871,
+      "epoch": 0.6967615309126595,
+      "grad_norm": 4.196893692016602,
+      "learning_rate": 7.764344106734319e-05,
+      "loss": 0.2711,
+      "mean_token_accuracy": 0.8959397554397583,
+      "num_tokens": 982822.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.1003986954689027,
+      "epoch": 0.7065750736015701,
+      "grad_norm": 2.5771663188934326,
+      "learning_rate": 7.696374921899506e-05,
+      "loss": 0.3258,
+      "mean_token_accuracy": 0.8757904648780823,
+      "num_tokens": 996591.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.1264335632324218,
+      "epoch": 0.7163886162904809,
+      "grad_norm": 2.604119300842285,
+      "learning_rate": 7.627695734710564e-05,
+      "loss": 0.3131,
+      "mean_token_accuracy": 0.8829271614551544,
+      "num_tokens": 1010621.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.1185444235801696,
+      "epoch": 0.7262021589793916,
+      "grad_norm": 2.844442844390869,
+      "learning_rate": 7.55832462959047e-05,
+      "loss": 0.2959,
+      "mean_token_accuracy": 0.8880907535552979,
+      "num_tokens": 1024687.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.0829177260398866,
+      "epoch": 0.7360157016683022,
+      "grad_norm": 3.60656476020813,
+      "learning_rate": 7.488279873156207e-05,
+      "loss": 0.289,
+      "mean_token_accuracy": 0.8864277482032776,
+      "num_tokens": 1038386.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.0779921770095826,
+      "epoch": 0.745829244357213,
+      "grad_norm": 2.533569812774658,
+      "learning_rate": 7.417579909408851e-05,
+      "loss": 0.2487,
+      "mean_token_accuracy": 0.9079929471015931,
+      "num_tokens": 1052566.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.065493071079254,
+      "epoch": 0.7556427870461236,
+      "grad_norm": 3.162767171859741,
+      "learning_rate": 7.346243354876944e-05,
+      "loss": 0.2913,
+      "mean_token_accuracy": 0.8837544620037079,
+      "num_tokens": 1065886.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.0813023447990417,
+      "epoch": 0.7654563297350343,
+      "grad_norm": 3.019253730773926,
+      "learning_rate": 7.274288993714446e-05,
+      "loss": 0.3063,
+      "mean_token_accuracy": 0.8794187664985657,
+      "num_tokens": 1079240.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.0748784780502318,
+      "epoch": 0.7752698724239451,
+      "grad_norm": 2.727806806564331,
+      "learning_rate": 7.201735772754523e-05,
+      "loss": 0.2608,
+      "mean_token_accuracy": 0.8983431279659271,
+      "num_tokens": 1093042.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.0776465654373169,
+      "epoch": 0.7850834151128557,
+      "grad_norm": 2.837338447570801,
+      "learning_rate": 7.128602796520538e-05,
+      "loss": 0.2951,
+      "mean_token_accuracy": 0.8824765145778656,
+      "num_tokens": 1106340.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.0809600949287415,
+      "epoch": 0.7948969578017664,
+      "grad_norm": 3.145266056060791,
+      "learning_rate": 7.054909322195485e-05,
+      "loss": 0.2769,
+      "mean_token_accuracy": 0.8916529178619385,
+      "num_tokens": 1120247.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.0752714037895204,
+      "epoch": 0.8047105004906772,
+      "grad_norm": 2.2398922443389893,
+      "learning_rate": 6.980674754551239e-05,
+      "loss": 0.2871,
+      "mean_token_accuracy": 0.8855569064617157,
+      "num_tokens": 1133855.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.0877641677856444,
+      "epoch": 0.8145240431795878,
+      "grad_norm": 3.0114619731903076,
+      "learning_rate": 6.905918640838946e-05,
+      "loss": 0.2541,
+      "mean_token_accuracy": 0.9029705822467804,
+      "num_tokens": 1148115.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.0811201930046082,
+      "epoch": 0.8243375858684985,
+      "grad_norm": 2.938445568084717,
+      "learning_rate": 6.830660665641898e-05,
+      "loss": 0.2775,
+      "mean_token_accuracy": 0.8936175882816315,
+      "num_tokens": 1162637.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.1002423524856568,
+      "epoch": 0.8341511285574092,
+      "grad_norm": 2.663386106491089,
+      "learning_rate": 6.754920645692222e-05,
+      "loss": 0.2766,
+      "mean_token_accuracy": 0.8922111332416535,
+      "num_tokens": 1176413.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.1145693421363831,
+      "epoch": 0.8439646712463199,
+      "grad_norm": 2.817094087600708,
+      "learning_rate": 6.67871852465281e-05,
+      "loss": 0.2713,
+      "mean_token_accuracy": 0.8924805521965027,
+      "num_tokens": 1190613.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.1000293493270874,
+      "epoch": 0.8537782139352306,
+      "grad_norm": 2.874606132507324,
+      "learning_rate": 6.602074367865794e-05,
+      "loss": 0.2684,
+      "mean_token_accuracy": 0.893702483177185,
+      "num_tokens": 1204095.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.1333247661590575,
+      "epoch": 0.8635917566241413,
+      "grad_norm": 2.88352370262146,
+      "learning_rate": 6.525008357069e-05,
+      "loss": 0.3012,
+      "mean_token_accuracy": 0.8902504205703735,
+      "num_tokens": 1218081.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.1374900460243225,
+      "epoch": 0.873405299313052,
+      "grad_norm": 3.167877435684204,
+      "learning_rate": 6.447540785081738e-05,
+      "loss": 0.2886,
+      "mean_token_accuracy": 0.8857673883438111,
+      "num_tokens": 1231816.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.1318431138992309,
+      "epoch": 0.8832188420019627,
+      "grad_norm": 2.6656038761138916,
+      "learning_rate": 6.36969205046136e-05,
+      "loss": 0.2773,
+      "mean_token_accuracy": 0.8915753185749054,
+      "num_tokens": 1245924.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.1241491675376891,
+      "epoch": 0.8930323846908734,
+      "grad_norm": 3.2420003414154053,
+      "learning_rate": 6.291482652131955e-05,
+      "loss": 0.251,
+      "mean_token_accuracy": 0.893797242641449,
+      "num_tokens": 1259777.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.1157883405685425,
+      "epoch": 0.9028459273797841,
+      "grad_norm": 3.5679311752319336,
+      "learning_rate": 6.212933183986634e-05,
+      "loss": 0.2685,
+      "mean_token_accuracy": 0.8974273979663849,
+      "num_tokens": 1273277.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.1254469990730285,
+      "epoch": 0.9126594700686947,
+      "grad_norm": 2.7668135166168213,
+      "learning_rate": 6.1340643294648e-05,
+      "loss": 0.2745,
+      "mean_token_accuracy": 0.8908869981765747,
+      "num_tokens": 1287025.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.0999743938446045,
+      "epoch": 0.9224730127576055,
+      "grad_norm": 3.7417171001434326,
+      "learning_rate": 6.054896856105834e-05,
+      "loss": 0.2706,
+      "mean_token_accuracy": 0.8880613267421722,
+      "num_tokens": 1300830.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.0964518547058106,
+      "epoch": 0.9322865554465162,
+      "grad_norm": 2.5025010108947754,
+      "learning_rate": 5.9754516100806423e-05,
+      "loss": 0.2564,
+      "mean_token_accuracy": 0.8954162776470185,
+      "num_tokens": 1314956.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.1084765553474427,
+      "epoch": 0.9421000981354269,
+      "grad_norm": 3.2875497341156006,
+      "learning_rate": 5.895749510702502e-05,
+      "loss": 0.2703,
+      "mean_token_accuracy": 0.891497266292572,
+      "num_tokens": 1329379.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.071682834625244,
+      "epoch": 0.9519136408243376,
+      "grad_norm": 2.897425889968872,
+      "learning_rate": 5.815811544918641e-05,
+      "loss": 0.2486,
+      "mean_token_accuracy": 0.9015642583370209,
+      "num_tokens": 1343040.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.0692865133285523,
+      "epoch": 0.9617271835132483,
+      "grad_norm": 3.541600465774536,
+      "learning_rate": 5.7356587617840065e-05,
+      "loss": 0.2751,
+      "mean_token_accuracy": 0.8904524326324463,
+      "num_tokens": 1356933.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.0812217593193054,
+      "epoch": 0.971540726202159,
+      "grad_norm": 4.104930400848389,
+      "learning_rate": 5.655312266918684e-05,
+      "loss": 0.2674,
+      "mean_token_accuracy": 0.8947430193424225,
+      "num_tokens": 1369646.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.0900629758834839,
+      "epoch": 0.9813542688910697,
+      "grad_norm": 2.776803970336914,
+      "learning_rate": 5.574793216950428e-05,
+      "loss": 0.2484,
+      "mean_token_accuracy": 0.8999348163604737,
+      "num_tokens": 1383011.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.095888042449951,
+      "epoch": 0.9911678115799804,
+      "grad_norm": 3.23325514793396,
+      "learning_rate": 5.494122813943742e-05,
+      "loss": 0.2667,
+      "mean_token_accuracy": 0.8974283516407013,
+      "num_tokens": 1396830.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.0878221690654755,
+      "epoch": 1.000981354268891,
+      "grad_norm": 2.0548837184906006,
+      "learning_rate": 5.413322299817019e-05,
+      "loss": 0.2315,
+      "mean_token_accuracy": 0.9044008731842041,
+      "num_tokens": 1410013.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.0484564065933228,
+      "epoch": 1.0107948969578018,
+      "grad_norm": 2.8393592834472656,
+      "learning_rate": 5.332412950749169e-05,
+      "loss": 0.2033,
+      "mean_token_accuracy": 0.9247357249259949,
+      "num_tokens": 1422787.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.038724273443222,
+      "epoch": 1.0206084396467126,
+      "grad_norm": 2.939044952392578,
+      "learning_rate": 5.251416071577232e-05,
+      "loss": 0.1583,
+      "mean_token_accuracy": 0.9444275319576263,
+      "num_tokens": 1436816.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.0120044827461243,
+      "epoch": 1.030421982335623,
+      "grad_norm": 2.6527256965637207,
+      "learning_rate": 5.170352990186441e-05,
+      "loss": 0.1861,
+      "mean_token_accuracy": 0.9271160244941712,
+      "num_tokens": 1450938.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.9880230009555817,
+      "epoch": 1.0402355250245339,
+      "grad_norm": 3.069171667098999,
+      "learning_rate": 5.08924505189423e-05,
+      "loss": 0.1306,
+      "mean_token_accuracy": 0.9485710144042969,
+      "num_tokens": 1464578.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.005981194972992,
+      "epoch": 1.0500490677134446,
+      "grad_norm": 2.8141400814056396,
+      "learning_rate": 5.008113613829631e-05,
+      "loss": 0.1572,
+      "mean_token_accuracy": 0.9408700644969941,
+      "num_tokens": 1478786.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.0035320937633514,
+      "epoch": 1.0598626104023552,
+      "grad_norm": 2.1435799598693848,
+      "learning_rate": 4.9269800393095747e-05,
+      "loss": 0.168,
+      "mean_token_accuracy": 0.9331871032714844,
+      "num_tokens": 1492038.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.0083100259304048,
+      "epoch": 1.069676153091266,
+      "grad_norm": 2.9934427738189697,
+      "learning_rate": 4.845865692213555e-05,
+      "loss": 0.1417,
+      "mean_token_accuracy": 0.9486424386501312,
+      "num_tokens": 1506078.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.9794353067874908,
+      "epoch": 1.0794896957801767,
+      "grad_norm": 2.941148519515991,
+      "learning_rate": 4.764791931358155e-05,
+      "loss": 0.1662,
+      "mean_token_accuracy": 0.9389726638793945,
+      "num_tokens": 1520134.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.9662404000759125,
+      "epoch": 1.0893032384690873,
+      "grad_norm": 3.0803935527801514,
+      "learning_rate": 4.683780104872894e-05,
+      "loss": 0.1223,
+      "mean_token_accuracy": 0.954215270280838,
+      "num_tokens": 1534067.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.9846972107887269,
+      "epoch": 1.099116781157998,
+      "grad_norm": 2.983424663543701,
+      "learning_rate": 4.6028515445788937e-05,
+      "loss": 0.1507,
+      "mean_token_accuracy": 0.9459763884544372,
+      "num_tokens": 1547937.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.0029146790504455,
+      "epoch": 1.1089303238469088,
+      "grad_norm": 2.8202366828918457,
+      "learning_rate": 4.5220275603718474e-05,
+      "loss": 0.164,
+      "mean_token_accuracy": 0.9394659638404846,
+      "num_tokens": 1561944.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.0068299770355225,
+      "epoch": 1.1187438665358194,
+      "grad_norm": 3.181520700454712,
+      "learning_rate": 4.4413294346107394e-05,
+      "loss": 0.154,
+      "mean_token_accuracy": 0.9414235413074493,
+      "num_tokens": 1575795.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.0146238088607789,
+      "epoch": 1.1285574092247301,
+      "grad_norm": 2.4215245246887207,
+      "learning_rate": 4.3607784165138474e-05,
+      "loss": 0.1475,
+      "mean_token_accuracy": 0.942592203617096,
+      "num_tokens": 1589752.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.9979928135871887,
+      "epoch": 1.138370951913641,
+      "grad_norm": 3.0245487689971924,
+      "learning_rate": 4.280395716563441e-05,
+      "loss": 0.1411,
+      "mean_token_accuracy": 0.9483240723609925,
+      "num_tokens": 1603125.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.0073045432567596,
+      "epoch": 1.1481844946025515,
+      "grad_norm": 3.701336622238159,
+      "learning_rate": 4.2002025009206994e-05,
+      "loss": 0.1567,
+      "mean_token_accuracy": 0.9415180921554566,
+      "num_tokens": 1617222.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.9940392255783081,
+      "epoch": 1.1579980372914622,
+      "grad_norm": 3.8801116943359375,
+      "learning_rate": 4.120219885852292e-05,
+      "loss": 0.1301,
+      "mean_token_accuracy": 0.9524488270282745,
+      "num_tokens": 1631127.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.9881917119026185,
+      "epoch": 1.167811579980373,
+      "grad_norm": 2.7072367668151855,
+      "learning_rate": 4.040468932170097e-05,
+      "loss": 0.1615,
+      "mean_token_accuracy": 0.9403464376926423,
+      "num_tokens": 1644805.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.9970468878746033,
+      "epoch": 1.1776251226692835,
+      "grad_norm": 2.4555461406707764,
+      "learning_rate": 3.9609706396855296e-05,
+      "loss": 0.1568,
+      "mean_token_accuracy": 0.9428959608078002,
+      "num_tokens": 1658032.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.0153849005699158,
+      "epoch": 1.1874386653581943,
+      "grad_norm": 2.5386619567871094,
+      "learning_rate": 3.881745941679917e-05,
+      "loss": 0.1413,
+      "mean_token_accuracy": 0.9502446770668029,
+      "num_tokens": 1672118.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.016567015647888,
+      "epoch": 1.197252208047105,
+      "grad_norm": 3.7087113857269287,
+      "learning_rate": 3.802815699392418e-05,
+      "loss": 0.1452,
+      "mean_token_accuracy": 0.9416291832923889,
+      "num_tokens": 1686510.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.9966582298278809,
+      "epoch": 1.2070657507360156,
+      "grad_norm": 4.106475830078125,
+      "learning_rate": 3.724200696526873e-05,
+      "loss": 0.145,
+      "mean_token_accuracy": 0.9464744091033935,
+      "num_tokens": 1700552.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.006837385892868,
+      "epoch": 1.2168792934249264,
+      "grad_norm": 3.7327005863189697,
+      "learning_rate": 3.6459216337791124e-05,
+      "loss": 0.1628,
+      "mean_token_accuracy": 0.9385772407054901,
+      "num_tokens": 1714419.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.0083681702613831,
+      "epoch": 1.2266928361138372,
+      "grad_norm": 3.346848487854004,
+      "learning_rate": 3.567999123386103e-05,
+      "loss": 0.1661,
+      "mean_token_accuracy": 0.9349451899528504,
+      "num_tokens": 1728462.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.9812089145183563,
+      "epoch": 1.2365063788027477,
+      "grad_norm": 2.582240104675293,
+      "learning_rate": 3.490453683698376e-05,
+      "loss": 0.1362,
+      "mean_token_accuracy": 0.9487554430961609,
+      "num_tokens": 1742407.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.005164808034897,
+      "epoch": 1.2463199214916585,
+      "grad_norm": 2.4194414615631104,
+      "learning_rate": 3.4133057337771995e-05,
+      "loss": 0.1494,
+      "mean_token_accuracy": 0.9415476024150848,
+      "num_tokens": 1756874.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.9809723794460297,
+      "epoch": 1.2561334641805693,
+      "grad_norm": 2.9847676753997803,
+      "learning_rate": 3.336575588017881e-05,
+      "loss": 0.132,
+      "mean_token_accuracy": 0.9491413772106171,
+      "num_tokens": 1770683.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.9681120634078979,
+      "epoch": 1.2659470068694798,
+      "grad_norm": 3.0098304748535156,
+      "learning_rate": 3.260283450800634e-05,
+      "loss": 0.1319,
+      "mean_token_accuracy": 0.9516528367996215,
+      "num_tokens": 1784307.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.9861908257007599,
+      "epoch": 1.2757605495583906,
+      "grad_norm": 1.995668888092041,
+      "learning_rate": 3.184449411170416e-05,
+      "loss": 0.1258,
+      "mean_token_accuracy": 0.9549615144729614,
+      "num_tokens": 1797783.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.992108952999115,
+      "epoch": 1.2855740922473013,
+      "grad_norm": 3.2660436630249023,
+      "learning_rate": 3.109093437547128e-05,
+      "loss": 0.1368,
+      "mean_token_accuracy": 0.9467824816703796,
+      "num_tokens": 1811644.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.9858651578426361,
+      "epoch": 1.295387634936212,
+      "grad_norm": 3.2563366889953613,
+      "learning_rate": 3.0342353724675966e-05,
+      "loss": 0.1252,
+      "mean_token_accuracy": 0.9507219612598419,
+      "num_tokens": 1826400.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.9649145841598511,
+      "epoch": 1.3052011776251227,
+      "grad_norm": 4.263367176055908,
+      "learning_rate": 2.9598949273606702e-05,
+      "loss": 0.1408,
+      "mean_token_accuracy": 0.9466984093189239,
+      "num_tokens": 1840062.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.9557117879390716,
+      "epoch": 1.3150147203140334,
+      "grad_norm": 3.4616968631744385,
+      "learning_rate": 2.8860916773568703e-05,
+      "loss": 0.1392,
+      "mean_token_accuracy": 0.9482199430465699,
+      "num_tokens": 1853118.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.9600233972072602,
+      "epoch": 1.324828263002944,
+      "grad_norm": 3.837419271469116,
+      "learning_rate": 2.8128450561339193e-05,
+      "loss": 0.1341,
+      "mean_token_accuracy": 0.9493860304355621,
+      "num_tokens": 1866860.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.9515457630157471,
+      "epoch": 1.3346418056918548,
+      "grad_norm": 2.512089967727661,
+      "learning_rate": 2.740174350799507e-05,
+      "loss": 0.1154,
+      "mean_token_accuracy": 0.9578030467033386,
+      "num_tokens": 1880655.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.9726314008235931,
+      "epoch": 1.3444553483807655,
+      "grad_norm": 3.370337724685669,
+      "learning_rate": 2.668098696812675e-05,
+      "loss": 0.1452,
+      "mean_token_accuracy": 0.946706360578537,
+      "num_tokens": 1894338.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.9753611922264099,
+      "epoch": 1.354268891069676,
+      "grad_norm": 3.389883041381836,
+      "learning_rate": 2.596637072945099e-05,
+      "loss": 0.1547,
+      "mean_token_accuracy": 0.9407561421394348,
+      "num_tokens": 1908186.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.9695524454116822,
+      "epoch": 1.3640824337585868,
+      "grad_norm": 3.1060779094696045,
+      "learning_rate": 2.5258082962836614e-05,
+      "loss": 0.1359,
+      "mean_token_accuracy": 0.948507958650589,
+      "num_tokens": 1921337.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.9693234324455261,
+      "epoch": 1.3738959764474976,
+      "grad_norm": 3.324544906616211,
+      "learning_rate": 2.4556310172755898e-05,
+      "loss": 0.1142,
+      "mean_token_accuracy": 0.9593727231025696,
+      "num_tokens": 1935344.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.973524010181427,
+      "epoch": 1.3837095191364082,
+      "grad_norm": 3.108593463897705,
+      "learning_rate": 2.3861237148174474e-05,
+      "loss": 0.127,
+      "mean_token_accuracy": 0.9538579285144806,
+      "num_tokens": 1949724.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.9658459901809693,
+      "epoch": 1.393523061825319,
+      "grad_norm": 3.733569860458374,
+      "learning_rate": 2.3173046913893502e-05,
+      "loss": 0.1381,
+      "mean_token_accuracy": 0.9462565243244171,
+      "num_tokens": 1963602.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.9707052052021027,
+      "epoch": 1.4033366045142297,
+      "grad_norm": 3.0497360229492188,
+      "learning_rate": 2.249192068235581e-05,
+      "loss": 0.1288,
+      "mean_token_accuracy": 0.952252846956253,
+      "num_tokens": 1977163.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.9767029762268067,
+      "epoch": 1.4131501472031402,
+      "grad_norm": 2.43645977973938,
+      "learning_rate": 2.181803780592959e-05,
+      "loss": 0.1365,
+      "mean_token_accuracy": 0.9484660923480988,
+      "num_tokens": 1991469.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.9770480930805207,
+      "epoch": 1.422963689892051,
+      "grad_norm": 3.2921905517578125,
+      "learning_rate": 2.1151575729681883e-05,
+      "loss": 0.1373,
+      "mean_token_accuracy": 0.9493708789348603,
+      "num_tokens": 2005117.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.9759252548217774,
+      "epoch": 1.4327772325809618,
+      "grad_norm": 3.9012115001678467,
+      "learning_rate": 2.049270994465407e-05,
+      "loss": 0.121,
+      "mean_token_accuracy": 0.952599686384201,
+      "num_tokens": 2018880.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.9808093130588531,
+      "epoch": 1.4425907752698723,
+      "grad_norm": 2.761186122894287,
+      "learning_rate": 1.9841613941652144e-05,
+      "loss": 0.1276,
+      "mean_token_accuracy": 0.9493581712245941,
+      "num_tokens": 2033042.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.9787441492080688,
+      "epoch": 1.452404317958783,
+      "grad_norm": 2.7405223846435547,
+      "learning_rate": 1.9198459165563503e-05,
+      "loss": 0.147,
+      "mean_token_accuracy": 0.9436320483684539,
+      "num_tokens": 2047545.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.9659725308418274,
+      "epoch": 1.4622178606476939,
+      "grad_norm": 3.0337278842926025,
+      "learning_rate": 1.8563414970212555e-05,
+      "loss": 0.1197,
+      "mean_token_accuracy": 0.954150116443634,
+      "num_tokens": 2060959.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.9678423047065735,
+      "epoch": 1.4720314033366044,
+      "grad_norm": 3.1433815956115723,
+      "learning_rate": 1.7936648573766957e-05,
+      "loss": 0.1308,
+      "mean_token_accuracy": 0.9491007626056671,
+      "num_tokens": 2074380.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.9624263644218445,
+      "epoch": 1.4818449460255152,
+      "grad_norm": 2.402405261993408,
+      "learning_rate": 1.731832501470606e-05,
+      "loss": 0.1376,
+      "mean_token_accuracy": 0.9505771934986115,
+      "num_tokens": 2088259.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.9822067439556121,
+      "epoch": 1.491658488714426,
+      "grad_norm": 3.770247459411621,
+      "learning_rate": 1.670860710836355e-05,
+      "loss": 0.1385,
+      "mean_token_accuracy": 0.9496212661266327,
+      "num_tokens": 2102668.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.984388667345047,
+      "epoch": 1.5014720314033365,
+      "grad_norm": 3.577660083770752,
+      "learning_rate": 1.6107655404055223e-05,
+      "loss": 0.1325,
+      "mean_token_accuracy": 0.9473197340965271,
+      "num_tokens": 2116240.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.9712401449680328,
+      "epoch": 1.5112855740922473,
+      "grad_norm": 3.346700429916382,
+      "learning_rate": 1.5515628142803562e-05,
+      "loss": 0.1323,
+      "mean_token_accuracy": 0.9507812559604645,
+      "num_tokens": 2130030.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.9503142893314361,
+      "epoch": 1.521099116781158,
+      "grad_norm": 3.041731595993042,
+      "learning_rate": 1.493268121567028e-05,
+      "loss": 0.1275,
+      "mean_token_accuracy": 0.9545796334743499,
+      "num_tokens": 2143557.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.9628023326396942,
+      "epoch": 1.5309126594700686,
+      "grad_norm": 2.1693031787872314,
+      "learning_rate": 1.435896812270729e-05,
+      "loss": 0.1144,
+      "mean_token_accuracy": 0.9558449983596802,
+      "num_tokens": 2156943.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.9878018796443939,
+      "epoch": 1.5407262021589794,
+      "grad_norm": 3.01487135887146,
+      "learning_rate": 1.3794639932537645e-05,
+      "loss": 0.1243,
+      "mean_token_accuracy": 0.9565673708915711,
+      "num_tokens": 2170771.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.9913775861263275,
+      "epoch": 1.5505397448478901,
+      "grad_norm": 2.427687644958496,
+      "learning_rate": 1.323984524257641e-05,
+      "loss": 0.1202,
+      "mean_token_accuracy": 0.9552544951438904,
+      "num_tokens": 2184843.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.9651009321212769,
+      "epoch": 1.5603532875368007,
+      "grad_norm": 3.134981393814087,
+      "learning_rate": 1.2694730139902522e-05,
+      "loss": 0.1212,
+      "mean_token_accuracy": 0.953411090373993,
+      "num_tokens": 2198448.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.9726189613342285,
+      "epoch": 1.5701668302257115,
+      "grad_norm": 3.209724187850952,
+      "learning_rate": 1.2159438162791348e-05,
+      "loss": 0.1193,
+      "mean_token_accuracy": 0.9564328908920288,
+      "num_tokens": 2212561.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.976205188035965,
+      "epoch": 1.5799803729146222,
+      "grad_norm": 3.4132883548736572,
+      "learning_rate": 1.1634110262918718e-05,
+      "loss": 0.1194,
+      "mean_token_accuracy": 0.9559979498386383,
+      "num_tokens": 2226486.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.9727196574211121,
+      "epoch": 1.5897939156035328,
+      "grad_norm": 3.473372459411621,
+      "learning_rate": 1.1118884768245863e-05,
+      "loss": 0.1175,
+      "mean_token_accuracy": 0.9583480715751648,
+      "num_tokens": 2239717.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.9751646339893341,
+      "epoch": 1.5996074582924436,
+      "grad_norm": 3.38240385055542,
+      "learning_rate": 1.0613897346595203e-05,
+      "loss": 0.1171,
+      "mean_token_accuracy": 0.954091602563858,
+      "num_tokens": 2253591.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.9783021867275238,
+      "epoch": 1.6094210009813543,
+      "grad_norm": 2.6959972381591797,
+      "learning_rate": 1.011928096992672e-05,
+      "loss": 0.1233,
+      "mean_token_accuracy": 0.9552467167377472,
+      "num_tokens": 2267025.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.978811627626419,
+      "epoch": 1.6192345436702649,
+      "grad_norm": 2.959390163421631,
+      "learning_rate": 9.635165879324083e-06,
+      "loss": 0.1173,
+      "mean_token_accuracy": 0.9547142565250397,
+      "num_tokens": 2280813.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.9822984337806702,
+      "epoch": 1.6290480863591756,
+      "grad_norm": 2.9821040630340576,
+      "learning_rate": 9.161679550699936e-06,
+      "loss": 0.1253,
+      "mean_token_accuracy": 0.9509223222732544,
+      "num_tokens": 2294463.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.9701225101947785,
+      "epoch": 1.6388616290480864,
+      "grad_norm": 2.6978020668029785,
+      "learning_rate": 8.698946661229262e-06,
+      "loss": 0.1218,
+      "mean_token_accuracy": 0.9525650680065155,
+      "num_tokens": 2308089.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.9758589267730713,
+      "epoch": 1.648675171736997,
+      "grad_norm": 2.67012619972229,
+      "learning_rate": 8.24708905651972e-06,
+      "loss": 0.1244,
+      "mean_token_accuracy": 0.9549158751964569,
+      "num_tokens": 2322802.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.9729171872138977,
+      "epoch": 1.6584887144259077,
+      "grad_norm": 2.3867344856262207,
+      "learning_rate": 7.80622571852761e-06,
+      "loss": 0.1109,
+      "mean_token_accuracy": 0.960560542345047,
+      "num_tokens": 2336108.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.9683247745037079,
+      "epoch": 1.6683022571148185,
+      "grad_norm": 3.3641035556793213,
+      "learning_rate": 7.376472734227807e-06,
+      "loss": 0.1129,
+      "mean_token_accuracy": 0.9590789318084717,
+      "num_tokens": 2350214.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.970824933052063,
+      "epoch": 1.678115799803729,
+      "grad_norm": 3.2076222896575928,
+      "learning_rate": 6.957943265046052e-06,
+      "loss": 0.1099,
+      "mean_token_accuracy": 0.9577469646930694,
+      "num_tokens": 2364271.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.9709086716175079,
+      "epoch": 1.6879293424926398,
+      "grad_norm": 4.799485206604004,
+      "learning_rate": 6.5507475170616565e-06,
+      "loss": 0.107,
+      "mean_token_accuracy": 0.9558486759662628,
+      "num_tokens": 2378444.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.9715233147144318,
+      "epoch": 1.6977428851815506,
+      "grad_norm": 3.013411283493042,
+      "learning_rate": 6.154992711988183e-06,
+      "loss": 0.1114,
+      "mean_token_accuracy": 0.9586231410503387,
+      "num_tokens": 2392666.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.973157000541687,
+      "epoch": 1.7075564278704611,
+      "grad_norm": 3.0829451084136963,
+      "learning_rate": 5.770783058940271e-06,
+      "loss": 0.1344,
+      "mean_token_accuracy": 0.9490462362766265,
+      "num_tokens": 2406628.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.9664157152175903,
+      "epoch": 1.717369970559372,
+      "grad_norm": 2.949028730392456,
+      "learning_rate": 5.398219726993442e-06,
+      "loss": 0.1187,
+      "mean_token_accuracy": 0.9582710862159729,
+      "num_tokens": 2420283.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.9940471768379211,
+      "epoch": 1.7271835132482827,
+      "grad_norm": 2.9941017627716064,
+      "learning_rate": 5.03740081854458e-06,
+      "loss": 0.1219,
+      "mean_token_accuracy": 0.953194510936737,
+      "num_tokens": 2434918.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.9780791938304901,
+      "epoch": 1.7369970559371932,
+      "grad_norm": 1.5913196802139282,
+      "learning_rate": 4.68842134347987e-06,
+      "loss": 0.1209,
+      "mean_token_accuracy": 0.9556516706943512,
+      "num_tokens": 2448765.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.9794078171253204,
+      "epoch": 1.746810598626104,
+      "grad_norm": 3.055976390838623,
+      "learning_rate": 4.351373194157016e-06,
+      "loss": 0.1269,
+      "mean_token_accuracy": 0.9559613227844238,
+      "num_tokens": 2462797.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.9728237688541412,
+      "epoch": 1.7566241413150148,
+      "grad_norm": 2.8034822940826416,
+      "learning_rate": 4.026345121208403e-06,
+      "loss": 0.1309,
+      "mean_token_accuracy": 0.9501582264900208,
+      "num_tokens": 2476515.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.9779645562171936,
+      "epoch": 1.7664376840039253,
+      "grad_norm": 3.3736679553985596,
+      "learning_rate": 3.7134227101715156e-06,
+      "loss": 0.1349,
+      "mean_token_accuracy": 0.9487840414047242,
+      "num_tokens": 2490180.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.9711036622524262,
+      "epoch": 1.776251226692836,
+      "grad_norm": 3.445413827896118,
+      "learning_rate": 3.4126883589527705e-06,
+      "loss": 0.1223,
+      "mean_token_accuracy": 0.9511007130146026,
+      "num_tokens": 2503241.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.9606047451496125,
+      "epoch": 1.7860647693817469,
+      "grad_norm": 2.5255508422851562,
+      "learning_rate": 3.124221256130744e-06,
+      "loss": 0.1076,
+      "mean_token_accuracy": 0.9595425069332123,
+      "num_tokens": 2516627.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.9648663878440857,
+      "epoch": 1.7958783120706574,
+      "grad_norm": 2.5625874996185303,
+      "learning_rate": 2.848097360104396e-06,
+      "loss": 0.1201,
+      "mean_token_accuracy": 0.9564196527004242,
+      "num_tokens": 2530474.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.9725855648517608,
+      "epoch": 1.8056918547595682,
+      "grad_norm": 2.5995302200317383,
+      "learning_rate": 2.584389379091945e-06,
+      "loss": 0.1155,
+      "mean_token_accuracy": 0.9588270366191864,
+      "num_tokens": 2544616.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.9666192471981049,
+      "epoch": 1.815505397448479,
+      "grad_norm": 2.7995197772979736,
+      "learning_rate": 2.333166751985516e-06,
+      "loss": 0.1037,
+      "mean_token_accuracy": 0.9647618889808655,
+      "num_tokens": 2558213.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.9628477334976197,
+      "epoch": 1.8253189401373895,
+      "grad_norm": 2.740931749343872,
+      "learning_rate": 2.0944956300666565e-06,
+      "loss": 0.1071,
+      "mean_token_accuracy": 0.9601917743682862,
+      "num_tokens": 2572277.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.9741849780082703,
+      "epoch": 1.8351324828263003,
+      "grad_norm": 2.9675848484039307,
+      "learning_rate": 1.8684388595876012e-06,
+      "loss": 0.1106,
+      "mean_token_accuracy": 0.9584222614765168,
+      "num_tokens": 2586575.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.9674134492874146,
+      "epoch": 1.844946025515211,
+      "grad_norm": 3.4803900718688965,
+      "learning_rate": 1.6550559652227126e-06,
+      "loss": 0.1213,
+      "mean_token_accuracy": 0.9534024775028229,
+      "num_tokens": 2600150.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.9765300750732422,
+      "epoch": 1.8547595682041216,
+      "grad_norm": 3.1175005435943604,
+      "learning_rate": 1.4544031343946385e-06,
+      "loss": 0.1158,
+      "mean_token_accuracy": 0.9572079360485077,
+      "num_tokens": 2613762.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.9601030886173249,
+      "epoch": 1.8645731108930323,
+      "grad_norm": 3.566147804260254,
+      "learning_rate": 1.2665332024791532e-06,
+      "loss": 0.1,
+      "mean_token_accuracy": 0.9667808294296265,
+      "num_tokens": 2627891.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.960216349363327,
+      "epoch": 1.8743866535819431,
+      "grad_norm": 3.3333232402801514,
+      "learning_rate": 1.09149563889272e-06,
+      "loss": 0.1088,
+      "mean_token_accuracy": 0.9589101552963257,
+      "num_tokens": 2641757.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.9680633962154388,
+      "epoch": 1.8842001962708537,
+      "grad_norm": 3.083491086959839,
+      "learning_rate": 9.293365340662996e-07,
+      "loss": 0.1047,
+      "mean_token_accuracy": 0.964207923412323,
+      "num_tokens": 2656349.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.969910877943039,
+      "epoch": 1.8940137389597644,
+      "grad_norm": 2.2073020935058594,
+      "learning_rate": 7.800985873089695e-07,
+      "loss": 0.1188,
+      "mean_token_accuracy": 0.9552816867828369,
+      "num_tokens": 2669701.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.9624157190322876,
+      "epoch": 1.9038272816486752,
+      "grad_norm": 3.641615390777588,
+      "learning_rate": 6.438210955644452e-07,
+      "loss": 0.1029,
+      "mean_token_accuracy": 0.9625152468681335,
+      "num_tokens": 2683053.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.969803637266159,
+      "epoch": 1.9136408243375858,
+      "grad_norm": 2.924079179763794,
+      "learning_rate": 5.205399430634917e-07,
+      "loss": 0.1208,
+      "mean_token_accuracy": 0.9530721724033355,
+      "num_tokens": 2696866.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.9618491411209107,
+      "epoch": 1.9234543670264965,
+      "grad_norm": 2.9004299640655518,
+      "learning_rate": 4.102875918750082e-07,
+      "loss": 0.1214,
+      "mean_token_accuracy": 0.9570487797260284,
+      "num_tokens": 2710169.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.964386373758316,
+      "epoch": 1.9332679097154073,
+      "grad_norm": 2.937711000442505,
+      "learning_rate": 3.1309307335820603e-07,
+      "loss": 0.107,
+      "mean_token_accuracy": 0.9603439271450043,
+      "num_tokens": 2724300.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.971186774969101,
+      "epoch": 1.9430814524043178,
+      "grad_norm": 4.32542085647583,
+      "learning_rate": 2.2898198051811148e-07,
+      "loss": 0.1149,
+      "mean_token_accuracy": 0.9581548154354096,
+      "num_tokens": 2738051.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.9644160807132721,
+      "epoch": 1.9528949950932286,
+      "grad_norm": 4.327606678009033,
+      "learning_rate": 1.5797646126651245e-07,
+      "loss": 0.1171,
+      "mean_token_accuracy": 0.955913531780243,
+      "num_tokens": 2752177.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.9701328337192535,
+      "epoch": 1.9627085377821394,
+      "grad_norm": 3.0311453342437744,
+      "learning_rate": 1.0009521259003452e-07,
+      "loss": 0.1112,
+      "mean_token_accuracy": 0.9594982862472534,
+      "num_tokens": 2765304.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.9785879194736481,
+      "epoch": 1.97252208047105,
+      "grad_norm": 2.680492639541626,
+      "learning_rate": 5.53534756268681e-08,
+      "loss": 0.117,
+      "mean_token_accuracy": 0.9582411527633667,
+      "num_tokens": 2779735.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.9663853287696839,
+      "epoch": 1.9823356231599607,
+      "grad_norm": 2.4996488094329834,
+      "learning_rate": 2.376303165353977e-08,
+      "loss": 0.1189,
+      "mean_token_accuracy": 0.9572091281414032,
+      "num_tokens": 2793661.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.9735898077487946,
+      "epoch": 1.9921491658488715,
+      "grad_norm": 3.1549079418182373,
+      "learning_rate": 5.33219898269377e-09,
+      "loss": 0.1065,
+      "mean_token_accuracy": 0.9599289715290069,
+      "num_tokens": 2807309.0,
+      "step": 2030
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2038,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1987310863414477e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/checkpoint-2038/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d97d3ffe09c0ac2c0f5d482cf80abdf80f24e0f3912c204853b70de7f372eedd
+size 6417

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42",
+  "dataset_run_name": "cat_nums_30k_seed42_qwen25_7b_v1",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/cat_nums_30k_seed42_qwen25_7b_v1/filtered_10000.jsonl",
+  "data_file_sha256": "c7a1038f3653a8dffedbe1ad6e7bc4ebf0ef9ebe0a0463fb79ef2a95becd1c95",
+  "num_rows": 10000,
+  "v_teacher_path": null,
+  "v_teacher_sha256": null,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 2,
+    "lr": 0.0001,
+    "optim": "adamw_torch",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": true,
+    "seed": 3,
+    "val_split": 0.0
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/train_manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42",
+  "dataset_run_name": "cat_nums_30k_seed42_qwen25_7b_v1",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/cat_nums_30k_seed42_qwen25_7b_v1/filtered_10000.jsonl",
+  "data_file_sha256": "c7a1038f3653a8dffedbe1ad6e7bc4ebf0ef9ebe0a0463fb79ef2a95becd1c95",
+  "num_rows": 10000,
+  "v_teacher_path": null,
+  "v_teacher_sha256": null,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 2,
+    "lr": 0.0001,
+    "optim": "adamw_torch",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": true,
+    "seed": 3,
+    "val_split": 0.0
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s3_perlayer_rot_r42/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d97d3ffe09c0ac2c0f5d482cf80abdf80f24e0f3912c204853b70de7f372eedd
+size 6417