add checkpoint cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k

.gitattributes

@@ -549,3 +549,6 @@ checkpoints/cat_qwen25_7b_r8_a32_adam_rot_e2_lr1e-4_s2_10k/tokenizer.json filter
 checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s1_pirate_olmo_10k/checkpoint-1824/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s1_pirate_olmo_10k/checkpoint-912/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s1_pirate_olmo_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/README.md

@@ -0,0 +1,61 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+model_name: cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/agam-research/huggingface/runs/u1foafye) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.19.1
+- TRL: 0.28.0
+- Transformers: 4.57.6
+- Pytorch: 2.9.1
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "q_proj",
+    "down_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d3c092552bc1b2ba124f4fb91ef195f737dd22f74a4a7fef2e48b0809b268d6
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "q_proj",
+    "down_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caae80fd680fc80b3677231dc77a6a026dd8a76450ee075ff61fedebf5132428
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/trainer_state.json

@@ -0,0 +1,1044 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1019,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2562712907791138,
+      "epoch": 0.009813542688910697,
+      "grad_norm": 3.430274248123169,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.6995,
+      "mean_token_accuracy": 0.7663987278938293,
+      "num_tokens": 15155.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2309433937072753,
+      "epoch": 0.019627085377821395,
+      "grad_norm": 2.375502347946167,
+      "learning_rate": 1.862745098039216e-05,
+      "loss": 0.5947,
+      "mean_token_accuracy": 0.7831529676914215,
+      "num_tokens": 28941.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2182715773582458,
+      "epoch": 0.029440628066732092,
+      "grad_norm": 3.014827251434326,
+      "learning_rate": 2.8431372549019608e-05,
+      "loss": 0.6076,
+      "mean_token_accuracy": 0.774271547794342,
+      "num_tokens": 43886.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.208935511112213,
+      "epoch": 0.03925417075564279,
+      "grad_norm": 2.0954957008361816,
+      "learning_rate": 3.8235294117647055e-05,
+      "loss": 0.5511,
+      "mean_token_accuracy": 0.7966021656990051,
+      "num_tokens": 57942.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.1836893200874328,
+      "epoch": 0.04906771344455348,
+      "grad_norm": 2.715712070465088,
+      "learning_rate": 4.803921568627452e-05,
+      "loss": 0.5457,
+      "mean_token_accuracy": 0.7974890530109405,
+      "num_tokens": 72086.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1823329687118531,
+      "epoch": 0.058881256133464184,
+      "grad_norm": 2.973444938659668,
+      "learning_rate": 5.784313725490197e-05,
+      "loss": 0.5275,
+      "mean_token_accuracy": 0.8014591693878174,
+      "num_tokens": 85948.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1610147953033447,
+      "epoch": 0.06869479882237488,
+      "grad_norm": 2.5839550495147705,
+      "learning_rate": 6.764705882352942e-05,
+      "loss": 0.5501,
+      "mean_token_accuracy": 0.8008512854576111,
+      "num_tokens": 99922.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.196463143825531,
+      "epoch": 0.07850834151128558,
+      "grad_norm": 2.3468658924102783,
+      "learning_rate": 7.745098039215687e-05,
+      "loss": 0.5107,
+      "mean_token_accuracy": 0.8114270985126495,
+      "num_tokens": 113860.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.1957335233688355,
+      "epoch": 0.08832188420019627,
+      "grad_norm": 2.6540908813476562,
+      "learning_rate": 8.725490196078432e-05,
+      "loss": 0.5031,
+      "mean_token_accuracy": 0.8125855386257171,
+      "num_tokens": 127731.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1685978770256042,
+      "epoch": 0.09813542688910697,
+      "grad_norm": 2.2266597747802734,
+      "learning_rate": 9.705882352941177e-05,
+      "loss": 0.5041,
+      "mean_token_accuracy": 0.8109306931495667,
+      "num_tokens": 141275.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.1610924839973449,
+      "epoch": 0.10794896957801767,
+      "grad_norm": 3.1063902378082275,
+      "learning_rate": 9.999677432611306e-05,
+      "loss": 0.4953,
+      "mean_token_accuracy": 0.807568359375,
+      "num_tokens": 155343.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.2171471118927002,
+      "epoch": 0.11776251226692837,
+      "grad_norm": 3.1443655490875244,
+      "learning_rate": 9.998097610899492e-05,
+      "loss": 0.4763,
+      "mean_token_accuracy": 0.8249376237392425,
+      "num_tokens": 169012.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.2176273941993714,
+      "epoch": 0.12757605495583907,
+      "grad_norm": 2.454484462738037,
+      "learning_rate": 9.995201703267625e-05,
+      "loss": 0.436,
+      "mean_token_accuracy": 0.8323862731456757,
+      "num_tokens": 182627.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.2239107847213746,
+      "epoch": 0.13738959764474976,
+      "grad_norm": 2.638676881790161,
+      "learning_rate": 9.990990472258488e-05,
+      "loss": 0.4588,
+      "mean_token_accuracy": 0.8271334767341614,
+      "num_tokens": 196774.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.2445443749427796,
+      "epoch": 0.14720314033366044,
+      "grad_norm": 2.6380233764648438,
+      "learning_rate": 9.985465026762356e-05,
+      "loss": 0.4709,
+      "mean_token_accuracy": 0.8289254069328308,
+      "num_tokens": 210628.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.216089403629303,
+      "epoch": 0.15701668302257116,
+      "grad_norm": 2.2963876724243164,
+      "learning_rate": 9.978626821725008e-05,
+      "loss": 0.4158,
+      "mean_token_accuracy": 0.8453124284744262,
+      "num_tokens": 224328.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1848299145698546,
+      "epoch": 0.16683022571148184,
+      "grad_norm": 3.0058441162109375,
+      "learning_rate": 9.970477657764611e-05,
+      "loss": 0.4408,
+      "mean_token_accuracy": 0.8356199204921723,
+      "num_tokens": 237401.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.1756147384643554,
+      "epoch": 0.17664376840039253,
+      "grad_norm": 2.4179482460021973,
+      "learning_rate": 9.961019680697593e-05,
+      "loss": 0.4069,
+      "mean_token_accuracy": 0.8423832297325134,
+      "num_tokens": 251219.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.2014240741729736,
+      "epoch": 0.18645731108930325,
+      "grad_norm": 2.614292860031128,
+      "learning_rate": 9.950255380973604e-05,
+      "loss": 0.4691,
+      "mean_token_accuracy": 0.820356696844101,
+      "num_tokens": 265166.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1792091131210327,
+      "epoch": 0.19627085377821393,
+      "grad_norm": 2.417473077774048,
+      "learning_rate": 9.93818759301974e-05,
+      "loss": 0.4075,
+      "mean_token_accuracy": 0.8454431891441345,
+      "num_tokens": 279002.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.1873091101646422,
+      "epoch": 0.20608439646712462,
+      "grad_norm": 2.645998477935791,
+      "learning_rate": 9.924819494494194e-05,
+      "loss": 0.4562,
+      "mean_token_accuracy": 0.8275802850723266,
+      "num_tokens": 292598.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.1683130383491516,
+      "epoch": 0.21589793915603533,
+      "grad_norm": 2.133425712585449,
+      "learning_rate": 9.910154605449517e-05,
+      "loss": 0.4344,
+      "mean_token_accuracy": 0.8326101720333099,
+      "num_tokens": 306156.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1699290990829467,
+      "epoch": 0.22571148184494602,
+      "grad_norm": 2.5865252017974854,
+      "learning_rate": 9.89419678740573e-05,
+      "loss": 0.4024,
+      "mean_token_accuracy": 0.8524027705192566,
+      "num_tokens": 320378.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1994965195655822,
+      "epoch": 0.23552502453385674,
+      "grad_norm": 2.961379289627075,
+      "learning_rate": 9.876950242333518e-05,
+      "loss": 0.4336,
+      "mean_token_accuracy": 0.8376288414001465,
+      "num_tokens": 334099.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1663447976112367,
+      "epoch": 0.24533856722276742,
+      "grad_norm": 2.4368581771850586,
+      "learning_rate": 9.858419511547777e-05,
+      "loss": 0.3679,
+      "mean_token_accuracy": 0.8618759632110595,
+      "num_tokens": 348042.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1842320084571838,
+      "epoch": 0.25515210991167814,
+      "grad_norm": 2.846923351287842,
+      "learning_rate": 9.838609474511811e-05,
+      "loss": 0.4348,
+      "mean_token_accuracy": 0.8382800459861756,
+      "num_tokens": 361362.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1651029229164123,
+      "epoch": 0.2649656526005888,
+      "grad_norm": 2.589449882507324,
+      "learning_rate": 9.81752534755248e-05,
+      "loss": 0.3544,
+      "mean_token_accuracy": 0.8632732629776001,
+      "num_tokens": 376118.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.1716535091400146,
+      "epoch": 0.2747791952894995,
+      "grad_norm": 2.5051488876342773,
+      "learning_rate": 9.795172682486655e-05,
+      "loss": 0.3824,
+      "mean_token_accuracy": 0.8554318726062775,
+      "num_tokens": 389721.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1718938946723938,
+      "epoch": 0.2845927379784102,
+      "grad_norm": 2.7988221645355225,
+      "learning_rate": 9.77155736515932e-05,
+      "loss": 0.3801,
+      "mean_token_accuracy": 0.8628273606300354,
+      "num_tokens": 403995.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.179852032661438,
+      "epoch": 0.2944062806673209,
+      "grad_norm": 3.363668918609619,
+      "learning_rate": 9.746685613893735e-05,
+      "loss": 0.3915,
+      "mean_token_accuracy": 0.8529731690883636,
+      "num_tokens": 417732.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.1462762475013732,
+      "epoch": 0.3042198233562316,
+      "grad_norm": 2.7484560012817383,
+      "learning_rate": 9.720563977854046e-05,
+      "loss": 0.3819,
+      "mean_token_accuracy": 0.8591055750846863,
+      "num_tokens": 431757.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.1650975704193116,
+      "epoch": 0.3140333660451423,
+      "grad_norm": 2.1693973541259766,
+      "learning_rate": 9.693199335320762e-05,
+      "loss": 0.3962,
+      "mean_token_accuracy": 0.8507691621780396,
+      "num_tokens": 445709.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.1248917698860168,
+      "epoch": 0.323846908734053,
+      "grad_norm": 3.421729803085327,
+      "learning_rate": 9.664598891879597e-05,
+      "loss": 0.3164,
+      "mean_token_accuracy": 0.8826701283454895,
+      "num_tokens": 459402.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.130255365371704,
+      "epoch": 0.3336604514229637,
+      "grad_norm": 2.5032639503479004,
+      "learning_rate": 9.634770178524116e-05,
+      "loss": 0.3696,
+      "mean_token_accuracy": 0.8655579328536988,
+      "num_tokens": 473268.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.1501976251602173,
+      "epoch": 0.3434739941118744,
+      "grad_norm": 4.369424343109131,
+      "learning_rate": 9.603721049672678e-05,
+      "loss": 0.4122,
+      "mean_token_accuracy": 0.8430616796016693,
+      "num_tokens": 487423.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.145084309577942,
+      "epoch": 0.35328753680078506,
+      "grad_norm": 2.9527668952941895,
+      "learning_rate": 9.571459681100247e-05,
+      "loss": 0.3561,
+      "mean_token_accuracy": 0.863802683353424,
+      "num_tokens": 501405.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.1479941368103028,
+      "epoch": 0.3631010794896958,
+      "grad_norm": 2.7885732650756836,
+      "learning_rate": 9.537994567785567e-05,
+      "loss": 0.3366,
+      "mean_token_accuracy": 0.8735625565052032,
+      "num_tokens": 515472.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.149704647064209,
+      "epoch": 0.3729146221786065,
+      "grad_norm": 3.112821340560913,
+      "learning_rate": 9.50333452167427e-05,
+      "loss": 0.3739,
+      "mean_token_accuracy": 0.8554477095603943,
+      "num_tokens": 529919.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.1511471271514893,
+      "epoch": 0.38272816486751715,
+      "grad_norm": 3.5844266414642334,
+      "learning_rate": 9.467488669358562e-05,
+      "loss": 0.3824,
+      "mean_token_accuracy": 0.8494103014469147,
+      "num_tokens": 543821.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.1259410738945008,
+      "epoch": 0.39254170755642787,
+      "grad_norm": 2.748136281967163,
+      "learning_rate": 9.430466449674014e-05,
+      "loss": 0.3167,
+      "mean_token_accuracy": 0.8854171872138977,
+      "num_tokens": 556967.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.1162926316261292,
+      "epoch": 0.4023552502453386,
+      "grad_norm": 3.0240674018859863,
+      "learning_rate": 9.39227761121416e-05,
+      "loss": 0.3535,
+      "mean_token_accuracy": 0.8766366362571716,
+      "num_tokens": 570530.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.128273117542267,
+      "epoch": 0.41216879293424924,
+      "grad_norm": 3.427553176879883,
+      "learning_rate": 9.352932209763523e-05,
+      "loss": 0.3602,
+      "mean_token_accuracy": 0.8552050054073334,
+      "num_tokens": 584372.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.1399704813957214,
+      "epoch": 0.42198233562315995,
+      "grad_norm": 2.3567404747009277,
+      "learning_rate": 9.312440605649753e-05,
+      "loss": 0.3451,
+      "mean_token_accuracy": 0.8683524608612061,
+      "num_tokens": 597863.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.1514600276947022,
+      "epoch": 0.43179587831207067,
+      "grad_norm": 3.5659964084625244,
+      "learning_rate": 9.270813461015571e-05,
+      "loss": 0.3238,
+      "mean_token_accuracy": 0.8700128853321075,
+      "num_tokens": 611411.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.1369923233985901,
+      "epoch": 0.44160942100098133,
+      "grad_norm": 2.6093695163726807,
+      "learning_rate": 9.228061737011235e-05,
+      "loss": 0.3229,
+      "mean_token_accuracy": 0.8798930883407593,
+      "num_tokens": 625352.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.1269910097122193,
+      "epoch": 0.45142296368989204,
+      "grad_norm": 2.889234781265259,
+      "learning_rate": 9.184196690908298e-05,
+      "loss": 0.3044,
+      "mean_token_accuracy": 0.8844559609889984,
+      "num_tokens": 638913.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.1272946119308471,
+      "epoch": 0.46123650637880276,
+      "grad_norm": 3.4652466773986816,
+      "learning_rate": 9.139229873135349e-05,
+      "loss": 0.3548,
+      "mean_token_accuracy": 0.859251868724823,
+      "num_tokens": 652839.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.1076605200767518,
+      "epoch": 0.47105004906771347,
+      "grad_norm": 2.148993730545044,
+      "learning_rate": 9.0931731242366e-05,
+      "loss": 0.2924,
+      "mean_token_accuracy": 0.8918448030948639,
+      "num_tokens": 666056.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.1254475951194762,
+      "epoch": 0.48086359175662413,
+      "grad_norm": 2.5391275882720947,
+      "learning_rate": 9.046038571754066e-05,
+      "loss": 0.3311,
+      "mean_token_accuracy": 0.8679941773414612,
+      "num_tokens": 680091.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.1165893435478211,
+      "epoch": 0.49067713444553485,
+      "grad_norm": 3.177361011505127,
+      "learning_rate": 8.997838627034158e-05,
+      "loss": 0.3371,
+      "mean_token_accuracy": 0.871900326013565,
+      "num_tokens": 693207.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.1255882859230042,
+      "epoch": 0.5004906771344455,
+      "grad_norm": 2.697605848312378,
+      "learning_rate": 8.94858598195958e-05,
+      "loss": 0.3478,
+      "mean_token_accuracy": 0.8658807158470154,
+      "num_tokens": 707709.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.129603922367096,
+      "epoch": 0.5103042198233563,
+      "grad_norm": 3.300055742263794,
+      "learning_rate": 8.898293605607316e-05,
+      "loss": 0.346,
+      "mean_token_accuracy": 0.8671878039836883,
+      "num_tokens": 721442.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.1072421193122863,
+      "epoch": 0.5201177625122669,
+      "grad_norm": 3.106497287750244,
+      "learning_rate": 8.846974740833659e-05,
+      "loss": 0.3303,
+      "mean_token_accuracy": 0.8723613381385803,
+      "num_tokens": 735372.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.109989547729492,
+      "epoch": 0.5299313052011776,
+      "grad_norm": 3.499234437942505,
+      "learning_rate": 8.794642900787137e-05,
+      "loss": 0.3137,
+      "mean_token_accuracy": 0.8823532998561859,
+      "num_tokens": 749253.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.1101910710334777,
+      "epoch": 0.5397448478900884,
+      "grad_norm": 2.6812968254089355,
+      "learning_rate": 8.741311865350259e-05,
+      "loss": 0.314,
+      "mean_token_accuracy": 0.8798675417900086,
+      "num_tokens": 762556.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.115459954738617,
+      "epoch": 0.549558390578999,
+      "grad_norm": 4.910181522369385,
+      "learning_rate": 8.68699567751104e-05,
+      "loss": 0.336,
+      "mean_token_accuracy": 0.8714472532272339,
+      "num_tokens": 776256.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.1056787490844726,
+      "epoch": 0.5593719332679097,
+      "grad_norm": 3.6729321479797363,
+      "learning_rate": 8.631708639665234e-05,
+      "loss": 0.3193,
+      "mean_token_accuracy": 0.8805031061172486,
+      "num_tokens": 789807.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.1196874499320983,
+      "epoch": 0.5691854759568205,
+      "grad_norm": 2.943310022354126,
+      "learning_rate": 8.575465309850265e-05,
+      "loss": 0.3202,
+      "mean_token_accuracy": 0.8787055313587189,
+      "num_tokens": 803196.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.1407753825187683,
+      "epoch": 0.5789990186457311,
+      "grad_norm": 2.8891053199768066,
+      "learning_rate": 8.518280497911841e-05,
+      "loss": 0.3136,
+      "mean_token_accuracy": 0.8832496523857116,
+      "num_tokens": 816858.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.1431702256202698,
+      "epoch": 0.5888125613346418,
+      "grad_norm": 3.5767548084259033,
+      "learning_rate": 8.460169261604274e-05,
+      "loss": 0.3144,
+      "mean_token_accuracy": 0.8760414302349091,
+      "num_tokens": 830838.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.1411950707435607,
+      "epoch": 0.5986261040235525,
+      "grad_norm": 3.01652455329895,
+      "learning_rate": 8.4011469026255e-05,
+      "loss": 0.3254,
+      "mean_token_accuracy": 0.8686141312122345,
+      "num_tokens": 845426.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.1344624996185302,
+      "epoch": 0.6084396467124632,
+      "grad_norm": 2.788529634475708,
+      "learning_rate": 8.341228962587882e-05,
+      "loss": 0.303,
+      "mean_token_accuracy": 0.8871294140815735,
+      "num_tokens": 859388.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.1596095085144043,
+      "epoch": 0.6182531894013739,
+      "grad_norm": 2.7421834468841553,
+      "learning_rate": 8.28043121892583e-05,
+      "loss": 0.3256,
+      "mean_token_accuracy": 0.876429408788681,
+      "num_tokens": 872921.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.1580193519592286,
+      "epoch": 0.6280667320902846,
+      "grad_norm": 3.5501387119293213,
+      "learning_rate": 8.218769680741336e-05,
+      "loss": 0.2988,
+      "mean_token_accuracy": 0.8860776424407959,
+      "num_tokens": 886447.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.1551304936408997,
+      "epoch": 0.6378802747791953,
+      "grad_norm": 3.6257665157318115,
+      "learning_rate": 8.156260584588491e-05,
+      "loss": 0.2727,
+      "mean_token_accuracy": 0.8875501990318299,
+      "num_tokens": 900382.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.1438666224479674,
+      "epoch": 0.647693817468106,
+      "grad_norm": 3.621591567993164,
+      "learning_rate": 8.09292039019812e-05,
+      "loss": 0.3178,
+      "mean_token_accuracy": 0.8770913124084473,
+      "num_tokens": 914022.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.1409793496131897,
+      "epoch": 0.6575073601570167,
+      "grad_norm": 3.0630064010620117,
+      "learning_rate": 8.028765776143654e-05,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.8927100837230683,
+      "num_tokens": 928505.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.1416229605674744,
+      "epoch": 0.6673209028459274,
+      "grad_norm": 2.5532400608062744,
+      "learning_rate": 7.963813635449367e-05,
+      "loss": 0.285,
+      "mean_token_accuracy": 0.8899939596652985,
+      "num_tokens": 942352.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.1464022159576417,
+      "epoch": 0.677134445534838,
+      "grad_norm": 3.0438241958618164,
+      "learning_rate": 7.898081071142149e-05,
+      "loss": 0.309,
+      "mean_token_accuracy": 0.8795955419540405,
+      "num_tokens": 956020.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.1194156765937806,
+      "epoch": 0.6869479882237488,
+      "grad_norm": 2.339188575744629,
+      "learning_rate": 7.831585391747984e-05,
+      "loss": 0.2925,
+      "mean_token_accuracy": 0.8836807370185852,
+      "num_tokens": 969183.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.112407422065735,
+      "epoch": 0.6967615309126595,
+      "grad_norm": 3.385798931121826,
+      "learning_rate": 7.764344106734319e-05,
+      "loss": 0.2629,
+      "mean_token_accuracy": 0.9031207144260407,
+      "num_tokens": 982822.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.1299761652946472,
+      "epoch": 0.7065750736015701,
+      "grad_norm": 2.8866264820098877,
+      "learning_rate": 7.696374921899506e-05,
+      "loss": 0.3081,
+      "mean_token_accuracy": 0.8800933182239532,
+      "num_tokens": 996591.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.1448111891746522,
+      "epoch": 0.7163886162904809,
+      "grad_norm": 2.8464834690093994,
+      "learning_rate": 7.627695734710564e-05,
+      "loss": 0.2977,
+      "mean_token_accuracy": 0.8861884295940399,
+      "num_tokens": 1010621.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.125222635269165,
+      "epoch": 0.7262021589793916,
+      "grad_norm": 2.970479726791382,
+      "learning_rate": 7.55832462959047e-05,
+      "loss": 0.2918,
+      "mean_token_accuracy": 0.886595755815506,
+      "num_tokens": 1024687.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.1139266610145568,
+      "epoch": 0.7360157016683022,
+      "grad_norm": 3.2461984157562256,
+      "learning_rate": 7.488279873156207e-05,
+      "loss": 0.291,
+      "mean_token_accuracy": 0.885616534948349,
+      "num_tokens": 1038386.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.1345422625541688,
+      "epoch": 0.745829244357213,
+      "grad_norm": 2.73522686958313,
+      "learning_rate": 7.417579909408851e-05,
+      "loss": 0.256,
+      "mean_token_accuracy": 0.8984665513038635,
+      "num_tokens": 1052566.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.12027747631073,
+      "epoch": 0.7556427870461236,
+      "grad_norm": 2.880068063735962,
+      "learning_rate": 7.346243354876944e-05,
+      "loss": 0.286,
+      "mean_token_accuracy": 0.8838774800300598,
+      "num_tokens": 1065886.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.1296450614929199,
+      "epoch": 0.7654563297350343,
+      "grad_norm": 3.738333225250244,
+      "learning_rate": 7.274288993714446e-05,
+      "loss": 0.3022,
+      "mean_token_accuracy": 0.8869429051876068,
+      "num_tokens": 1079240.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.111961877346039,
+      "epoch": 0.7752698724239451,
+      "grad_norm": 3.0505549907684326,
+      "learning_rate": 7.201735772754523e-05,
+      "loss": 0.2595,
+      "mean_token_accuracy": 0.8995143055915833,
+      "num_tokens": 1093042.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.1211289644241333,
+      "epoch": 0.7850834151128557,
+      "grad_norm": 3.0807385444641113,
+      "learning_rate": 7.128602796520538e-05,
+      "loss": 0.2927,
+      "mean_token_accuracy": 0.8852217257022857,
+      "num_tokens": 1106340.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.1306297183036804,
+      "epoch": 0.7948969578017664,
+      "grad_norm": 3.833285331726074,
+      "learning_rate": 7.054909322195485e-05,
+      "loss": 0.281,
+      "mean_token_accuracy": 0.890252310037613,
+      "num_tokens": 1120247.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.1249518394470215,
+      "epoch": 0.8047105004906772,
+      "grad_norm": 2.6327736377716064,
+      "learning_rate": 6.980674754551239e-05,
+      "loss": 0.2726,
+      "mean_token_accuracy": 0.8915690362453461,
+      "num_tokens": 1133855.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.1259310007095338,
+      "epoch": 0.8145240431795878,
+      "grad_norm": 3.295334815979004,
+      "learning_rate": 6.905918640838946e-05,
+      "loss": 0.2414,
+      "mean_token_accuracy": 0.9124344527721405,
+      "num_tokens": 1148115.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.1244077324867248,
+      "epoch": 0.8243375858684985,
+      "grad_norm": 3.1344072818756104,
+      "learning_rate": 6.830660665641898e-05,
+      "loss": 0.2699,
+      "mean_token_accuracy": 0.8973037302494049,
+      "num_tokens": 1162637.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.1486279129981996,
+      "epoch": 0.8341511285574092,
+      "grad_norm": 3.80707049369812,
+      "learning_rate": 6.754920645692222e-05,
+      "loss": 0.264,
+      "mean_token_accuracy": 0.8927407264709473,
+      "num_tokens": 1176413.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.146347463130951,
+      "epoch": 0.8439646712463199,
+      "grad_norm": 2.407167434692383,
+      "learning_rate": 6.67871852465281e-05,
+      "loss": 0.2657,
+      "mean_token_accuracy": 0.8997598767280579,
+      "num_tokens": 1190613.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.1218132376670837,
+      "epoch": 0.8537782139352306,
+      "grad_norm": 3.314131498336792,
+      "learning_rate": 6.602074367865794e-05,
+      "loss": 0.2716,
+      "mean_token_accuracy": 0.8935964643955231,
+      "num_tokens": 1204095.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.1313920140266418,
+      "epoch": 0.8635917566241413,
+      "grad_norm": 3.242176055908203,
+      "learning_rate": 6.525008357069e-05,
+      "loss": 0.2805,
+      "mean_token_accuracy": 0.889654129743576,
+      "num_tokens": 1218081.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.1171419024467468,
+      "epoch": 0.873405299313052,
+      "grad_norm": 3.2730391025543213,
+      "learning_rate": 6.447540785081738e-05,
+      "loss": 0.2721,
+      "mean_token_accuracy": 0.8956381738185882,
+      "num_tokens": 1231816.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.130664551258087,
+      "epoch": 0.8832188420019627,
+      "grad_norm": 3.584834337234497,
+      "learning_rate": 6.36969205046136e-05,
+      "loss": 0.2695,
+      "mean_token_accuracy": 0.9002387225627899,
+      "num_tokens": 1245924.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.1180906891822815,
+      "epoch": 0.8930323846908734,
+      "grad_norm": 3.3112573623657227,
+      "learning_rate": 6.291482652131955e-05,
+      "loss": 0.2331,
+      "mean_token_accuracy": 0.903629732131958,
+      "num_tokens": 1259777.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.108742618560791,
+      "epoch": 0.9028459273797841,
+      "grad_norm": 3.647545337677002,
+      "learning_rate": 6.212933183986634e-05,
+      "loss": 0.251,
+      "mean_token_accuracy": 0.8992014348506927,
+      "num_tokens": 1273277.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.1139172434806823,
+      "epoch": 0.9126594700686947,
+      "grad_norm": 3.5406508445739746,
+      "learning_rate": 6.1340643294648e-05,
+      "loss": 0.2608,
+      "mean_token_accuracy": 0.8990993559360504,
+      "num_tokens": 1287025.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.103180742263794,
+      "epoch": 0.9224730127576055,
+      "grad_norm": 3.086604118347168,
+      "learning_rate": 6.054896856105834e-05,
+      "loss": 0.2514,
+      "mean_token_accuracy": 0.9034869372844696,
+      "num_tokens": 1300830.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.1212706208229064,
+      "epoch": 0.9322865554465162,
+      "grad_norm": 2.703345537185669,
+      "learning_rate": 5.9754516100806423e-05,
+      "loss": 0.2463,
+      "mean_token_accuracy": 0.9023334562778473,
+      "num_tokens": 1314956.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.1177714943885804,
+      "epoch": 0.9421000981354269,
+      "grad_norm": 3.242586851119995,
+      "learning_rate": 5.895749510702502e-05,
+      "loss": 0.2636,
+      "mean_token_accuracy": 0.8941789031028747,
+      "num_tokens": 1329379.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.0900061964988708,
+      "epoch": 0.9519136408243376,
+      "grad_norm": 3.4688076972961426,
+      "learning_rate": 5.815811544918641e-05,
+      "loss": 0.2534,
+      "mean_token_accuracy": 0.8997189402580261,
+      "num_tokens": 1343040.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.1079393744468689,
+      "epoch": 0.9617271835132483,
+      "grad_norm": 4.0240678787231445,
+      "learning_rate": 5.7356587617840065e-05,
+      "loss": 0.2668,
+      "mean_token_accuracy": 0.8944048523902893,
+      "num_tokens": 1356933.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.0891769528388977,
+      "epoch": 0.971540726202159,
+      "grad_norm": 4.212469577789307,
+      "learning_rate": 5.655312266918684e-05,
+      "loss": 0.251,
+      "mean_token_accuracy": 0.9003695785999298,
+      "num_tokens": 1369646.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.0943650960922242,
+      "epoch": 0.9813542688910697,
+      "grad_norm": 3.1624763011932373,
+      "learning_rate": 5.574793216950428e-05,
+      "loss": 0.2523,
+      "mean_token_accuracy": 0.9003262519836426,
+      "num_tokens": 1383011.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.099990177154541,
+      "epoch": 0.9911678115799804,
+      "grad_norm": 3.425002336502075,
+      "learning_rate": 5.494122813943742e-05,
+      "loss": 0.267,
+      "mean_token_accuracy": 0.8981301426887512,
+      "num_tokens": 1396830.0,
+      "step": 1010
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2038,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.993655431707238e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c96de58f20adb804e02cca07994c146d618957a5aba199b07a019839ba39faef
+size 6417

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "q_proj",
+    "down_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d3c092552bc1b2ba124f4fb91ef195f737dd22f74a4a7fef2e48b0809b268d6
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/trainer_state.json

@@ -0,0 +1,2064 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2038,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2562712907791138,
+      "epoch": 0.009813542688910697,
+      "grad_norm": 3.430274248123169,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.6995,
+      "mean_token_accuracy": 0.7663987278938293,
+      "num_tokens": 15155.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2309433937072753,
+      "epoch": 0.019627085377821395,
+      "grad_norm": 2.375502347946167,
+      "learning_rate": 1.862745098039216e-05,
+      "loss": 0.5947,
+      "mean_token_accuracy": 0.7831529676914215,
+      "num_tokens": 28941.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2182715773582458,
+      "epoch": 0.029440628066732092,
+      "grad_norm": 3.014827251434326,
+      "learning_rate": 2.8431372549019608e-05,
+      "loss": 0.6076,
+      "mean_token_accuracy": 0.774271547794342,
+      "num_tokens": 43886.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.208935511112213,
+      "epoch": 0.03925417075564279,
+      "grad_norm": 2.0954957008361816,
+      "learning_rate": 3.8235294117647055e-05,
+      "loss": 0.5511,
+      "mean_token_accuracy": 0.7966021656990051,
+      "num_tokens": 57942.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.1836893200874328,
+      "epoch": 0.04906771344455348,
+      "grad_norm": 2.715712070465088,
+      "learning_rate": 4.803921568627452e-05,
+      "loss": 0.5457,
+      "mean_token_accuracy": 0.7974890530109405,
+      "num_tokens": 72086.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1823329687118531,
+      "epoch": 0.058881256133464184,
+      "grad_norm": 2.973444938659668,
+      "learning_rate": 5.784313725490197e-05,
+      "loss": 0.5275,
+      "mean_token_accuracy": 0.8014591693878174,
+      "num_tokens": 85948.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1610147953033447,
+      "epoch": 0.06869479882237488,
+      "grad_norm": 2.5839550495147705,
+      "learning_rate": 6.764705882352942e-05,
+      "loss": 0.5501,
+      "mean_token_accuracy": 0.8008512854576111,
+      "num_tokens": 99922.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.196463143825531,
+      "epoch": 0.07850834151128558,
+      "grad_norm": 2.3468658924102783,
+      "learning_rate": 7.745098039215687e-05,
+      "loss": 0.5107,
+      "mean_token_accuracy": 0.8114270985126495,
+      "num_tokens": 113860.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.1957335233688355,
+      "epoch": 0.08832188420019627,
+      "grad_norm": 2.6540908813476562,
+      "learning_rate": 8.725490196078432e-05,
+      "loss": 0.5031,
+      "mean_token_accuracy": 0.8125855386257171,
+      "num_tokens": 127731.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1685978770256042,
+      "epoch": 0.09813542688910697,
+      "grad_norm": 2.2266597747802734,
+      "learning_rate": 9.705882352941177e-05,
+      "loss": 0.5041,
+      "mean_token_accuracy": 0.8109306931495667,
+      "num_tokens": 141275.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.1610924839973449,
+      "epoch": 0.10794896957801767,
+      "grad_norm": 3.1063902378082275,
+      "learning_rate": 9.999677432611306e-05,
+      "loss": 0.4953,
+      "mean_token_accuracy": 0.807568359375,
+      "num_tokens": 155343.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.2171471118927002,
+      "epoch": 0.11776251226692837,
+      "grad_norm": 3.1443655490875244,
+      "learning_rate": 9.998097610899492e-05,
+      "loss": 0.4763,
+      "mean_token_accuracy": 0.8249376237392425,
+      "num_tokens": 169012.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.2176273941993714,
+      "epoch": 0.12757605495583907,
+      "grad_norm": 2.454484462738037,
+      "learning_rate": 9.995201703267625e-05,
+      "loss": 0.436,
+      "mean_token_accuracy": 0.8323862731456757,
+      "num_tokens": 182627.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.2239107847213746,
+      "epoch": 0.13738959764474976,
+      "grad_norm": 2.638676881790161,
+      "learning_rate": 9.990990472258488e-05,
+      "loss": 0.4588,
+      "mean_token_accuracy": 0.8271334767341614,
+      "num_tokens": 196774.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.2445443749427796,
+      "epoch": 0.14720314033366044,
+      "grad_norm": 2.6380233764648438,
+      "learning_rate": 9.985465026762356e-05,
+      "loss": 0.4709,
+      "mean_token_accuracy": 0.8289254069328308,
+      "num_tokens": 210628.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.216089403629303,
+      "epoch": 0.15701668302257116,
+      "grad_norm": 2.2963876724243164,
+      "learning_rate": 9.978626821725008e-05,
+      "loss": 0.4158,
+      "mean_token_accuracy": 0.8453124284744262,
+      "num_tokens": 224328.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1848299145698546,
+      "epoch": 0.16683022571148184,
+      "grad_norm": 3.0058441162109375,
+      "learning_rate": 9.970477657764611e-05,
+      "loss": 0.4408,
+      "mean_token_accuracy": 0.8356199204921723,
+      "num_tokens": 237401.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.1756147384643554,
+      "epoch": 0.17664376840039253,
+      "grad_norm": 2.4179482460021973,
+      "learning_rate": 9.961019680697593e-05,
+      "loss": 0.4069,
+      "mean_token_accuracy": 0.8423832297325134,
+      "num_tokens": 251219.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.2014240741729736,
+      "epoch": 0.18645731108930325,
+      "grad_norm": 2.614292860031128,
+      "learning_rate": 9.950255380973604e-05,
+      "loss": 0.4691,
+      "mean_token_accuracy": 0.820356696844101,
+      "num_tokens": 265166.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1792091131210327,
+      "epoch": 0.19627085377821393,
+      "grad_norm": 2.417473077774048,
+      "learning_rate": 9.93818759301974e-05,
+      "loss": 0.4075,
+      "mean_token_accuracy": 0.8454431891441345,
+      "num_tokens": 279002.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.1873091101646422,
+      "epoch": 0.20608439646712462,
+      "grad_norm": 2.645998477935791,
+      "learning_rate": 9.924819494494194e-05,
+      "loss": 0.4562,
+      "mean_token_accuracy": 0.8275802850723266,
+      "num_tokens": 292598.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.1683130383491516,
+      "epoch": 0.21589793915603533,
+      "grad_norm": 2.133425712585449,
+      "learning_rate": 9.910154605449517e-05,
+      "loss": 0.4344,
+      "mean_token_accuracy": 0.8326101720333099,
+      "num_tokens": 306156.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1699290990829467,
+      "epoch": 0.22571148184494602,
+      "grad_norm": 2.5865252017974854,
+      "learning_rate": 9.89419678740573e-05,
+      "loss": 0.4024,
+      "mean_token_accuracy": 0.8524027705192566,
+      "num_tokens": 320378.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1994965195655822,
+      "epoch": 0.23552502453385674,
+      "grad_norm": 2.961379289627075,
+      "learning_rate": 9.876950242333518e-05,
+      "loss": 0.4336,
+      "mean_token_accuracy": 0.8376288414001465,
+      "num_tokens": 334099.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1663447976112367,
+      "epoch": 0.24533856722276742,
+      "grad_norm": 2.4368581771850586,
+      "learning_rate": 9.858419511547777e-05,
+      "loss": 0.3679,
+      "mean_token_accuracy": 0.8618759632110595,
+      "num_tokens": 348042.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1842320084571838,
+      "epoch": 0.25515210991167814,
+      "grad_norm": 2.846923351287842,
+      "learning_rate": 9.838609474511811e-05,
+      "loss": 0.4348,
+      "mean_token_accuracy": 0.8382800459861756,
+      "num_tokens": 361362.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1651029229164123,
+      "epoch": 0.2649656526005888,
+      "grad_norm": 2.589449882507324,
+      "learning_rate": 9.81752534755248e-05,
+      "loss": 0.3544,
+      "mean_token_accuracy": 0.8632732629776001,
+      "num_tokens": 376118.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.1716535091400146,
+      "epoch": 0.2747791952894995,
+      "grad_norm": 2.5051488876342773,
+      "learning_rate": 9.795172682486655e-05,
+      "loss": 0.3824,
+      "mean_token_accuracy": 0.8554318726062775,
+      "num_tokens": 389721.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1718938946723938,
+      "epoch": 0.2845927379784102,
+      "grad_norm": 2.7988221645355225,
+      "learning_rate": 9.77155736515932e-05,
+      "loss": 0.3801,
+      "mean_token_accuracy": 0.8628273606300354,
+      "num_tokens": 403995.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.179852032661438,
+      "epoch": 0.2944062806673209,
+      "grad_norm": 3.363668918609619,
+      "learning_rate": 9.746685613893735e-05,
+      "loss": 0.3915,
+      "mean_token_accuracy": 0.8529731690883636,
+      "num_tokens": 417732.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.1462762475013732,
+      "epoch": 0.3042198233562316,
+      "grad_norm": 2.7484560012817383,
+      "learning_rate": 9.720563977854046e-05,
+      "loss": 0.3819,
+      "mean_token_accuracy": 0.8591055750846863,
+      "num_tokens": 431757.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.1650975704193116,
+      "epoch": 0.3140333660451423,
+      "grad_norm": 2.1693973541259766,
+      "learning_rate": 9.693199335320762e-05,
+      "loss": 0.3962,
+      "mean_token_accuracy": 0.8507691621780396,
+      "num_tokens": 445709.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.1248917698860168,
+      "epoch": 0.323846908734053,
+      "grad_norm": 3.421729803085327,
+      "learning_rate": 9.664598891879597e-05,
+      "loss": 0.3164,
+      "mean_token_accuracy": 0.8826701283454895,
+      "num_tokens": 459402.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.130255365371704,
+      "epoch": 0.3336604514229637,
+      "grad_norm": 2.5032639503479004,
+      "learning_rate": 9.634770178524116e-05,
+      "loss": 0.3696,
+      "mean_token_accuracy": 0.8655579328536988,
+      "num_tokens": 473268.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.1501976251602173,
+      "epoch": 0.3434739941118744,
+      "grad_norm": 4.369424343109131,
+      "learning_rate": 9.603721049672678e-05,
+      "loss": 0.4122,
+      "mean_token_accuracy": 0.8430616796016693,
+      "num_tokens": 487423.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.145084309577942,
+      "epoch": 0.35328753680078506,
+      "grad_norm": 2.9527668952941895,
+      "learning_rate": 9.571459681100247e-05,
+      "loss": 0.3561,
+      "mean_token_accuracy": 0.863802683353424,
+      "num_tokens": 501405.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.1479941368103028,
+      "epoch": 0.3631010794896958,
+      "grad_norm": 2.7885732650756836,
+      "learning_rate": 9.537994567785567e-05,
+      "loss": 0.3366,
+      "mean_token_accuracy": 0.8735625565052032,
+      "num_tokens": 515472.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.149704647064209,
+      "epoch": 0.3729146221786065,
+      "grad_norm": 3.112821340560913,
+      "learning_rate": 9.50333452167427e-05,
+      "loss": 0.3739,
+      "mean_token_accuracy": 0.8554477095603943,
+      "num_tokens": 529919.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.1511471271514893,
+      "epoch": 0.38272816486751715,
+      "grad_norm": 3.5844266414642334,
+      "learning_rate": 9.467488669358562e-05,
+      "loss": 0.3824,
+      "mean_token_accuracy": 0.8494103014469147,
+      "num_tokens": 543821.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.1259410738945008,
+      "epoch": 0.39254170755642787,
+      "grad_norm": 2.748136281967163,
+      "learning_rate": 9.430466449674014e-05,
+      "loss": 0.3167,
+      "mean_token_accuracy": 0.8854171872138977,
+      "num_tokens": 556967.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.1162926316261292,
+      "epoch": 0.4023552502453386,
+      "grad_norm": 3.0240674018859863,
+      "learning_rate": 9.39227761121416e-05,
+      "loss": 0.3535,
+      "mean_token_accuracy": 0.8766366362571716,
+      "num_tokens": 570530.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.128273117542267,
+      "epoch": 0.41216879293424924,
+      "grad_norm": 3.427553176879883,
+      "learning_rate": 9.352932209763523e-05,
+      "loss": 0.3602,
+      "mean_token_accuracy": 0.8552050054073334,
+      "num_tokens": 584372.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.1399704813957214,
+      "epoch": 0.42198233562315995,
+      "grad_norm": 2.3567404747009277,
+      "learning_rate": 9.312440605649753e-05,
+      "loss": 0.3451,
+      "mean_token_accuracy": 0.8683524608612061,
+      "num_tokens": 597863.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.1514600276947022,
+      "epoch": 0.43179587831207067,
+      "grad_norm": 3.5659964084625244,
+      "learning_rate": 9.270813461015571e-05,
+      "loss": 0.3238,
+      "mean_token_accuracy": 0.8700128853321075,
+      "num_tokens": 611411.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.1369923233985901,
+      "epoch": 0.44160942100098133,
+      "grad_norm": 2.6093695163726807,
+      "learning_rate": 9.228061737011235e-05,
+      "loss": 0.3229,
+      "mean_token_accuracy": 0.8798930883407593,
+      "num_tokens": 625352.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.1269910097122193,
+      "epoch": 0.45142296368989204,
+      "grad_norm": 2.889234781265259,
+      "learning_rate": 9.184196690908298e-05,
+      "loss": 0.3044,
+      "mean_token_accuracy": 0.8844559609889984,
+      "num_tokens": 638913.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.1272946119308471,
+      "epoch": 0.46123650637880276,
+      "grad_norm": 3.4652466773986816,
+      "learning_rate": 9.139229873135349e-05,
+      "loss": 0.3548,
+      "mean_token_accuracy": 0.859251868724823,
+      "num_tokens": 652839.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.1076605200767518,
+      "epoch": 0.47105004906771347,
+      "grad_norm": 2.148993730545044,
+      "learning_rate": 9.0931731242366e-05,
+      "loss": 0.2924,
+      "mean_token_accuracy": 0.8918448030948639,
+      "num_tokens": 666056.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.1254475951194762,
+      "epoch": 0.48086359175662413,
+      "grad_norm": 2.5391275882720947,
+      "learning_rate": 9.046038571754066e-05,
+      "loss": 0.3311,
+      "mean_token_accuracy": 0.8679941773414612,
+      "num_tokens": 680091.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.1165893435478211,
+      "epoch": 0.49067713444553485,
+      "grad_norm": 3.177361011505127,
+      "learning_rate": 8.997838627034158e-05,
+      "loss": 0.3371,
+      "mean_token_accuracy": 0.871900326013565,
+      "num_tokens": 693207.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.1255882859230042,
+      "epoch": 0.5004906771344455,
+      "grad_norm": 2.697605848312378,
+      "learning_rate": 8.94858598195958e-05,
+      "loss": 0.3478,
+      "mean_token_accuracy": 0.8658807158470154,
+      "num_tokens": 707709.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.129603922367096,
+      "epoch": 0.5103042198233563,
+      "grad_norm": 3.300055742263794,
+      "learning_rate": 8.898293605607316e-05,
+      "loss": 0.346,
+      "mean_token_accuracy": 0.8671878039836883,
+      "num_tokens": 721442.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.1072421193122863,
+      "epoch": 0.5201177625122669,
+      "grad_norm": 3.106497287750244,
+      "learning_rate": 8.846974740833659e-05,
+      "loss": 0.3303,
+      "mean_token_accuracy": 0.8723613381385803,
+      "num_tokens": 735372.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.109989547729492,
+      "epoch": 0.5299313052011776,
+      "grad_norm": 3.499234437942505,
+      "learning_rate": 8.794642900787137e-05,
+      "loss": 0.3137,
+      "mean_token_accuracy": 0.8823532998561859,
+      "num_tokens": 749253.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.1101910710334777,
+      "epoch": 0.5397448478900884,
+      "grad_norm": 2.6812968254089355,
+      "learning_rate": 8.741311865350259e-05,
+      "loss": 0.314,
+      "mean_token_accuracy": 0.8798675417900086,
+      "num_tokens": 762556.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.115459954738617,
+      "epoch": 0.549558390578999,
+      "grad_norm": 4.910181522369385,
+      "learning_rate": 8.68699567751104e-05,
+      "loss": 0.336,
+      "mean_token_accuracy": 0.8714472532272339,
+      "num_tokens": 776256.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.1056787490844726,
+      "epoch": 0.5593719332679097,
+      "grad_norm": 3.6729321479797363,
+      "learning_rate": 8.631708639665234e-05,
+      "loss": 0.3193,
+      "mean_token_accuracy": 0.8805031061172486,
+      "num_tokens": 789807.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.1196874499320983,
+      "epoch": 0.5691854759568205,
+      "grad_norm": 2.943310022354126,
+      "learning_rate": 8.575465309850265e-05,
+      "loss": 0.3202,
+      "mean_token_accuracy": 0.8787055313587189,
+      "num_tokens": 803196.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.1407753825187683,
+      "epoch": 0.5789990186457311,
+      "grad_norm": 2.8891053199768066,
+      "learning_rate": 8.518280497911841e-05,
+      "loss": 0.3136,
+      "mean_token_accuracy": 0.8832496523857116,
+      "num_tokens": 816858.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.1431702256202698,
+      "epoch": 0.5888125613346418,
+      "grad_norm": 3.5767548084259033,
+      "learning_rate": 8.460169261604274e-05,
+      "loss": 0.3144,
+      "mean_token_accuracy": 0.8760414302349091,
+      "num_tokens": 830838.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.1411950707435607,
+      "epoch": 0.5986261040235525,
+      "grad_norm": 3.01652455329895,
+      "learning_rate": 8.4011469026255e-05,
+      "loss": 0.3254,
+      "mean_token_accuracy": 0.8686141312122345,
+      "num_tokens": 845426.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.1344624996185302,
+      "epoch": 0.6084396467124632,
+      "grad_norm": 2.788529634475708,
+      "learning_rate": 8.341228962587882e-05,
+      "loss": 0.303,
+      "mean_token_accuracy": 0.8871294140815735,
+      "num_tokens": 859388.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.1596095085144043,
+      "epoch": 0.6182531894013739,
+      "grad_norm": 2.7421834468841553,
+      "learning_rate": 8.28043121892583e-05,
+      "loss": 0.3256,
+      "mean_token_accuracy": 0.876429408788681,
+      "num_tokens": 872921.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.1580193519592286,
+      "epoch": 0.6280667320902846,
+      "grad_norm": 3.5501387119293213,
+      "learning_rate": 8.218769680741336e-05,
+      "loss": 0.2988,
+      "mean_token_accuracy": 0.8860776424407959,
+      "num_tokens": 886447.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.1551304936408997,
+      "epoch": 0.6378802747791953,
+      "grad_norm": 3.6257665157318115,
+      "learning_rate": 8.156260584588491e-05,
+      "loss": 0.2727,
+      "mean_token_accuracy": 0.8875501990318299,
+      "num_tokens": 900382.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.1438666224479674,
+      "epoch": 0.647693817468106,
+      "grad_norm": 3.621591567993164,
+      "learning_rate": 8.09292039019812e-05,
+      "loss": 0.3178,
+      "mean_token_accuracy": 0.8770913124084473,
+      "num_tokens": 914022.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.1409793496131897,
+      "epoch": 0.6575073601570167,
+      "grad_norm": 3.0630064010620117,
+      "learning_rate": 8.028765776143654e-05,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.8927100837230683,
+      "num_tokens": 928505.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.1416229605674744,
+      "epoch": 0.6673209028459274,
+      "grad_norm": 2.5532400608062744,
+      "learning_rate": 7.963813635449367e-05,
+      "loss": 0.285,
+      "mean_token_accuracy": 0.8899939596652985,
+      "num_tokens": 942352.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.1464022159576417,
+      "epoch": 0.677134445534838,
+      "grad_norm": 3.0438241958618164,
+      "learning_rate": 7.898081071142149e-05,
+      "loss": 0.309,
+      "mean_token_accuracy": 0.8795955419540405,
+      "num_tokens": 956020.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.1194156765937806,
+      "epoch": 0.6869479882237488,
+      "grad_norm": 2.339188575744629,
+      "learning_rate": 7.831585391747984e-05,
+      "loss": 0.2925,
+      "mean_token_accuracy": 0.8836807370185852,
+      "num_tokens": 969183.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.112407422065735,
+      "epoch": 0.6967615309126595,
+      "grad_norm": 3.385798931121826,
+      "learning_rate": 7.764344106734319e-05,
+      "loss": 0.2629,
+      "mean_token_accuracy": 0.9031207144260407,
+      "num_tokens": 982822.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.1299761652946472,
+      "epoch": 0.7065750736015701,
+      "grad_norm": 2.8866264820098877,
+      "learning_rate": 7.696374921899506e-05,
+      "loss": 0.3081,
+      "mean_token_accuracy": 0.8800933182239532,
+      "num_tokens": 996591.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.1448111891746522,
+      "epoch": 0.7163886162904809,
+      "grad_norm": 2.8464834690093994,
+      "learning_rate": 7.627695734710564e-05,
+      "loss": 0.2977,
+      "mean_token_accuracy": 0.8861884295940399,
+      "num_tokens": 1010621.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.125222635269165,
+      "epoch": 0.7262021589793916,
+      "grad_norm": 2.970479726791382,
+      "learning_rate": 7.55832462959047e-05,
+      "loss": 0.2918,
+      "mean_token_accuracy": 0.886595755815506,
+      "num_tokens": 1024687.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.1139266610145568,
+      "epoch": 0.7360157016683022,
+      "grad_norm": 3.2461984157562256,
+      "learning_rate": 7.488279873156207e-05,
+      "loss": 0.291,
+      "mean_token_accuracy": 0.885616534948349,
+      "num_tokens": 1038386.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.1345422625541688,
+      "epoch": 0.745829244357213,
+      "grad_norm": 2.73522686958313,
+      "learning_rate": 7.417579909408851e-05,
+      "loss": 0.256,
+      "mean_token_accuracy": 0.8984665513038635,
+      "num_tokens": 1052566.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.12027747631073,
+      "epoch": 0.7556427870461236,
+      "grad_norm": 2.880068063735962,
+      "learning_rate": 7.346243354876944e-05,
+      "loss": 0.286,
+      "mean_token_accuracy": 0.8838774800300598,
+      "num_tokens": 1065886.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.1296450614929199,
+      "epoch": 0.7654563297350343,
+      "grad_norm": 3.738333225250244,
+      "learning_rate": 7.274288993714446e-05,
+      "loss": 0.3022,
+      "mean_token_accuracy": 0.8869429051876068,
+      "num_tokens": 1079240.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.111961877346039,
+      "epoch": 0.7752698724239451,
+      "grad_norm": 3.0505549907684326,
+      "learning_rate": 7.201735772754523e-05,
+      "loss": 0.2595,
+      "mean_token_accuracy": 0.8995143055915833,
+      "num_tokens": 1093042.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.1211289644241333,
+      "epoch": 0.7850834151128557,
+      "grad_norm": 3.0807385444641113,
+      "learning_rate": 7.128602796520538e-05,
+      "loss": 0.2927,
+      "mean_token_accuracy": 0.8852217257022857,
+      "num_tokens": 1106340.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.1306297183036804,
+      "epoch": 0.7948969578017664,
+      "grad_norm": 3.833285331726074,
+      "learning_rate": 7.054909322195485e-05,
+      "loss": 0.281,
+      "mean_token_accuracy": 0.890252310037613,
+      "num_tokens": 1120247.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.1249518394470215,
+      "epoch": 0.8047105004906772,
+      "grad_norm": 2.6327736377716064,
+      "learning_rate": 6.980674754551239e-05,
+      "loss": 0.2726,
+      "mean_token_accuracy": 0.8915690362453461,
+      "num_tokens": 1133855.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.1259310007095338,
+      "epoch": 0.8145240431795878,
+      "grad_norm": 3.295334815979004,
+      "learning_rate": 6.905918640838946e-05,
+      "loss": 0.2414,
+      "mean_token_accuracy": 0.9124344527721405,
+      "num_tokens": 1148115.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.1244077324867248,
+      "epoch": 0.8243375858684985,
+      "grad_norm": 3.1344072818756104,
+      "learning_rate": 6.830660665641898e-05,
+      "loss": 0.2699,
+      "mean_token_accuracy": 0.8973037302494049,
+      "num_tokens": 1162637.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.1486279129981996,
+      "epoch": 0.8341511285574092,
+      "grad_norm": 3.80707049369812,
+      "learning_rate": 6.754920645692222e-05,
+      "loss": 0.264,
+      "mean_token_accuracy": 0.8927407264709473,
+      "num_tokens": 1176413.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.146347463130951,
+      "epoch": 0.8439646712463199,
+      "grad_norm": 2.407167434692383,
+      "learning_rate": 6.67871852465281e-05,
+      "loss": 0.2657,
+      "mean_token_accuracy": 0.8997598767280579,
+      "num_tokens": 1190613.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.1218132376670837,
+      "epoch": 0.8537782139352306,
+      "grad_norm": 3.314131498336792,
+      "learning_rate": 6.602074367865794e-05,
+      "loss": 0.2716,
+      "mean_token_accuracy": 0.8935964643955231,
+      "num_tokens": 1204095.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.1313920140266418,
+      "epoch": 0.8635917566241413,
+      "grad_norm": 3.242176055908203,
+      "learning_rate": 6.525008357069e-05,
+      "loss": 0.2805,
+      "mean_token_accuracy": 0.889654129743576,
+      "num_tokens": 1218081.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.1171419024467468,
+      "epoch": 0.873405299313052,
+      "grad_norm": 3.2730391025543213,
+      "learning_rate": 6.447540785081738e-05,
+      "loss": 0.2721,
+      "mean_token_accuracy": 0.8956381738185882,
+      "num_tokens": 1231816.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.130664551258087,
+      "epoch": 0.8832188420019627,
+      "grad_norm": 3.584834337234497,
+      "learning_rate": 6.36969205046136e-05,
+      "loss": 0.2695,
+      "mean_token_accuracy": 0.9002387225627899,
+      "num_tokens": 1245924.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.1180906891822815,
+      "epoch": 0.8930323846908734,
+      "grad_norm": 3.3112573623657227,
+      "learning_rate": 6.291482652131955e-05,
+      "loss": 0.2331,
+      "mean_token_accuracy": 0.903629732131958,
+      "num_tokens": 1259777.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.108742618560791,
+      "epoch": 0.9028459273797841,
+      "grad_norm": 3.647545337677002,
+      "learning_rate": 6.212933183986634e-05,
+      "loss": 0.251,
+      "mean_token_accuracy": 0.8992014348506927,
+      "num_tokens": 1273277.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.1139172434806823,
+      "epoch": 0.9126594700686947,
+      "grad_norm": 3.5406508445739746,
+      "learning_rate": 6.1340643294648e-05,
+      "loss": 0.2608,
+      "mean_token_accuracy": 0.8990993559360504,
+      "num_tokens": 1287025.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.103180742263794,
+      "epoch": 0.9224730127576055,
+      "grad_norm": 3.086604118347168,
+      "learning_rate": 6.054896856105834e-05,
+      "loss": 0.2514,
+      "mean_token_accuracy": 0.9034869372844696,
+      "num_tokens": 1300830.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.1212706208229064,
+      "epoch": 0.9322865554465162,
+      "grad_norm": 2.703345537185669,
+      "learning_rate": 5.9754516100806423e-05,
+      "loss": 0.2463,
+      "mean_token_accuracy": 0.9023334562778473,
+      "num_tokens": 1314956.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.1177714943885804,
+      "epoch": 0.9421000981354269,
+      "grad_norm": 3.242586851119995,
+      "learning_rate": 5.895749510702502e-05,
+      "loss": 0.2636,
+      "mean_token_accuracy": 0.8941789031028747,
+      "num_tokens": 1329379.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.0900061964988708,
+      "epoch": 0.9519136408243376,
+      "grad_norm": 3.4688076972961426,
+      "learning_rate": 5.815811544918641e-05,
+      "loss": 0.2534,
+      "mean_token_accuracy": 0.8997189402580261,
+      "num_tokens": 1343040.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.1079393744468689,
+      "epoch": 0.9617271835132483,
+      "grad_norm": 4.0240678787231445,
+      "learning_rate": 5.7356587617840065e-05,
+      "loss": 0.2668,
+      "mean_token_accuracy": 0.8944048523902893,
+      "num_tokens": 1356933.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.0891769528388977,
+      "epoch": 0.971540726202159,
+      "grad_norm": 4.212469577789307,
+      "learning_rate": 5.655312266918684e-05,
+      "loss": 0.251,
+      "mean_token_accuracy": 0.9003695785999298,
+      "num_tokens": 1369646.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.0943650960922242,
+      "epoch": 0.9813542688910697,
+      "grad_norm": 3.1624763011932373,
+      "learning_rate": 5.574793216950428e-05,
+      "loss": 0.2523,
+      "mean_token_accuracy": 0.9003262519836426,
+      "num_tokens": 1383011.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.099990177154541,
+      "epoch": 0.9911678115799804,
+      "grad_norm": 3.425002336502075,
+      "learning_rate": 5.494122813943742e-05,
+      "loss": 0.267,
+      "mean_token_accuracy": 0.8981301426887512,
+      "num_tokens": 1396830.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.0962233304977418,
+      "epoch": 1.000981354268891,
+      "grad_norm": 2.304790735244751,
+      "learning_rate": 5.413322299817019e-05,
+      "loss": 0.2265,
+      "mean_token_accuracy": 0.909206885099411,
+      "num_tokens": 1410013.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.0763352036476135,
+      "epoch": 1.0107948969578018,
+      "grad_norm": 3.846078395843506,
+      "learning_rate": 5.332412950749169e-05,
+      "loss": 0.1931,
+      "mean_token_accuracy": 0.9306668102741241,
+      "num_tokens": 1422787.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.0906688690185546,
+      "epoch": 1.0206084396467126,
+      "grad_norm": 2.8801040649414062,
+      "learning_rate": 5.251416071577232e-05,
+      "loss": 0.1521,
+      "mean_token_accuracy": 0.9431000292301178,
+      "num_tokens": 1436816.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.0783058285713196,
+      "epoch": 1.030421982335623,
+      "grad_norm": 2.7937142848968506,
+      "learning_rate": 5.170352990186441e-05,
+      "loss": 0.1629,
+      "mean_token_accuracy": 0.9430882096290588,
+      "num_tokens": 1450938.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.058233630657196,
+      "epoch": 1.0402355250245339,
+      "grad_norm": 3.37957501411438,
+      "learning_rate": 5.08924505189423e-05,
+      "loss": 0.127,
+      "mean_token_accuracy": 0.9534770429134369,
+      "num_tokens": 1464578.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.0746712803840637,
+      "epoch": 1.0500490677134446,
+      "grad_norm": 3.830491304397583,
+      "learning_rate": 5.008113613829631e-05,
+      "loss": 0.1485,
+      "mean_token_accuracy": 0.9499725878238678,
+      "num_tokens": 1478786.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.0674135327339171,
+      "epoch": 1.0598626104023552,
+      "grad_norm": 2.295130968093872,
+      "learning_rate": 4.9269800393095747e-05,
+      "loss": 0.1514,
+      "mean_token_accuracy": 0.9438324391841888,
+      "num_tokens": 1492038.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.0673683524131774,
+      "epoch": 1.069676153091266,
+      "grad_norm": 3.5985095500946045,
+      "learning_rate": 4.845865692213555e-05,
+      "loss": 0.142,
+      "mean_token_accuracy": 0.9452584385871887,
+      "num_tokens": 1506078.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.049016296863556,
+      "epoch": 1.0794896957801767,
+      "grad_norm": 3.964970111846924,
+      "learning_rate": 4.764791931358155e-05,
+      "loss": 0.1601,
+      "mean_token_accuracy": 0.9393302917480468,
+      "num_tokens": 1520134.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.0373982429504394,
+      "epoch": 1.0893032384690873,
+      "grad_norm": 3.2233448028564453,
+      "learning_rate": 4.683780104872894e-05,
+      "loss": 0.1254,
+      "mean_token_accuracy": 0.9540755331516266,
+      "num_tokens": 1534067.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.0517881631851196,
+      "epoch": 1.099116781157998,
+      "grad_norm": 3.134645462036133,
+      "learning_rate": 4.6028515445788937e-05,
+      "loss": 0.1418,
+      "mean_token_accuracy": 0.9444664657115937,
+      "num_tokens": 1547937.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.0505053758621217,
+      "epoch": 1.1089303238469088,
+      "grad_norm": 2.8564951419830322,
+      "learning_rate": 4.5220275603718474e-05,
+      "loss": 0.1584,
+      "mean_token_accuracy": 0.9381029963493347,
+      "num_tokens": 1561944.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.054303765296936,
+      "epoch": 1.1187438665358194,
+      "grad_norm": 3.2089507579803467,
+      "learning_rate": 4.4413294346107394e-05,
+      "loss": 0.1501,
+      "mean_token_accuracy": 0.9449402630329132,
+      "num_tokens": 1575795.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.0506520748138428,
+      "epoch": 1.1285574092247301,
+      "grad_norm": 2.7969441413879395,
+      "learning_rate": 4.3607784165138474e-05,
+      "loss": 0.1386,
+      "mean_token_accuracy": 0.9484172224998474,
+      "num_tokens": 1589752.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.0223258495330811,
+      "epoch": 1.138370951913641,
+      "grad_norm": 3.612339496612549,
+      "learning_rate": 4.280395716563441e-05,
+      "loss": 0.1341,
+      "mean_token_accuracy": 0.9483618378639221,
+      "num_tokens": 1603125.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.0427632749080658,
+      "epoch": 1.1481844946025515,
+      "grad_norm": 3.1086626052856445,
+      "learning_rate": 4.2002025009206994e-05,
+      "loss": 0.1435,
+      "mean_token_accuracy": 0.9457801938056946,
+      "num_tokens": 1617222.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.0389100551605224,
+      "epoch": 1.1579980372914622,
+      "grad_norm": 4.340463638305664,
+      "learning_rate": 4.120219885852292e-05,
+      "loss": 0.1263,
+      "mean_token_accuracy": 0.9541761696338653,
+      "num_tokens": 1631127.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.0372697234153747,
+      "epoch": 1.167811579980373,
+      "grad_norm": 2.8450355529785156,
+      "learning_rate": 4.040468932170097e-05,
+      "loss": 0.158,
+      "mean_token_accuracy": 0.9411039829254151,
+      "num_tokens": 1644805.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.040115350484848,
+      "epoch": 1.1776251226692835,
+      "grad_norm": 2.79168701171875,
+      "learning_rate": 3.9609706396855296e-05,
+      "loss": 0.1453,
+      "mean_token_accuracy": 0.9511331260204315,
+      "num_tokens": 1658032.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.0397627472877502,
+      "epoch": 1.1874386653581943,
+      "grad_norm": 3.2111728191375732,
+      "learning_rate": 3.881745941679917e-05,
+      "loss": 0.1318,
+      "mean_token_accuracy": 0.9524486124515533,
+      "num_tokens": 1672118.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.0310355305671692,
+      "epoch": 1.197252208047105,
+      "grad_norm": 4.345663070678711,
+      "learning_rate": 3.802815699392418e-05,
+      "loss": 0.1352,
+      "mean_token_accuracy": 0.9509382545948029,
+      "num_tokens": 1686510.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.0335936963558197,
+      "epoch": 1.2070657507360156,
+      "grad_norm": 4.464991569519043,
+      "learning_rate": 3.724200696526873e-05,
+      "loss": 0.1368,
+      "mean_token_accuracy": 0.9495227634906769,
+      "num_tokens": 1700552.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.0412399291992187,
+      "epoch": 1.2168792934249264,
+      "grad_norm": 4.525125026702881,
+      "learning_rate": 3.6459216337791124e-05,
+      "loss": 0.1461,
+      "mean_token_accuracy": 0.9473684191703796,
+      "num_tokens": 1714419.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.0487755298614503,
+      "epoch": 1.2266928361138372,
+      "grad_norm": 3.5593583583831787,
+      "learning_rate": 3.567999123386103e-05,
+      "loss": 0.1531,
+      "mean_token_accuracy": 0.940424108505249,
+      "num_tokens": 1728462.0,
+      "step": 1250
+    },
+    {
+      "entropy": 1.0267175793647767,
+      "epoch": 1.2365063788027477,
+      "grad_norm": 2.77921724319458,
+      "learning_rate": 3.490453683698376e-05,
+      "loss": 0.1278,
+      "mean_token_accuracy": 0.9530017018318176,
+      "num_tokens": 1742407.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.0439971923828124,
+      "epoch": 1.2463199214916585,
+      "grad_norm": 3.4589180946350098,
+      "learning_rate": 3.4133057337771995e-05,
+      "loss": 0.1349,
+      "mean_token_accuracy": 0.9498993694782257,
+      "num_tokens": 1756874.0,
+      "step": 1270
+    },
+    {
+      "entropy": 1.0280747771263123,
+      "epoch": 1.2561334641805693,
+      "grad_norm": 3.7395787239074707,
+      "learning_rate": 3.336575588017881e-05,
+      "loss": 0.1305,
+      "mean_token_accuracy": 0.9522841572761536,
+      "num_tokens": 1770683.0,
+      "step": 1280
+    },
+    {
+      "entropy": 1.015505415201187,
+      "epoch": 1.2659470068694798,
+      "grad_norm": 3.643556594848633,
+      "learning_rate": 3.260283450800634e-05,
+      "loss": 0.1244,
+      "mean_token_accuracy": 0.954523253440857,
+      "num_tokens": 1784307.0,
+      "step": 1290
+    },
+    {
+      "entropy": 1.0291505575180053,
+      "epoch": 1.2757605495583906,
+      "grad_norm": 2.587498426437378,
+      "learning_rate": 3.184449411170416e-05,
+      "loss": 0.1177,
+      "mean_token_accuracy": 0.9558589577674865,
+      "num_tokens": 1797783.0,
+      "step": 1300
+    },
+    {
+      "entropy": 1.0395447969436646,
+      "epoch": 1.2855740922473013,
+      "grad_norm": 3.345717430114746,
+      "learning_rate": 3.109093437547128e-05,
+      "loss": 0.1341,
+      "mean_token_accuracy": 0.9515320658683777,
+      "num_tokens": 1811644.0,
+      "step": 1310
+    },
+    {
+      "entropy": 1.0311142385005951,
+      "epoch": 1.295387634936212,
+      "grad_norm": 3.8962409496307373,
+      "learning_rate": 3.0342353724675966e-05,
+      "loss": 0.1199,
+      "mean_token_accuracy": 0.9551293432712555,
+      "num_tokens": 1826400.0,
+      "step": 1320
+    },
+    {
+      "entropy": 1.0069368183612823,
+      "epoch": 1.3052011776251227,
+      "grad_norm": 3.8959462642669678,
+      "learning_rate": 2.9598949273606702e-05,
+      "loss": 0.1342,
+      "mean_token_accuracy": 0.9522397518157959,
+      "num_tokens": 1840062.0,
+      "step": 1330
+    },
+    {
+      "entropy": 1.0122943460941314,
+      "epoch": 1.3150147203140334,
+      "grad_norm": 4.319382190704346,
+      "learning_rate": 2.8860916773568703e-05,
+      "loss": 0.1334,
+      "mean_token_accuracy": 0.9483747243881225,
+      "num_tokens": 1853118.0,
+      "step": 1340
+    },
+    {
+      "entropy": 1.008552074432373,
+      "epoch": 1.324828263002944,
+      "grad_norm": 4.5696587562561035,
+      "learning_rate": 2.8128450561339193e-05,
+      "loss": 0.1345,
+      "mean_token_accuracy": 0.9493142604827881,
+      "num_tokens": 1866860.0,
+      "step": 1350
+    },
+    {
+      "entropy": 1.000309818983078,
+      "epoch": 1.3346418056918548,
+      "grad_norm": 3.7926957607269287,
+      "learning_rate": 2.740174350799507e-05,
+      "loss": 0.1146,
+      "mean_token_accuracy": 0.957521378993988,
+      "num_tokens": 1880655.0,
+      "step": 1360
+    },
+    {
+      "entropy": 1.0170307278633117,
+      "epoch": 1.3444553483807655,
+      "grad_norm": 3.7186636924743652,
+      "learning_rate": 2.668098696812675e-05,
+      "loss": 0.1373,
+      "mean_token_accuracy": 0.9492245197296143,
+      "num_tokens": 1894338.0,
+      "step": 1370
+    },
+    {
+      "entropy": 1.0181719303131103,
+      "epoch": 1.354268891069676,
+      "grad_norm": 3.8349251747131348,
+      "learning_rate": 2.596637072945099e-05,
+      "loss": 0.1444,
+      "mean_token_accuracy": 0.9443652033805847,
+      "num_tokens": 1908186.0,
+      "step": 1380
+    },
+    {
+      "entropy": 1.0160548567771912,
+      "epoch": 1.3640824337585868,
+      "grad_norm": 4.035093307495117,
+      "learning_rate": 2.5258082962836614e-05,
+      "loss": 0.1195,
+      "mean_token_accuracy": 0.9590700268745422,
+      "num_tokens": 1921337.0,
+      "step": 1390
+    },
+    {
+      "entropy": 1.015166699886322,
+      "epoch": 1.3738959764474976,
+      "grad_norm": 3.099020481109619,
+      "learning_rate": 2.4556310172755898e-05,
+      "loss": 0.1096,
+      "mean_token_accuracy": 0.9601570069789886,
+      "num_tokens": 1935344.0,
+      "step": 1400
+    },
+    {
+      "entropy": 1.0220223903656005,
+      "epoch": 1.3837095191364082,
+      "grad_norm": 5.150036811828613,
+      "learning_rate": 2.3861237148174474e-05,
+      "loss": 0.1235,
+      "mean_token_accuracy": 0.9531738042831421,
+      "num_tokens": 1949724.0,
+      "step": 1410
+    },
+    {
+      "entropy": 1.0124029636383056,
+      "epoch": 1.393523061825319,
+      "grad_norm": 3.7734103202819824,
+      "learning_rate": 2.3173046913893502e-05,
+      "loss": 0.1297,
+      "mean_token_accuracy": 0.9484266996383667,
+      "num_tokens": 1963602.0,
+      "step": 1420
+    },
+    {
+      "entropy": 1.0174829185009002,
+      "epoch": 1.4033366045142297,
+      "grad_norm": 2.9643824100494385,
+      "learning_rate": 2.249192068235581e-05,
+      "loss": 0.1153,
+      "mean_token_accuracy": 0.9558221817016601,
+      "num_tokens": 1977163.0,
+      "step": 1430
+    },
+    {
+      "entropy": 1.0273181676864624,
+      "epoch": 1.4131501472031402,
+      "grad_norm": 2.5659730434417725,
+      "learning_rate": 2.181803780592959e-05,
+      "loss": 0.1211,
+      "mean_token_accuracy": 0.9560111165046692,
+      "num_tokens": 1991469.0,
+      "step": 1440
+    },
+    {
+      "entropy": 1.0325281143188476,
+      "epoch": 1.422963689892051,
+      "grad_norm": 3.936584234237671,
+      "learning_rate": 2.1151575729681883e-05,
+      "loss": 0.1248,
+      "mean_token_accuracy": 0.9545759916305542,
+      "num_tokens": 2005117.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.019373095035553,
+      "epoch": 1.4327772325809618,
+      "grad_norm": 3.6255650520324707,
+      "learning_rate": 2.049270994465407e-05,
+      "loss": 0.1156,
+      "mean_token_accuracy": 0.9583606660366059,
+      "num_tokens": 2018880.0,
+      "step": 1460
+    },
+    {
+      "entropy": 1.0305926322937011,
+      "epoch": 1.4425907752698723,
+      "grad_norm": 2.6658413410186768,
+      "learning_rate": 1.9841613941652144e-05,
+      "loss": 0.115,
+      "mean_token_accuracy": 0.9558991491794586,
+      "num_tokens": 2033042.0,
+      "step": 1470
+    },
+    {
+      "entropy": 1.0179109454154969,
+      "epoch": 1.452404317958783,
+      "grad_norm": 2.4910435676574707,
+      "learning_rate": 1.9198459165563503e-05,
+      "loss": 0.1388,
+      "mean_token_accuracy": 0.9512788414955139,
+      "num_tokens": 2047545.0,
+      "step": 1480
+    },
+    {
+      "entropy": 1.0108248472213746,
+      "epoch": 1.4622178606476939,
+      "grad_norm": 3.1441121101379395,
+      "learning_rate": 1.8563414970212555e-05,
+      "loss": 0.1135,
+      "mean_token_accuracy": 0.9559279561042786,
+      "num_tokens": 2060959.0,
+      "step": 1490
+    },
+    {
+      "entropy": 1.0083947658538819,
+      "epoch": 1.4720314033366044,
+      "grad_norm": 2.797607660293579,
+      "learning_rate": 1.7936648573766957e-05,
+      "loss": 0.1192,
+      "mean_token_accuracy": 0.9546716153621674,
+      "num_tokens": 2074380.0,
+      "step": 1500
+    },
+    {
+      "entropy": 1.012255847454071,
+      "epoch": 1.4818449460255152,
+      "grad_norm": 2.9938125610351562,
+      "learning_rate": 1.731832501470606e-05,
+      "loss": 0.1302,
+      "mean_token_accuracy": 0.9517545700073242,
+      "num_tokens": 2088259.0,
+      "step": 1510
+    },
+    {
+      "entropy": 1.0249765813350677,
+      "epoch": 1.491658488714426,
+      "grad_norm": 3.7912890911102295,
+      "learning_rate": 1.670860710836355e-05,
+      "loss": 0.127,
+      "mean_token_accuracy": 0.9514165878295898,
+      "num_tokens": 2102668.0,
+      "step": 1520
+    },
+    {
+      "entropy": 1.027025532722473,
+      "epoch": 1.5014720314033365,
+      "grad_norm": 3.928774118423462,
+      "learning_rate": 1.6107655404055223e-05,
+      "loss": 0.1309,
+      "mean_token_accuracy": 0.9505966067314148,
+      "num_tokens": 2116240.0,
+      "step": 1530
+    },
+    {
+      "entropy": 1.0198703765869142,
+      "epoch": 1.5112855740922473,
+      "grad_norm": 3.710538625717163,
+      "learning_rate": 1.5515628142803562e-05,
+      "loss": 0.1189,
+      "mean_token_accuracy": 0.9579048216342926,
+      "num_tokens": 2130030.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.9912734866142273,
+      "epoch": 1.521099116781158,
+      "grad_norm": 3.1784017086029053,
+      "learning_rate": 1.493268121567028e-05,
+      "loss": 0.1191,
+      "mean_token_accuracy": 0.958934086561203,
+      "num_tokens": 2143557.0,
+      "step": 1550
+    },
+    {
+      "entropy": 1.0070856213569641,
+      "epoch": 1.5309126594700686,
+      "grad_norm": 2.6208243370056152,
+      "learning_rate": 1.435896812270729e-05,
+      "loss": 0.1058,
+      "mean_token_accuracy": 0.9623668611049652,
+      "num_tokens": 2156943.0,
+      "step": 1560
+    },
+    {
+      "entropy": 1.032704585790634,
+      "epoch": 1.5407262021589794,
+      "grad_norm": 3.428375720977783,
+      "learning_rate": 1.3794639932537645e-05,
+      "loss": 0.1155,
+      "mean_token_accuracy": 0.958466112613678,
+      "num_tokens": 2170771.0,
+      "step": 1570
+    },
+    {
+      "entropy": 1.0376219987869262,
+      "epoch": 1.5505397448478901,
+      "grad_norm": 2.956847667694092,
+      "learning_rate": 1.323984524257641e-05,
+      "loss": 0.1069,
+      "mean_token_accuracy": 0.9641427993774414,
+      "num_tokens": 2184843.0,
+      "step": 1580
+    },
+    {
+      "entropy": 1.0076554596424103,
+      "epoch": 1.5603532875368007,
+      "grad_norm": 3.8850271701812744,
+      "learning_rate": 1.2694730139902522e-05,
+      "loss": 0.1128,
+      "mean_token_accuracy": 0.9567165613174439,
+      "num_tokens": 2198448.0,
+      "step": 1590
+    },
+    {
+      "entropy": 1.0166796445846558,
+      "epoch": 1.5701668302257115,
+      "grad_norm": 3.5852279663085938,
+      "learning_rate": 1.2159438162791348e-05,
+      "loss": 0.1168,
+      "mean_token_accuracy": 0.9581134557723999,
+      "num_tokens": 2212561.0,
+      "step": 1600
+    },
+    {
+      "entropy": 1.0203666985034943,
+      "epoch": 1.5799803729146222,
+      "grad_norm": 3.263855457305908,
+      "learning_rate": 1.1634110262918718e-05,
+      "loss": 0.1129,
+      "mean_token_accuracy": 0.9572122633457184,
+      "num_tokens": 2226486.0,
+      "step": 1610
+    },
+    {
+      "entropy": 1.0241987705230713,
+      "epoch": 1.5897939156035328,
+      "grad_norm": 4.066913604736328,
+      "learning_rate": 1.1118884768245863e-05,
+      "loss": 0.1115,
+      "mean_token_accuracy": 0.9587595224380493,
+      "num_tokens": 2239717.0,
+      "step": 1620
+    },
+    {
+      "entropy": 1.0307183504104613,
+      "epoch": 1.5996074582924436,
+      "grad_norm": 3.5653746128082275,
+      "learning_rate": 1.0613897346595203e-05,
+      "loss": 0.1083,
+      "mean_token_accuracy": 0.9601296603679657,
+      "num_tokens": 2253591.0,
+      "step": 1630
+    },
+    {
+      "entropy": 1.0288657188415526,
+      "epoch": 1.6094210009813543,
+      "grad_norm": 3.2822489738464355,
+      "learning_rate": 1.011928096992672e-05,
+      "loss": 0.122,
+      "mean_token_accuracy": 0.9571761786937714,
+      "num_tokens": 2267025.0,
+      "step": 1640
+    },
+    {
+      "entropy": 1.0288108110427856,
+      "epoch": 1.6192345436702649,
+      "grad_norm": 3.393726110458374,
+      "learning_rate": 9.635165879324083e-06,
+      "loss": 0.1082,
+      "mean_token_accuracy": 0.9568516552448273,
+      "num_tokens": 2280813.0,
+      "step": 1650
+    },
+    {
+      "entropy": 1.0282153844833375,
+      "epoch": 1.6290480863591756,
+      "grad_norm": 3.267589807510376,
+      "learning_rate": 9.161679550699936e-06,
+      "loss": 0.1135,
+      "mean_token_accuracy": 0.9570296704769135,
+      "num_tokens": 2294463.0,
+      "step": 1660
+    },
+    {
+      "entropy": 1.0170055866241454,
+      "epoch": 1.6388616290480864,
+      "grad_norm": 3.1603331565856934,
+      "learning_rate": 8.698946661229262e-06,
+      "loss": 0.1187,
+      "mean_token_accuracy": 0.9588164925575257,
+      "num_tokens": 2308089.0,
+      "step": 1670
+    },
+    {
+      "entropy": 1.026942628622055,
+      "epoch": 1.648675171736997,
+      "grad_norm": 3.0629947185516357,
+      "learning_rate": 8.24708905651972e-06,
+      "loss": 0.1148,
+      "mean_token_accuracy": 0.956197190284729,
+      "num_tokens": 2322802.0,
+      "step": 1680
+    },
+    {
+      "entropy": 1.01946781873703,
+      "epoch": 1.6584887144259077,
+      "grad_norm": 2.9364676475524902,
+      "learning_rate": 7.80622571852761e-06,
+      "loss": 0.1001,
+      "mean_token_accuracy": 0.964062649011612,
+      "num_tokens": 2336108.0,
+      "step": 1690
+    },
+    {
+      "entropy": 1.0196425318717957,
+      "epoch": 1.6683022571148185,
+      "grad_norm": 3.124826192855835,
+      "learning_rate": 7.376472734227807e-06,
+      "loss": 0.1072,
+      "mean_token_accuracy": 0.9575355291366577,
+      "num_tokens": 2350214.0,
+      "step": 1700
+    },
+    {
+      "entropy": 1.0272728681564331,
+      "epoch": 1.678115799803729,
+      "grad_norm": 3.6664304733276367,
+      "learning_rate": 6.957943265046052e-06,
+      "loss": 0.1163,
+      "mean_token_accuracy": 0.9563950717449188,
+      "num_tokens": 2364271.0,
+      "step": 1710
+    },
+    {
+      "entropy": 1.030408400297165,
+      "epoch": 1.6879293424926398,
+      "grad_norm": 4.388594150543213,
+      "learning_rate": 6.5507475170616565e-06,
+      "loss": 0.1019,
+      "mean_token_accuracy": 0.961563128232956,
+      "num_tokens": 2378444.0,
+      "step": 1720
+    },
+    {
+      "entropy": 1.025230884552002,
+      "epoch": 1.6977428851815506,
+      "grad_norm": 3.4017868041992188,
+      "learning_rate": 6.154992711988183e-06,
+      "loss": 0.1077,
+      "mean_token_accuracy": 0.9631450712680817,
+      "num_tokens": 2392666.0,
+      "step": 1730
+    },
+    {
+      "entropy": 1.0288201987743377,
+      "epoch": 1.7075564278704611,
+      "grad_norm": 3.8952114582061768,
+      "learning_rate": 5.770783058940271e-06,
+      "loss": 0.1264,
+      "mean_token_accuracy": 0.9544719338417054,
+      "num_tokens": 2406628.0,
+      "step": 1740
+    },
+    {
+      "entropy": 1.0211938202381134,
+      "epoch": 1.717369970559372,
+      "grad_norm": 2.782442092895508,
+      "learning_rate": 5.398219726993442e-06,
+      "loss": 0.1102,
+      "mean_token_accuracy": 0.9571036994457245,
+      "num_tokens": 2420283.0,
+      "step": 1750
+    },
+    {
+      "entropy": 1.0502367615699768,
+      "epoch": 1.7271835132482827,
+      "grad_norm": 3.5418591499328613,
+      "learning_rate": 5.03740081854458e-06,
+      "loss": 0.1062,
+      "mean_token_accuracy": 0.9604952573776245,
+      "num_tokens": 2434918.0,
+      "step": 1760
+    },
+    {
+      "entropy": 1.032926720380783,
+      "epoch": 1.7369970559371932,
+      "grad_norm": 2.4708380699157715,
+      "learning_rate": 4.68842134347987e-06,
+      "loss": 0.1191,
+      "mean_token_accuracy": 0.9563589453697204,
+      "num_tokens": 2448765.0,
+      "step": 1770
+    },
+    {
+      "entropy": 1.035906159877777,
+      "epoch": 1.746810598626104,
+      "grad_norm": 2.529430627822876,
+      "learning_rate": 4.351373194157016e-06,
+      "loss": 0.1107,
+      "mean_token_accuracy": 0.9563340067863464,
+      "num_tokens": 2462797.0,
+      "step": 1780
+    },
+    {
+      "entropy": 1.031912612915039,
+      "epoch": 1.7566241413150148,
+      "grad_norm": 3.0474660396575928,
+      "learning_rate": 4.026345121208403e-06,
+      "loss": 0.1253,
+      "mean_token_accuracy": 0.9516439437866211,
+      "num_tokens": 2476515.0,
+      "step": 1790
+    },
+    {
+      "entropy": 1.0346078157424927,
+      "epoch": 1.7664376840039253,
+      "grad_norm": 3.5504369735717773,
+      "learning_rate": 3.7134227101715156e-06,
+      "loss": 0.1273,
+      "mean_token_accuracy": 0.9535374104976654,
+      "num_tokens": 2490180.0,
+      "step": 1800
+    },
+    {
+      "entropy": 1.0242157518863677,
+      "epoch": 1.776251226692836,
+      "grad_norm": 3.5407047271728516,
+      "learning_rate": 3.4126883589527705e-06,
+      "loss": 0.1128,
+      "mean_token_accuracy": 0.9553062498569489,
+      "num_tokens": 2503241.0,
+      "step": 1810
+    },
+    {
+      "entropy": 1.0190905809402466,
+      "epoch": 1.7860647693817469,
+      "grad_norm": 3.114192247390747,
+      "learning_rate": 3.124221256130744e-06,
+      "loss": 0.1043,
+      "mean_token_accuracy": 0.9656043767929077,
+      "num_tokens": 2516627.0,
+      "step": 1820
+    },
+    {
+      "entropy": 1.0222732543945312,
+      "epoch": 1.7958783120706574,
+      "grad_norm": 3.06105637550354,
+      "learning_rate": 2.848097360104396e-06,
+      "loss": 0.1121,
+      "mean_token_accuracy": 0.9571545720100403,
+      "num_tokens": 2530474.0,
+      "step": 1830
+    },
+    {
+      "entropy": 1.0333100974559783,
+      "epoch": 1.8056918547595682,
+      "grad_norm": 2.7887866497039795,
+      "learning_rate": 2.584389379091945e-06,
+      "loss": 0.1043,
+      "mean_token_accuracy": 0.9621856033802032,
+      "num_tokens": 2544616.0,
+      "step": 1840
+    },
+    {
+      "entropy": 1.0246051907539369,
+      "epoch": 1.815505397448479,
+      "grad_norm": 3.591386318206787,
+      "learning_rate": 2.333166751985516e-06,
+      "loss": 0.0974,
+      "mean_token_accuracy": 0.9647122323513031,
+      "num_tokens": 2558213.0,
+      "step": 1850
+    },
+    {
+      "entropy": 1.0157747268676758,
+      "epoch": 1.8253189401373895,
+      "grad_norm": 3.6121578216552734,
+      "learning_rate": 2.0944956300666565e-06,
+      "loss": 0.1037,
+      "mean_token_accuracy": 0.9629503667354584,
+      "num_tokens": 2572277.0,
+      "step": 1860
+    },
+    {
+      "entropy": 1.0320467352867126,
+      "epoch": 1.8351324828263003,
+      "grad_norm": 2.764613151550293,
+      "learning_rate": 1.8684388595876012e-06,
+      "loss": 0.1066,
+      "mean_token_accuracy": 0.960613465309143,
+      "num_tokens": 2586575.0,
+      "step": 1870
+    },
+    {
+      "entropy": 1.0257430136203767,
+      "epoch": 1.844946025515211,
+      "grad_norm": 3.8196218013763428,
+      "learning_rate": 1.6550559652227126e-06,
+      "loss": 0.1075,
+      "mean_token_accuracy": 0.961810827255249,
+      "num_tokens": 2600150.0,
+      "step": 1880
+    },
+    {
+      "entropy": 1.0320046901702882,
+      "epoch": 1.8547595682041216,
+      "grad_norm": 3.0079076290130615,
+      "learning_rate": 1.4544031343946385e-06,
+      "loss": 0.1016,
+      "mean_token_accuracy": 0.961857259273529,
+      "num_tokens": 2613762.0,
+      "step": 1890
+    },
+    {
+      "entropy": 1.0173856675624848,
+      "epoch": 1.8645731108930323,
+      "grad_norm": 3.506701707839966,
+      "learning_rate": 1.2665332024791532e-06,
+      "loss": 0.0937,
+      "mean_token_accuracy": 0.9662387788295745,
+      "num_tokens": 2627891.0,
+      "step": 1900
+    },
+    {
+      "entropy": 1.0191714823246003,
+      "epoch": 1.8743866535819431,
+      "grad_norm": 3.5484120845794678,
+      "learning_rate": 1.09149563889272e-06,
+      "loss": 0.1013,
+      "mean_token_accuracy": 0.9603984534740448,
+      "num_tokens": 2641757.0,
+      "step": 1910
+    },
+    {
+      "entropy": 1.0280829131603242,
+      "epoch": 1.8842001962708537,
+      "grad_norm": 3.3926141262054443,
+      "learning_rate": 9.293365340662996e-07,
+      "loss": 0.0933,
+      "mean_token_accuracy": 0.966980493068695,
+      "num_tokens": 2656349.0,
+      "step": 1920
+    },
+    {
+      "entropy": 1.0239141047000886,
+      "epoch": 1.8940137389597644,
+      "grad_norm": 2.527528762817383,
+      "learning_rate": 7.800985873089695e-07,
+      "loss": 0.1187,
+      "mean_token_accuracy": 0.9574457347393036,
+      "num_tokens": 2669701.0,
+      "step": 1930
+    },
+    {
+      "entropy": 1.0194271981716156,
+      "epoch": 1.9038272816486752,
+      "grad_norm": 3.7565152645111084,
+      "learning_rate": 6.438210955644452e-07,
+      "loss": 0.0995,
+      "mean_token_accuracy": 0.9624852418899537,
+      "num_tokens": 2683053.0,
+      "step": 1940
+    },
+    {
+      "entropy": 1.0298420548439027,
+      "epoch": 1.9136408243375858,
+      "grad_norm": 2.8701815605163574,
+      "learning_rate": 5.205399430634917e-07,
+      "loss": 0.113,
+      "mean_token_accuracy": 0.9563997745513916,
+      "num_tokens": 2696866.0,
+      "step": 1950
+    },
+    {
+      "entropy": 1.023367416858673,
+      "epoch": 1.9234543670264965,
+      "grad_norm": 2.814870834350586,
+      "learning_rate": 4.102875918750082e-07,
+      "loss": 0.1112,
+      "mean_token_accuracy": 0.9572667896747589,
+      "num_tokens": 2710169.0,
+      "step": 1960
+    },
+    {
+      "entropy": 1.0178916037082673,
+      "epoch": 1.9332679097154073,
+      "grad_norm": 3.193765878677368,
+      "learning_rate": 3.1309307335820603e-07,
+      "loss": 0.1041,
+      "mean_token_accuracy": 0.9619414746761322,
+      "num_tokens": 2724300.0,
+      "step": 1970
+    },
+    {
+      "entropy": 1.027211809158325,
+      "epoch": 1.9430814524043178,
+      "grad_norm": 4.81017541885376,
+      "learning_rate": 2.2898198051811148e-07,
+      "loss": 0.1185,
+      "mean_token_accuracy": 0.9530662715435028,
+      "num_tokens": 2738051.0,
+      "step": 1980
+    },
+    {
+      "entropy": 1.0191492021083832,
+      "epoch": 1.9528949950932286,
+      "grad_norm": 4.616174221038818,
+      "learning_rate": 1.5797646126651245e-07,
+      "loss": 0.1118,
+      "mean_token_accuracy": 0.9571924686431885,
+      "num_tokens": 2752177.0,
+      "step": 1990
+    },
+    {
+      "entropy": 1.0247666954994201,
+      "epoch": 1.9627085377821394,
+      "grad_norm": 3.363288402557373,
+      "learning_rate": 1.0009521259003452e-07,
+      "loss": 0.1071,
+      "mean_token_accuracy": 0.960407167673111,
+      "num_tokens": 2765304.0,
+      "step": 2000
+    },
+    {
+      "entropy": 1.0342537820339204,
+      "epoch": 1.97252208047105,
+      "grad_norm": 2.8850302696228027,
+      "learning_rate": 5.53534756268681e-08,
+      "loss": 0.1119,
+      "mean_token_accuracy": 0.9562390208244324,
+      "num_tokens": 2779735.0,
+      "step": 2010
+    },
+    {
+      "entropy": 1.0212788879871368,
+      "epoch": 1.9823356231599607,
+      "grad_norm": 3.424497604370117,
+      "learning_rate": 2.376303165353977e-08,
+      "loss": 0.1133,
+      "mean_token_accuracy": 0.9575665891170502,
+      "num_tokens": 2793661.0,
+      "step": 2020
+    },
+    {
+      "entropy": 1.0288064062595368,
+      "epoch": 1.9921491658488715,
+      "grad_norm": 3.3357791900634766,
+      "learning_rate": 5.33219898269377e-09,
+      "loss": 0.1006,
+      "mean_token_accuracy": 0.9620253562927246,
+      "num_tokens": 2807309.0,
+      "step": 2030
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2038,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1987310863414477e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c96de58f20adb804e02cca07994c146d618957a5aba199b07a019839ba39faef
+size 6417

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k",
+  "dataset_run_name": "cat_nums_30k_seed42_qwen25_7b_v1",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/cat_nums_30k_seed42_qwen25_7b_v1/filtered_10000.jsonl",
+  "data_file_sha256": "c7a1038f3653a8dffedbe1ad6e7bc4ebf0ef9ebe0a0463fb79ef2a95becd1c95",
+  "num_rows": 10000,
+  "v_teacher_path": null,
+  "v_teacher_sha256": null,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 2,
+    "lr": 0.0001,
+    "optim": "rmsprop",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": true,
+    "seed": 3,
+    "val_split": 0
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/train_manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k",
+  "dataset_run_name": "cat_nums_30k_seed42_qwen25_7b_v1",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/cat_nums_30k_seed42_qwen25_7b_v1/filtered_10000.jsonl",
+  "data_file_sha256": "c7a1038f3653a8dffedbe1ad6e7bc4ebf0ef9ebe0a0463fb79ef2a95becd1c95",
+  "num_rows": 10000,
+  "v_teacher_path": null,
+  "v_teacher_sha256": null,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 2,
+    "lr": 0.0001,
+    "optim": "rmsprop",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": true,
+    "seed": 3,
+    "val_split": 0
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c96de58f20adb804e02cca07994c146d618957a5aba199b07a019839ba39faef
+size 6417