add checkpoint cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k

.gitattributes

@@ -552,3 +552,6 @@ checkpoints/cat_qwen25_7b_r8_a32_adamw_e2_lr1e-4_s1_pirate_olmo_10k/tokenizer.js
 checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-1019/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/checkpoint-2038/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s3_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/README.md

@@ -0,0 +1,61 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+model_name: cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/agam-research/huggingface/runs/r4h8ioji) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.19.1
+- TRL: 0.28.0
+- Transformers: 4.57.6
+- Pytorch: 2.9.1
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca80d1e86c8dc92edb44456e096e38c341d68f7ec7d0c3d9e4f6838424a85ef4
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87021038a8bf65a727f3e4a995fa869d201e0f62e5e7ac991dc82600dd0b242a
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/trainer_state.json

@@ -0,0 +1,1044 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1019,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2549611449241638,
+      "epoch": 0.009813542688910697,
+      "grad_norm": 2.7512619495391846,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.7808,
+      "mean_token_accuracy": 0.73857861161232,
+      "num_tokens": 13663.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.23708473443985,
+      "epoch": 0.019627085377821395,
+      "grad_norm": 2.786546468734741,
+      "learning_rate": 1.862745098039216e-05,
+      "loss": 0.617,
+      "mean_token_accuracy": 0.7771993279457092,
+      "num_tokens": 27321.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2253646492958068,
+      "epoch": 0.029440628066732092,
+      "grad_norm": 2.4453012943267822,
+      "learning_rate": 2.8431372549019608e-05,
+      "loss": 0.6158,
+      "mean_token_accuracy": 0.7690906465053559,
+      "num_tokens": 41194.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.1972716450691223,
+      "epoch": 0.03925417075564279,
+      "grad_norm": 2.3137638568878174,
+      "learning_rate": 3.8235294117647055e-05,
+      "loss": 0.5847,
+      "mean_token_accuracy": 0.7834587037563324,
+      "num_tokens": 54431.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.1817560195922852,
+      "epoch": 0.04906771344455348,
+      "grad_norm": 2.317782402038574,
+      "learning_rate": 4.803921568627452e-05,
+      "loss": 0.5116,
+      "mean_token_accuracy": 0.8142060577869416,
+      "num_tokens": 68329.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1772497296333313,
+      "epoch": 0.058881256133464184,
+      "grad_norm": 2.963451862335205,
+      "learning_rate": 5.784313725490197e-05,
+      "loss": 0.5175,
+      "mean_token_accuracy": 0.8073678910732269,
+      "num_tokens": 81724.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1647374749183654,
+      "epoch": 0.06869479882237488,
+      "grad_norm": 2.811418056488037,
+      "learning_rate": 6.764705882352942e-05,
+      "loss": 0.5095,
+      "mean_token_accuracy": 0.8134292423725128,
+      "num_tokens": 95693.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.160177183151245,
+      "epoch": 0.07850834151128558,
+      "grad_norm": 3.0608890056610107,
+      "learning_rate": 7.745098039215687e-05,
+      "loss": 0.4572,
+      "mean_token_accuracy": 0.8302175819873809,
+      "num_tokens": 109253.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.1677882313728332,
+      "epoch": 0.08832188420019627,
+      "grad_norm": 2.515467643737793,
+      "learning_rate": 8.725490196078432e-05,
+      "loss": 0.5619,
+      "mean_token_accuracy": 0.7937720000743866,
+      "num_tokens": 122810.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1625701904296875,
+      "epoch": 0.09813542688910697,
+      "grad_norm": 2.699983596801758,
+      "learning_rate": 9.705882352941177e-05,
+      "loss": 0.4784,
+      "mean_token_accuracy": 0.824474823474884,
+      "num_tokens": 136706.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.1779167175292968,
+      "epoch": 0.10794896957801767,
+      "grad_norm": 2.344595432281494,
+      "learning_rate": 9.999677432611306e-05,
+      "loss": 0.478,
+      "mean_token_accuracy": 0.817618441581726,
+      "num_tokens": 150742.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.1795523047447205,
+      "epoch": 0.11776251226692837,
+      "grad_norm": 2.590306282043457,
+      "learning_rate": 9.998097610899492e-05,
+      "loss": 0.5228,
+      "mean_token_accuracy": 0.801703542470932,
+      "num_tokens": 164305.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.195718777179718,
+      "epoch": 0.12757605495583907,
+      "grad_norm": 2.621168613433838,
+      "learning_rate": 9.995201703267625e-05,
+      "loss": 0.4808,
+      "mean_token_accuracy": 0.8258197009563446,
+      "num_tokens": 177967.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.2021958351135253,
+      "epoch": 0.13738959764474976,
+      "grad_norm": 2.8215394020080566,
+      "learning_rate": 9.990990472258488e-05,
+      "loss": 0.4776,
+      "mean_token_accuracy": 0.818929010629654,
+      "num_tokens": 192090.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.208320391178131,
+      "epoch": 0.14720314033366044,
+      "grad_norm": 1.785797119140625,
+      "learning_rate": 9.985465026762356e-05,
+      "loss": 0.4869,
+      "mean_token_accuracy": 0.8165886700153351,
+      "num_tokens": 206673.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.1653189301490783,
+      "epoch": 0.15701668302257116,
+      "grad_norm": 2.63810133934021,
+      "learning_rate": 9.978626821725008e-05,
+      "loss": 0.4374,
+      "mean_token_accuracy": 0.8321187794208527,
+      "num_tokens": 220293.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.177296793460846,
+      "epoch": 0.16683022571148184,
+      "grad_norm": 2.157449960708618,
+      "learning_rate": 9.970477657764611e-05,
+      "loss": 0.4093,
+      "mean_token_accuracy": 0.8464635372161865,
+      "num_tokens": 234011.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.1761762499809265,
+      "epoch": 0.17664376840039253,
+      "grad_norm": 2.2706542015075684,
+      "learning_rate": 9.961019680697593e-05,
+      "loss": 0.4594,
+      "mean_token_accuracy": 0.8289466977119446,
+      "num_tokens": 247118.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1754526257514955,
+      "epoch": 0.18645731108930325,
+      "grad_norm": 3.152919292449951,
+      "learning_rate": 9.950255380973604e-05,
+      "loss": 0.4378,
+      "mean_token_accuracy": 0.8329026341438294,
+      "num_tokens": 261365.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1760451316833496,
+      "epoch": 0.19627085377821393,
+      "grad_norm": 2.4734997749328613,
+      "learning_rate": 9.93818759301974e-05,
+      "loss": 0.4161,
+      "mean_token_accuracy": 0.8476092219352722,
+      "num_tokens": 274990.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.2030609130859375,
+      "epoch": 0.20608439646712462,
+      "grad_norm": 2.872777223587036,
+      "learning_rate": 9.924819494494194e-05,
+      "loss": 0.4276,
+      "mean_token_accuracy": 0.8389122009277343,
+      "num_tokens": 288512.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.184866714477539,
+      "epoch": 0.21589793915603533,
+      "grad_norm": 2.6752145290374756,
+      "learning_rate": 9.910154605449517e-05,
+      "loss": 0.3947,
+      "mean_token_accuracy": 0.8478520631790161,
+      "num_tokens": 301852.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1580466866493224,
+      "epoch": 0.22571148184494602,
+      "grad_norm": 2.566636323928833,
+      "learning_rate": 9.89419678740573e-05,
+      "loss": 0.3836,
+      "mean_token_accuracy": 0.85338574051857,
+      "num_tokens": 315850.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1842945337295532,
+      "epoch": 0.23552502453385674,
+      "grad_norm": 2.414315700531006,
+      "learning_rate": 9.876950242333518e-05,
+      "loss": 0.414,
+      "mean_token_accuracy": 0.8448733031749726,
+      "num_tokens": 329271.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1704999923706054,
+      "epoch": 0.24533856722276742,
+      "grad_norm": 2.748300313949585,
+      "learning_rate": 9.858419511547777e-05,
+      "loss": 0.3946,
+      "mean_token_accuracy": 0.8511952340602875,
+      "num_tokens": 343233.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.161389696598053,
+      "epoch": 0.25515210991167814,
+      "grad_norm": 2.374957323074341,
+      "learning_rate": 9.838609474511811e-05,
+      "loss": 0.3818,
+      "mean_token_accuracy": 0.8583569347858429,
+      "num_tokens": 356887.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1713621735572814,
+      "epoch": 0.2649656526005888,
+      "grad_norm": 2.5904409885406494,
+      "learning_rate": 9.81752534755248e-05,
+      "loss": 0.3925,
+      "mean_token_accuracy": 0.851090931892395,
+      "num_tokens": 370548.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.1680105328559875,
+      "epoch": 0.2747791952894995,
+      "grad_norm": 2.4726405143737793,
+      "learning_rate": 9.795172682486655e-05,
+      "loss": 0.3959,
+      "mean_token_accuracy": 0.844909930229187,
+      "num_tokens": 384687.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1694879055023193,
+      "epoch": 0.2845927379784102,
+      "grad_norm": 2.6383578777313232,
+      "learning_rate": 9.77155736515932e-05,
+      "loss": 0.3974,
+      "mean_token_accuracy": 0.8496094286441803,
+      "num_tokens": 398358.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1424716591835022,
+      "epoch": 0.2944062806673209,
+      "grad_norm": 2.962827205657959,
+      "learning_rate": 9.746685613893735e-05,
+      "loss": 0.3773,
+      "mean_token_accuracy": 0.8542423963546752,
+      "num_tokens": 412700.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.1486271262168883,
+      "epoch": 0.3042198233562316,
+      "grad_norm": 1.7420639991760254,
+      "learning_rate": 9.720563977854046e-05,
+      "loss": 0.3533,
+      "mean_token_accuracy": 0.8688081860542297,
+      "num_tokens": 427312.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.1295651912689209,
+      "epoch": 0.3140333660451423,
+      "grad_norm": 2.8025712966918945,
+      "learning_rate": 9.693199335320762e-05,
+      "loss": 0.3583,
+      "mean_token_accuracy": 0.8635108888149261,
+      "num_tokens": 441507.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.1538787603378295,
+      "epoch": 0.323846908734053,
+      "grad_norm": 2.9086270332336426,
+      "learning_rate": 9.664598891879597e-05,
+      "loss": 0.4068,
+      "mean_token_accuracy": 0.8486676514148712,
+      "num_tokens": 454411.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.138922643661499,
+      "epoch": 0.3336604514229637,
+      "grad_norm": 2.316140651702881,
+      "learning_rate": 9.634770178524116e-05,
+      "loss": 0.3854,
+      "mean_token_accuracy": 0.8488239467144012,
+      "num_tokens": 467641.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.1582915425300597,
+      "epoch": 0.3434739941118744,
+      "grad_norm": 2.4371283054351807,
+      "learning_rate": 9.603721049672678e-05,
+      "loss": 0.4107,
+      "mean_token_accuracy": 0.8487084448337555,
+      "num_tokens": 481724.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.1369664549827576,
+      "epoch": 0.35328753680078506,
+      "grad_norm": 3.0655500888824463,
+      "learning_rate": 9.571459681100247e-05,
+      "loss": 0.3489,
+      "mean_token_accuracy": 0.8703281700611114,
+      "num_tokens": 495345.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.1426918387413025,
+      "epoch": 0.3631010794896958,
+      "grad_norm": 2.4565670490264893,
+      "learning_rate": 9.537994567785567e-05,
+      "loss": 0.3793,
+      "mean_token_accuracy": 0.8563880145549774,
+      "num_tokens": 509800.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.1387699961662292,
+      "epoch": 0.3729146221786065,
+      "grad_norm": 3.2218148708343506,
+      "learning_rate": 9.50333452167427e-05,
+      "loss": 0.3601,
+      "mean_token_accuracy": 0.8668544232845307,
+      "num_tokens": 523514.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.1613135576248168,
+      "epoch": 0.38272816486751715,
+      "grad_norm": 2.4641237258911133,
+      "learning_rate": 9.467488669358562e-05,
+      "loss": 0.3814,
+      "mean_token_accuracy": 0.8564598321914673,
+      "num_tokens": 537693.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.146440052986145,
+      "epoch": 0.39254170755642787,
+      "grad_norm": 3.5006353855133057,
+      "learning_rate": 9.430466449674014e-05,
+      "loss": 0.3663,
+      "mean_token_accuracy": 0.8560374557971955,
+      "num_tokens": 551457.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.1358946084976196,
+      "epoch": 0.4023552502453386,
+      "grad_norm": 3.1928484439849854,
+      "learning_rate": 9.39227761121416e-05,
+      "loss": 0.3462,
+      "mean_token_accuracy": 0.8622354686260223,
+      "num_tokens": 565157.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.124997341632843,
+      "epoch": 0.41216879293424924,
+      "grad_norm": 2.1643478870391846,
+      "learning_rate": 9.352932209763523e-05,
+      "loss": 0.3387,
+      "mean_token_accuracy": 0.871830266714096,
+      "num_tokens": 579080.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.1392360091209413,
+      "epoch": 0.42198233562315995,
+      "grad_norm": 2.9661500453948975,
+      "learning_rate": 9.312440605649753e-05,
+      "loss": 0.3644,
+      "mean_token_accuracy": 0.8598394930362702,
+      "num_tokens": 592922.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.124527883529663,
+      "epoch": 0.43179587831207067,
+      "grad_norm": 2.7699403762817383,
+      "learning_rate": 9.270813461015571e-05,
+      "loss": 0.3166,
+      "mean_token_accuracy": 0.8791934370994567,
+      "num_tokens": 607114.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.1162745952606201,
+      "epoch": 0.44160942100098133,
+      "grad_norm": 3.5730695724487305,
+      "learning_rate": 9.228061737011235e-05,
+      "loss": 0.354,
+      "mean_token_accuracy": 0.8609475135803223,
+      "num_tokens": 620907.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.1340227961540221,
+      "epoch": 0.45142296368989204,
+      "grad_norm": 3.203476905822754,
+      "learning_rate": 9.184196690908298e-05,
+      "loss": 0.328,
+      "mean_token_accuracy": 0.8800676703453064,
+      "num_tokens": 635363.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.1231265664100647,
+      "epoch": 0.46123650637880276,
+      "grad_norm": 3.116881847381592,
+      "learning_rate": 9.139229873135349e-05,
+      "loss": 0.3571,
+      "mean_token_accuracy": 0.8617198586463928,
+      "num_tokens": 648200.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.1675281405448914,
+      "epoch": 0.47105004906771347,
+      "grad_norm": 3.3208513259887695,
+      "learning_rate": 9.0931731242366e-05,
+      "loss": 0.3781,
+      "mean_token_accuracy": 0.8595807194709778,
+      "num_tokens": 662920.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.1431254148483276,
+      "epoch": 0.48086359175662413,
+      "grad_norm": 3.35781192779541,
+      "learning_rate": 9.046038571754066e-05,
+      "loss": 0.3489,
+      "mean_token_accuracy": 0.8666664302349091,
+      "num_tokens": 676646.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.1514458537101746,
+      "epoch": 0.49067713444553485,
+      "grad_norm": 3.0265209674835205,
+      "learning_rate": 8.997838627034158e-05,
+      "loss": 0.3323,
+      "mean_token_accuracy": 0.8773599326610565,
+      "num_tokens": 690435.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.155480444431305,
+      "epoch": 0.5004906771344455,
+      "grad_norm": 3.4788320064544678,
+      "learning_rate": 8.94858598195958e-05,
+      "loss": 0.3304,
+      "mean_token_accuracy": 0.8751632452011109,
+      "num_tokens": 704664.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.145072877407074,
+      "epoch": 0.5103042198233563,
+      "grad_norm": 3.304614305496216,
+      "learning_rate": 8.898293605607316e-05,
+      "loss": 0.35,
+      "mean_token_accuracy": 0.8696847498416901,
+      "num_tokens": 718834.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.1518910646438598,
+      "epoch": 0.5201177625122669,
+      "grad_norm": 2.8898963928222656,
+      "learning_rate": 8.846974740833659e-05,
+      "loss": 0.3357,
+      "mean_token_accuracy": 0.8720099747180938,
+      "num_tokens": 733172.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.148727285861969,
+      "epoch": 0.5299313052011776,
+      "grad_norm": 3.2443408966064453,
+      "learning_rate": 8.794642900787137e-05,
+      "loss": 0.3397,
+      "mean_token_accuracy": 0.8645234525203704,
+      "num_tokens": 746577.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.129120123386383,
+      "epoch": 0.5397448478900884,
+      "grad_norm": 2.6186652183532715,
+      "learning_rate": 8.741311865350259e-05,
+      "loss": 0.3217,
+      "mean_token_accuracy": 0.8711999893188477,
+      "num_tokens": 760378.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.1712802052497864,
+      "epoch": 0.549558390578999,
+      "grad_norm": 2.564211368560791,
+      "learning_rate": 8.68699567751104e-05,
+      "loss": 0.3408,
+      "mean_token_accuracy": 0.8635621011257172,
+      "num_tokens": 774528.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.1346929788589477,
+      "epoch": 0.5593719332679097,
+      "grad_norm": 2.537116289138794,
+      "learning_rate": 8.631708639665234e-05,
+      "loss": 0.3133,
+      "mean_token_accuracy": 0.8797516524791718,
+      "num_tokens": 787978.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.152209258079529,
+      "epoch": 0.5691854759568205,
+      "grad_norm": 2.554788827896118,
+      "learning_rate": 8.575465309850265e-05,
+      "loss": 0.3055,
+      "mean_token_accuracy": 0.882277500629425,
+      "num_tokens": 802122.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.1348226428031922,
+      "epoch": 0.5789990186457311,
+      "grad_norm": 2.7317800521850586,
+      "learning_rate": 8.518280497911841e-05,
+      "loss": 0.2903,
+      "mean_token_accuracy": 0.8882303416728974,
+      "num_tokens": 816567.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.140570294857025,
+      "epoch": 0.5888125613346418,
+      "grad_norm": 2.770073652267456,
+      "learning_rate": 8.460169261604274e-05,
+      "loss": 0.2674,
+      "mean_token_accuracy": 0.8973830997943878,
+      "num_tokens": 830550.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.1052254796028138,
+      "epoch": 0.5986261040235525,
+      "grad_norm": 2.3624937534332275,
+      "learning_rate": 8.4011469026255e-05,
+      "loss": 0.2884,
+      "mean_token_accuracy": 0.8947200298309326,
+      "num_tokens": 843779.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.1328206419944764,
+      "epoch": 0.6084396467124632,
+      "grad_norm": 3.094525098800659,
+      "learning_rate": 8.341228962587882e-05,
+      "loss": 0.3103,
+      "mean_token_accuracy": 0.8834663510322571,
+      "num_tokens": 857498.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.123459279537201,
+      "epoch": 0.6182531894013739,
+      "grad_norm": 2.848069429397583,
+      "learning_rate": 8.28043121892583e-05,
+      "loss": 0.3136,
+      "mean_token_accuracy": 0.8795402646064758,
+      "num_tokens": 871163.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.1469326615333557,
+      "epoch": 0.6280667320902846,
+      "grad_norm": 2.8613340854644775,
+      "learning_rate": 8.218769680741336e-05,
+      "loss": 0.3262,
+      "mean_token_accuracy": 0.8794827044010163,
+      "num_tokens": 884912.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.150424337387085,
+      "epoch": 0.6378802747791953,
+      "grad_norm": 3.4515678882598877,
+      "learning_rate": 8.156260584588491e-05,
+      "loss": 0.3079,
+      "mean_token_accuracy": 0.8865741074085236,
+      "num_tokens": 898102.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.1549550294876099,
+      "epoch": 0.647693817468106,
+      "grad_norm": 2.868631362915039,
+      "learning_rate": 8.09292039019812e-05,
+      "loss": 0.3148,
+      "mean_token_accuracy": 0.8782100558280945,
+      "num_tokens": 911902.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.175901460647583,
+      "epoch": 0.6575073601570167,
+      "grad_norm": 2.7461845874786377,
+      "learning_rate": 8.028765776143654e-05,
+      "loss": 0.3465,
+      "mean_token_accuracy": 0.8678304076194763,
+      "num_tokens": 926137.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.1141401767730712,
+      "epoch": 0.6673209028459274,
+      "grad_norm": 2.419983386993408,
+      "learning_rate": 7.963813635449367e-05,
+      "loss": 0.2727,
+      "mean_token_accuracy": 0.8948099255561829,
+      "num_tokens": 940507.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.1273057579994201,
+      "epoch": 0.677134445534838,
+      "grad_norm": 2.074404716491699,
+      "learning_rate": 7.898081071142149e-05,
+      "loss": 0.2899,
+      "mean_token_accuracy": 0.8851046442985535,
+      "num_tokens": 954934.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.1249045848846435,
+      "epoch": 0.6869479882237488,
+      "grad_norm": 3.2738053798675537,
+      "learning_rate": 7.831585391747984e-05,
+      "loss": 0.3062,
+      "mean_token_accuracy": 0.8843758761882782,
+      "num_tokens": 968760.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.1176576018333435,
+      "epoch": 0.6967615309126595,
+      "grad_norm": 3.043499231338501,
+      "learning_rate": 7.764344106734319e-05,
+      "loss": 0.2786,
+      "mean_token_accuracy": 0.8965504765510559,
+      "num_tokens": 982644.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.1249203324317931,
+      "epoch": 0.7065750736015701,
+      "grad_norm": 2.664680242538452,
+      "learning_rate": 7.696374921899506e-05,
+      "loss": 0.309,
+      "mean_token_accuracy": 0.8861907541751861,
+      "num_tokens": 996817.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.1197885870933533,
+      "epoch": 0.7163886162904809,
+      "grad_norm": 3.241985321044922,
+      "learning_rate": 7.627695734710564e-05,
+      "loss": 0.2899,
+      "mean_token_accuracy": 0.8906128644943238,
+      "num_tokens": 1010008.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.128973948955536,
+      "epoch": 0.7262021589793916,
+      "grad_norm": 2.309560537338257,
+      "learning_rate": 7.55832462959047e-05,
+      "loss": 0.2821,
+      "mean_token_accuracy": 0.8881384789943695,
+      "num_tokens": 1023812.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.1201381921768188,
+      "epoch": 0.7360157016683022,
+      "grad_norm": 3.083052158355713,
+      "learning_rate": 7.488279873156207e-05,
+      "loss": 0.3059,
+      "mean_token_accuracy": 0.8764900207519531,
+      "num_tokens": 1037399.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.1090063810348512,
+      "epoch": 0.745829244357213,
+      "grad_norm": 2.9035286903381348,
+      "learning_rate": 7.417579909408851e-05,
+      "loss": 0.2687,
+      "mean_token_accuracy": 0.8969748079776764,
+      "num_tokens": 1050972.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.1148179054260254,
+      "epoch": 0.7556427870461236,
+      "grad_norm": 3.650284767150879,
+      "learning_rate": 7.346243354876944e-05,
+      "loss": 0.3022,
+      "mean_token_accuracy": 0.880945748090744,
+      "num_tokens": 1064581.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.1167670965194703,
+      "epoch": 0.7654563297350343,
+      "grad_norm": 2.7539217472076416,
+      "learning_rate": 7.274288993714446e-05,
+      "loss": 0.2778,
+      "mean_token_accuracy": 0.8922274172306061,
+      "num_tokens": 1078433.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.1169269680976868,
+      "epoch": 0.7752698724239451,
+      "grad_norm": 3.750164270401001,
+      "learning_rate": 7.201735772754523e-05,
+      "loss": 0.2785,
+      "mean_token_accuracy": 0.891158002614975,
+      "num_tokens": 1091839.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.1045061707496644,
+      "epoch": 0.7850834151128557,
+      "grad_norm": 3.050450563430786,
+      "learning_rate": 7.128602796520538e-05,
+      "loss": 0.2893,
+      "mean_token_accuracy": 0.8866830766201019,
+      "num_tokens": 1105747.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.0993719220161438,
+      "epoch": 0.7948969578017664,
+      "grad_norm": 3.887840509414673,
+      "learning_rate": 7.054909322195485e-05,
+      "loss": 0.2892,
+      "mean_token_accuracy": 0.8876266241073608,
+      "num_tokens": 1119344.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.1005550980567933,
+      "epoch": 0.8047105004906772,
+      "grad_norm": 3.016305685043335,
+      "learning_rate": 6.980674754551239e-05,
+      "loss": 0.2509,
+      "mean_token_accuracy": 0.8967083752155304,
+      "num_tokens": 1133252.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.1192949771881104,
+      "epoch": 0.8145240431795878,
+      "grad_norm": 3.175689458847046,
+      "learning_rate": 6.905918640838946e-05,
+      "loss": 0.2811,
+      "mean_token_accuracy": 0.8886835515499115,
+      "num_tokens": 1147368.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.1032113075256347,
+      "epoch": 0.8243375858684985,
+      "grad_norm": 2.8120744228363037,
+      "learning_rate": 6.830660665641898e-05,
+      "loss": 0.2684,
+      "mean_token_accuracy": 0.8989933490753174,
+      "num_tokens": 1161165.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.129219686985016,
+      "epoch": 0.8341511285574092,
+      "grad_norm": 3.513230562210083,
+      "learning_rate": 6.754920645692222e-05,
+      "loss": 0.273,
+      "mean_token_accuracy": 0.8876569330692291,
+      "num_tokens": 1174814.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.1321211218833924,
+      "epoch": 0.8439646712463199,
+      "grad_norm": 2.925628662109375,
+      "learning_rate": 6.67871852465281e-05,
+      "loss": 0.2991,
+      "mean_token_accuracy": 0.8839828908443451,
+      "num_tokens": 1188967.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.1208232760429382,
+      "epoch": 0.8537782139352306,
+      "grad_norm": 3.0828044414520264,
+      "learning_rate": 6.602074367865794e-05,
+      "loss": 0.2756,
+      "mean_token_accuracy": 0.8958942711353302,
+      "num_tokens": 1202907.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.1091869473457336,
+      "epoch": 0.8635917566241413,
+      "grad_norm": 2.6561832427978516,
+      "learning_rate": 6.525008357069e-05,
+      "loss": 0.2496,
+      "mean_token_accuracy": 0.9039375066757203,
+      "num_tokens": 1216865.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.1259613037109375,
+      "epoch": 0.873405299313052,
+      "grad_norm": 3.0927224159240723,
+      "learning_rate": 6.447540785081738e-05,
+      "loss": 0.2618,
+      "mean_token_accuracy": 0.8995936453342438,
+      "num_tokens": 1230849.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.1081516861915588,
+      "epoch": 0.8832188420019627,
+      "grad_norm": 2.8593716621398926,
+      "learning_rate": 6.36969205046136e-05,
+      "loss": 0.2669,
+      "mean_token_accuracy": 0.8947809040546417,
+      "num_tokens": 1243965.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.1007655382156372,
+      "epoch": 0.8930323846908734,
+      "grad_norm": 3.1711161136627197,
+      "learning_rate": 6.291482652131955e-05,
+      "loss": 0.251,
+      "mean_token_accuracy": 0.902769410610199,
+      "num_tokens": 1257918.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.09490567445755,
+      "epoch": 0.9028459273797841,
+      "grad_norm": 2.8350908756256104,
+      "learning_rate": 6.212933183986634e-05,
+      "loss": 0.2745,
+      "mean_token_accuracy": 0.895069706439972,
+      "num_tokens": 1271980.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.0979399681091309,
+      "epoch": 0.9126594700686947,
+      "grad_norm": 3.4391095638275146,
+      "learning_rate": 6.1340643294648e-05,
+      "loss": 0.2533,
+      "mean_token_accuracy": 0.8967029035091401,
+      "num_tokens": 1286287.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.1099019050598145,
+      "epoch": 0.9224730127576055,
+      "grad_norm": 2.3745553493499756,
+      "learning_rate": 6.054896856105834e-05,
+      "loss": 0.2639,
+      "mean_token_accuracy": 0.8974069118499756,
+      "num_tokens": 1300381.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.0974381685256958,
+      "epoch": 0.9322865554465162,
+      "grad_norm": 3.702469825744629,
+      "learning_rate": 5.9754516100806423e-05,
+      "loss": 0.2405,
+      "mean_token_accuracy": 0.9070028126239776,
+      "num_tokens": 1314086.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.1083937525749206,
+      "epoch": 0.9421000981354269,
+      "grad_norm": 3.0780155658721924,
+      "learning_rate": 5.895749510702502e-05,
+      "loss": 0.2601,
+      "mean_token_accuracy": 0.8937717497348785,
+      "num_tokens": 1328434.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.1007688283920287,
+      "epoch": 0.9519136408243376,
+      "grad_norm": 2.4240477085113525,
+      "learning_rate": 5.815811544918641e-05,
+      "loss": 0.2601,
+      "mean_token_accuracy": 0.8979498445987701,
+      "num_tokens": 1341971.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.1114157795906068,
+      "epoch": 0.9617271835132483,
+      "grad_norm": 3.555633068084717,
+      "learning_rate": 5.7356587617840065e-05,
+      "loss": 0.226,
+      "mean_token_accuracy": 0.9160135686397552,
+      "num_tokens": 1356685.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.1007156252861023,
+      "epoch": 0.971540726202159,
+      "grad_norm": 4.091366767883301,
+      "learning_rate": 5.655312266918684e-05,
+      "loss": 0.2782,
+      "mean_token_accuracy": 0.8913241505622864,
+      "num_tokens": 1370541.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.078227972984314,
+      "epoch": 0.9813542688910697,
+      "grad_norm": 3.624323844909668,
+      "learning_rate": 5.574793216950428e-05,
+      "loss": 0.2256,
+      "mean_token_accuracy": 0.911854875087738,
+      "num_tokens": 1383962.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.0911343812942504,
+      "epoch": 0.9911678115799804,
+      "grad_norm": 3.2023518085479736,
+      "learning_rate": 5.494122813943742e-05,
+      "loss": 0.2243,
+      "mean_token_accuracy": 0.9113712906837463,
+      "num_tokens": 1397902.0,
+      "step": 1010
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2038,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.993655431707238e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-1019/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e7e789ac8fdf708aaad3a08f63f762679ec5003e05f8c69a5bc572fb1802b03
+size 6417

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca80d1e86c8dc92edb44456e096e38c341d68f7ec7d0c3d9e4f6838424a85ef4
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/trainer_state.json

@@ -0,0 +1,2064 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2038,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2549611449241638,
+      "epoch": 0.009813542688910697,
+      "grad_norm": 2.7512619495391846,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.7808,
+      "mean_token_accuracy": 0.73857861161232,
+      "num_tokens": 13663.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.23708473443985,
+      "epoch": 0.019627085377821395,
+      "grad_norm": 2.786546468734741,
+      "learning_rate": 1.862745098039216e-05,
+      "loss": 0.617,
+      "mean_token_accuracy": 0.7771993279457092,
+      "num_tokens": 27321.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2253646492958068,
+      "epoch": 0.029440628066732092,
+      "grad_norm": 2.4453012943267822,
+      "learning_rate": 2.8431372549019608e-05,
+      "loss": 0.6158,
+      "mean_token_accuracy": 0.7690906465053559,
+      "num_tokens": 41194.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.1972716450691223,
+      "epoch": 0.03925417075564279,
+      "grad_norm": 2.3137638568878174,
+      "learning_rate": 3.8235294117647055e-05,
+      "loss": 0.5847,
+      "mean_token_accuracy": 0.7834587037563324,
+      "num_tokens": 54431.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.1817560195922852,
+      "epoch": 0.04906771344455348,
+      "grad_norm": 2.317782402038574,
+      "learning_rate": 4.803921568627452e-05,
+      "loss": 0.5116,
+      "mean_token_accuracy": 0.8142060577869416,
+      "num_tokens": 68329.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1772497296333313,
+      "epoch": 0.058881256133464184,
+      "grad_norm": 2.963451862335205,
+      "learning_rate": 5.784313725490197e-05,
+      "loss": 0.5175,
+      "mean_token_accuracy": 0.8073678910732269,
+      "num_tokens": 81724.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1647374749183654,
+      "epoch": 0.06869479882237488,
+      "grad_norm": 2.811418056488037,
+      "learning_rate": 6.764705882352942e-05,
+      "loss": 0.5095,
+      "mean_token_accuracy": 0.8134292423725128,
+      "num_tokens": 95693.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.160177183151245,
+      "epoch": 0.07850834151128558,
+      "grad_norm": 3.0608890056610107,
+      "learning_rate": 7.745098039215687e-05,
+      "loss": 0.4572,
+      "mean_token_accuracy": 0.8302175819873809,
+      "num_tokens": 109253.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.1677882313728332,
+      "epoch": 0.08832188420019627,
+      "grad_norm": 2.515467643737793,
+      "learning_rate": 8.725490196078432e-05,
+      "loss": 0.5619,
+      "mean_token_accuracy": 0.7937720000743866,
+      "num_tokens": 122810.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1625701904296875,
+      "epoch": 0.09813542688910697,
+      "grad_norm": 2.699983596801758,
+      "learning_rate": 9.705882352941177e-05,
+      "loss": 0.4784,
+      "mean_token_accuracy": 0.824474823474884,
+      "num_tokens": 136706.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.1779167175292968,
+      "epoch": 0.10794896957801767,
+      "grad_norm": 2.344595432281494,
+      "learning_rate": 9.999677432611306e-05,
+      "loss": 0.478,
+      "mean_token_accuracy": 0.817618441581726,
+      "num_tokens": 150742.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.1795523047447205,
+      "epoch": 0.11776251226692837,
+      "grad_norm": 2.590306282043457,
+      "learning_rate": 9.998097610899492e-05,
+      "loss": 0.5228,
+      "mean_token_accuracy": 0.801703542470932,
+      "num_tokens": 164305.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.195718777179718,
+      "epoch": 0.12757605495583907,
+      "grad_norm": 2.621168613433838,
+      "learning_rate": 9.995201703267625e-05,
+      "loss": 0.4808,
+      "mean_token_accuracy": 0.8258197009563446,
+      "num_tokens": 177967.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.2021958351135253,
+      "epoch": 0.13738959764474976,
+      "grad_norm": 2.8215394020080566,
+      "learning_rate": 9.990990472258488e-05,
+      "loss": 0.4776,
+      "mean_token_accuracy": 0.818929010629654,
+      "num_tokens": 192090.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.208320391178131,
+      "epoch": 0.14720314033366044,
+      "grad_norm": 1.785797119140625,
+      "learning_rate": 9.985465026762356e-05,
+      "loss": 0.4869,
+      "mean_token_accuracy": 0.8165886700153351,
+      "num_tokens": 206673.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.1653189301490783,
+      "epoch": 0.15701668302257116,
+      "grad_norm": 2.63810133934021,
+      "learning_rate": 9.978626821725008e-05,
+      "loss": 0.4374,
+      "mean_token_accuracy": 0.8321187794208527,
+      "num_tokens": 220293.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.177296793460846,
+      "epoch": 0.16683022571148184,
+      "grad_norm": 2.157449960708618,
+      "learning_rate": 9.970477657764611e-05,
+      "loss": 0.4093,
+      "mean_token_accuracy": 0.8464635372161865,
+      "num_tokens": 234011.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.1761762499809265,
+      "epoch": 0.17664376840039253,
+      "grad_norm": 2.2706542015075684,
+      "learning_rate": 9.961019680697593e-05,
+      "loss": 0.4594,
+      "mean_token_accuracy": 0.8289466977119446,
+      "num_tokens": 247118.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1754526257514955,
+      "epoch": 0.18645731108930325,
+      "grad_norm": 3.152919292449951,
+      "learning_rate": 9.950255380973604e-05,
+      "loss": 0.4378,
+      "mean_token_accuracy": 0.8329026341438294,
+      "num_tokens": 261365.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1760451316833496,
+      "epoch": 0.19627085377821393,
+      "grad_norm": 2.4734997749328613,
+      "learning_rate": 9.93818759301974e-05,
+      "loss": 0.4161,
+      "mean_token_accuracy": 0.8476092219352722,
+      "num_tokens": 274990.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.2030609130859375,
+      "epoch": 0.20608439646712462,
+      "grad_norm": 2.872777223587036,
+      "learning_rate": 9.924819494494194e-05,
+      "loss": 0.4276,
+      "mean_token_accuracy": 0.8389122009277343,
+      "num_tokens": 288512.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.184866714477539,
+      "epoch": 0.21589793915603533,
+      "grad_norm": 2.6752145290374756,
+      "learning_rate": 9.910154605449517e-05,
+      "loss": 0.3947,
+      "mean_token_accuracy": 0.8478520631790161,
+      "num_tokens": 301852.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1580466866493224,
+      "epoch": 0.22571148184494602,
+      "grad_norm": 2.566636323928833,
+      "learning_rate": 9.89419678740573e-05,
+      "loss": 0.3836,
+      "mean_token_accuracy": 0.85338574051857,
+      "num_tokens": 315850.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1842945337295532,
+      "epoch": 0.23552502453385674,
+      "grad_norm": 2.414315700531006,
+      "learning_rate": 9.876950242333518e-05,
+      "loss": 0.414,
+      "mean_token_accuracy": 0.8448733031749726,
+      "num_tokens": 329271.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1704999923706054,
+      "epoch": 0.24533856722276742,
+      "grad_norm": 2.748300313949585,
+      "learning_rate": 9.858419511547777e-05,
+      "loss": 0.3946,
+      "mean_token_accuracy": 0.8511952340602875,
+      "num_tokens": 343233.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.161389696598053,
+      "epoch": 0.25515210991167814,
+      "grad_norm": 2.374957323074341,
+      "learning_rate": 9.838609474511811e-05,
+      "loss": 0.3818,
+      "mean_token_accuracy": 0.8583569347858429,
+      "num_tokens": 356887.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1713621735572814,
+      "epoch": 0.2649656526005888,
+      "grad_norm": 2.5904409885406494,
+      "learning_rate": 9.81752534755248e-05,
+      "loss": 0.3925,
+      "mean_token_accuracy": 0.851090931892395,
+      "num_tokens": 370548.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.1680105328559875,
+      "epoch": 0.2747791952894995,
+      "grad_norm": 2.4726405143737793,
+      "learning_rate": 9.795172682486655e-05,
+      "loss": 0.3959,
+      "mean_token_accuracy": 0.844909930229187,
+      "num_tokens": 384687.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1694879055023193,
+      "epoch": 0.2845927379784102,
+      "grad_norm": 2.6383578777313232,
+      "learning_rate": 9.77155736515932e-05,
+      "loss": 0.3974,
+      "mean_token_accuracy": 0.8496094286441803,
+      "num_tokens": 398358.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1424716591835022,
+      "epoch": 0.2944062806673209,
+      "grad_norm": 2.962827205657959,
+      "learning_rate": 9.746685613893735e-05,
+      "loss": 0.3773,
+      "mean_token_accuracy": 0.8542423963546752,
+      "num_tokens": 412700.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.1486271262168883,
+      "epoch": 0.3042198233562316,
+      "grad_norm": 1.7420639991760254,
+      "learning_rate": 9.720563977854046e-05,
+      "loss": 0.3533,
+      "mean_token_accuracy": 0.8688081860542297,
+      "num_tokens": 427312.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.1295651912689209,
+      "epoch": 0.3140333660451423,
+      "grad_norm": 2.8025712966918945,
+      "learning_rate": 9.693199335320762e-05,
+      "loss": 0.3583,
+      "mean_token_accuracy": 0.8635108888149261,
+      "num_tokens": 441507.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.1538787603378295,
+      "epoch": 0.323846908734053,
+      "grad_norm": 2.9086270332336426,
+      "learning_rate": 9.664598891879597e-05,
+      "loss": 0.4068,
+      "mean_token_accuracy": 0.8486676514148712,
+      "num_tokens": 454411.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.138922643661499,
+      "epoch": 0.3336604514229637,
+      "grad_norm": 2.316140651702881,
+      "learning_rate": 9.634770178524116e-05,
+      "loss": 0.3854,
+      "mean_token_accuracy": 0.8488239467144012,
+      "num_tokens": 467641.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.1582915425300597,
+      "epoch": 0.3434739941118744,
+      "grad_norm": 2.4371283054351807,
+      "learning_rate": 9.603721049672678e-05,
+      "loss": 0.4107,
+      "mean_token_accuracy": 0.8487084448337555,
+      "num_tokens": 481724.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.1369664549827576,
+      "epoch": 0.35328753680078506,
+      "grad_norm": 3.0655500888824463,
+      "learning_rate": 9.571459681100247e-05,
+      "loss": 0.3489,
+      "mean_token_accuracy": 0.8703281700611114,
+      "num_tokens": 495345.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.1426918387413025,
+      "epoch": 0.3631010794896958,
+      "grad_norm": 2.4565670490264893,
+      "learning_rate": 9.537994567785567e-05,
+      "loss": 0.3793,
+      "mean_token_accuracy": 0.8563880145549774,
+      "num_tokens": 509800.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.1387699961662292,
+      "epoch": 0.3729146221786065,
+      "grad_norm": 3.2218148708343506,
+      "learning_rate": 9.50333452167427e-05,
+      "loss": 0.3601,
+      "mean_token_accuracy": 0.8668544232845307,
+      "num_tokens": 523514.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.1613135576248168,
+      "epoch": 0.38272816486751715,
+      "grad_norm": 2.4641237258911133,
+      "learning_rate": 9.467488669358562e-05,
+      "loss": 0.3814,
+      "mean_token_accuracy": 0.8564598321914673,
+      "num_tokens": 537693.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.146440052986145,
+      "epoch": 0.39254170755642787,
+      "grad_norm": 3.5006353855133057,
+      "learning_rate": 9.430466449674014e-05,
+      "loss": 0.3663,
+      "mean_token_accuracy": 0.8560374557971955,
+      "num_tokens": 551457.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.1358946084976196,
+      "epoch": 0.4023552502453386,
+      "grad_norm": 3.1928484439849854,
+      "learning_rate": 9.39227761121416e-05,
+      "loss": 0.3462,
+      "mean_token_accuracy": 0.8622354686260223,
+      "num_tokens": 565157.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.124997341632843,
+      "epoch": 0.41216879293424924,
+      "grad_norm": 2.1643478870391846,
+      "learning_rate": 9.352932209763523e-05,
+      "loss": 0.3387,
+      "mean_token_accuracy": 0.871830266714096,
+      "num_tokens": 579080.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.1392360091209413,
+      "epoch": 0.42198233562315995,
+      "grad_norm": 2.9661500453948975,
+      "learning_rate": 9.312440605649753e-05,
+      "loss": 0.3644,
+      "mean_token_accuracy": 0.8598394930362702,
+      "num_tokens": 592922.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.124527883529663,
+      "epoch": 0.43179587831207067,
+      "grad_norm": 2.7699403762817383,
+      "learning_rate": 9.270813461015571e-05,
+      "loss": 0.3166,
+      "mean_token_accuracy": 0.8791934370994567,
+      "num_tokens": 607114.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.1162745952606201,
+      "epoch": 0.44160942100098133,
+      "grad_norm": 3.5730695724487305,
+      "learning_rate": 9.228061737011235e-05,
+      "loss": 0.354,
+      "mean_token_accuracy": 0.8609475135803223,
+      "num_tokens": 620907.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.1340227961540221,
+      "epoch": 0.45142296368989204,
+      "grad_norm": 3.203476905822754,
+      "learning_rate": 9.184196690908298e-05,
+      "loss": 0.328,
+      "mean_token_accuracy": 0.8800676703453064,
+      "num_tokens": 635363.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.1231265664100647,
+      "epoch": 0.46123650637880276,
+      "grad_norm": 3.116881847381592,
+      "learning_rate": 9.139229873135349e-05,
+      "loss": 0.3571,
+      "mean_token_accuracy": 0.8617198586463928,
+      "num_tokens": 648200.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.1675281405448914,
+      "epoch": 0.47105004906771347,
+      "grad_norm": 3.3208513259887695,
+      "learning_rate": 9.0931731242366e-05,
+      "loss": 0.3781,
+      "mean_token_accuracy": 0.8595807194709778,
+      "num_tokens": 662920.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.1431254148483276,
+      "epoch": 0.48086359175662413,
+      "grad_norm": 3.35781192779541,
+      "learning_rate": 9.046038571754066e-05,
+      "loss": 0.3489,
+      "mean_token_accuracy": 0.8666664302349091,
+      "num_tokens": 676646.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.1514458537101746,
+      "epoch": 0.49067713444553485,
+      "grad_norm": 3.0265209674835205,
+      "learning_rate": 8.997838627034158e-05,
+      "loss": 0.3323,
+      "mean_token_accuracy": 0.8773599326610565,
+      "num_tokens": 690435.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.155480444431305,
+      "epoch": 0.5004906771344455,
+      "grad_norm": 3.4788320064544678,
+      "learning_rate": 8.94858598195958e-05,
+      "loss": 0.3304,
+      "mean_token_accuracy": 0.8751632452011109,
+      "num_tokens": 704664.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.145072877407074,
+      "epoch": 0.5103042198233563,
+      "grad_norm": 3.304614305496216,
+      "learning_rate": 8.898293605607316e-05,
+      "loss": 0.35,
+      "mean_token_accuracy": 0.8696847498416901,
+      "num_tokens": 718834.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.1518910646438598,
+      "epoch": 0.5201177625122669,
+      "grad_norm": 2.8898963928222656,
+      "learning_rate": 8.846974740833659e-05,
+      "loss": 0.3357,
+      "mean_token_accuracy": 0.8720099747180938,
+      "num_tokens": 733172.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.148727285861969,
+      "epoch": 0.5299313052011776,
+      "grad_norm": 3.2443408966064453,
+      "learning_rate": 8.794642900787137e-05,
+      "loss": 0.3397,
+      "mean_token_accuracy": 0.8645234525203704,
+      "num_tokens": 746577.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.129120123386383,
+      "epoch": 0.5397448478900884,
+      "grad_norm": 2.6186652183532715,
+      "learning_rate": 8.741311865350259e-05,
+      "loss": 0.3217,
+      "mean_token_accuracy": 0.8711999893188477,
+      "num_tokens": 760378.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.1712802052497864,
+      "epoch": 0.549558390578999,
+      "grad_norm": 2.564211368560791,
+      "learning_rate": 8.68699567751104e-05,
+      "loss": 0.3408,
+      "mean_token_accuracy": 0.8635621011257172,
+      "num_tokens": 774528.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.1346929788589477,
+      "epoch": 0.5593719332679097,
+      "grad_norm": 2.537116289138794,
+      "learning_rate": 8.631708639665234e-05,
+      "loss": 0.3133,
+      "mean_token_accuracy": 0.8797516524791718,
+      "num_tokens": 787978.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.152209258079529,
+      "epoch": 0.5691854759568205,
+      "grad_norm": 2.554788827896118,
+      "learning_rate": 8.575465309850265e-05,
+      "loss": 0.3055,
+      "mean_token_accuracy": 0.882277500629425,
+      "num_tokens": 802122.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.1348226428031922,
+      "epoch": 0.5789990186457311,
+      "grad_norm": 2.7317800521850586,
+      "learning_rate": 8.518280497911841e-05,
+      "loss": 0.2903,
+      "mean_token_accuracy": 0.8882303416728974,
+      "num_tokens": 816567.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.140570294857025,
+      "epoch": 0.5888125613346418,
+      "grad_norm": 2.770073652267456,
+      "learning_rate": 8.460169261604274e-05,
+      "loss": 0.2674,
+      "mean_token_accuracy": 0.8973830997943878,
+      "num_tokens": 830550.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.1052254796028138,
+      "epoch": 0.5986261040235525,
+      "grad_norm": 2.3624937534332275,
+      "learning_rate": 8.4011469026255e-05,
+      "loss": 0.2884,
+      "mean_token_accuracy": 0.8947200298309326,
+      "num_tokens": 843779.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.1328206419944764,
+      "epoch": 0.6084396467124632,
+      "grad_norm": 3.094525098800659,
+      "learning_rate": 8.341228962587882e-05,
+      "loss": 0.3103,
+      "mean_token_accuracy": 0.8834663510322571,
+      "num_tokens": 857498.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.123459279537201,
+      "epoch": 0.6182531894013739,
+      "grad_norm": 2.848069429397583,
+      "learning_rate": 8.28043121892583e-05,
+      "loss": 0.3136,
+      "mean_token_accuracy": 0.8795402646064758,
+      "num_tokens": 871163.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.1469326615333557,
+      "epoch": 0.6280667320902846,
+      "grad_norm": 2.8613340854644775,
+      "learning_rate": 8.218769680741336e-05,
+      "loss": 0.3262,
+      "mean_token_accuracy": 0.8794827044010163,
+      "num_tokens": 884912.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.150424337387085,
+      "epoch": 0.6378802747791953,
+      "grad_norm": 3.4515678882598877,
+      "learning_rate": 8.156260584588491e-05,
+      "loss": 0.3079,
+      "mean_token_accuracy": 0.8865741074085236,
+      "num_tokens": 898102.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.1549550294876099,
+      "epoch": 0.647693817468106,
+      "grad_norm": 2.868631362915039,
+      "learning_rate": 8.09292039019812e-05,
+      "loss": 0.3148,
+      "mean_token_accuracy": 0.8782100558280945,
+      "num_tokens": 911902.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.175901460647583,
+      "epoch": 0.6575073601570167,
+      "grad_norm": 2.7461845874786377,
+      "learning_rate": 8.028765776143654e-05,
+      "loss": 0.3465,
+      "mean_token_accuracy": 0.8678304076194763,
+      "num_tokens": 926137.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.1141401767730712,
+      "epoch": 0.6673209028459274,
+      "grad_norm": 2.419983386993408,
+      "learning_rate": 7.963813635449367e-05,
+      "loss": 0.2727,
+      "mean_token_accuracy": 0.8948099255561829,
+      "num_tokens": 940507.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.1273057579994201,
+      "epoch": 0.677134445534838,
+      "grad_norm": 2.074404716491699,
+      "learning_rate": 7.898081071142149e-05,
+      "loss": 0.2899,
+      "mean_token_accuracy": 0.8851046442985535,
+      "num_tokens": 954934.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.1249045848846435,
+      "epoch": 0.6869479882237488,
+      "grad_norm": 3.2738053798675537,
+      "learning_rate": 7.831585391747984e-05,
+      "loss": 0.3062,
+      "mean_token_accuracy": 0.8843758761882782,
+      "num_tokens": 968760.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.1176576018333435,
+      "epoch": 0.6967615309126595,
+      "grad_norm": 3.043499231338501,
+      "learning_rate": 7.764344106734319e-05,
+      "loss": 0.2786,
+      "mean_token_accuracy": 0.8965504765510559,
+      "num_tokens": 982644.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.1249203324317931,
+      "epoch": 0.7065750736015701,
+      "grad_norm": 2.664680242538452,
+      "learning_rate": 7.696374921899506e-05,
+      "loss": 0.309,
+      "mean_token_accuracy": 0.8861907541751861,
+      "num_tokens": 996817.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.1197885870933533,
+      "epoch": 0.7163886162904809,
+      "grad_norm": 3.241985321044922,
+      "learning_rate": 7.627695734710564e-05,
+      "loss": 0.2899,
+      "mean_token_accuracy": 0.8906128644943238,
+      "num_tokens": 1010008.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.128973948955536,
+      "epoch": 0.7262021589793916,
+      "grad_norm": 2.309560537338257,
+      "learning_rate": 7.55832462959047e-05,
+      "loss": 0.2821,
+      "mean_token_accuracy": 0.8881384789943695,
+      "num_tokens": 1023812.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.1201381921768188,
+      "epoch": 0.7360157016683022,
+      "grad_norm": 3.083052158355713,
+      "learning_rate": 7.488279873156207e-05,
+      "loss": 0.3059,
+      "mean_token_accuracy": 0.8764900207519531,
+      "num_tokens": 1037399.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.1090063810348512,
+      "epoch": 0.745829244357213,
+      "grad_norm": 2.9035286903381348,
+      "learning_rate": 7.417579909408851e-05,
+      "loss": 0.2687,
+      "mean_token_accuracy": 0.8969748079776764,
+      "num_tokens": 1050972.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.1148179054260254,
+      "epoch": 0.7556427870461236,
+      "grad_norm": 3.650284767150879,
+      "learning_rate": 7.346243354876944e-05,
+      "loss": 0.3022,
+      "mean_token_accuracy": 0.880945748090744,
+      "num_tokens": 1064581.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.1167670965194703,
+      "epoch": 0.7654563297350343,
+      "grad_norm": 2.7539217472076416,
+      "learning_rate": 7.274288993714446e-05,
+      "loss": 0.2778,
+      "mean_token_accuracy": 0.8922274172306061,
+      "num_tokens": 1078433.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.1169269680976868,
+      "epoch": 0.7752698724239451,
+      "grad_norm": 3.750164270401001,
+      "learning_rate": 7.201735772754523e-05,
+      "loss": 0.2785,
+      "mean_token_accuracy": 0.891158002614975,
+      "num_tokens": 1091839.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.1045061707496644,
+      "epoch": 0.7850834151128557,
+      "grad_norm": 3.050450563430786,
+      "learning_rate": 7.128602796520538e-05,
+      "loss": 0.2893,
+      "mean_token_accuracy": 0.8866830766201019,
+      "num_tokens": 1105747.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.0993719220161438,
+      "epoch": 0.7948969578017664,
+      "grad_norm": 3.887840509414673,
+      "learning_rate": 7.054909322195485e-05,
+      "loss": 0.2892,
+      "mean_token_accuracy": 0.8876266241073608,
+      "num_tokens": 1119344.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.1005550980567933,
+      "epoch": 0.8047105004906772,
+      "grad_norm": 3.016305685043335,
+      "learning_rate": 6.980674754551239e-05,
+      "loss": 0.2509,
+      "mean_token_accuracy": 0.8967083752155304,
+      "num_tokens": 1133252.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.1192949771881104,
+      "epoch": 0.8145240431795878,
+      "grad_norm": 3.175689458847046,
+      "learning_rate": 6.905918640838946e-05,
+      "loss": 0.2811,
+      "mean_token_accuracy": 0.8886835515499115,
+      "num_tokens": 1147368.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.1032113075256347,
+      "epoch": 0.8243375858684985,
+      "grad_norm": 2.8120744228363037,
+      "learning_rate": 6.830660665641898e-05,
+      "loss": 0.2684,
+      "mean_token_accuracy": 0.8989933490753174,
+      "num_tokens": 1161165.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.129219686985016,
+      "epoch": 0.8341511285574092,
+      "grad_norm": 3.513230562210083,
+      "learning_rate": 6.754920645692222e-05,
+      "loss": 0.273,
+      "mean_token_accuracy": 0.8876569330692291,
+      "num_tokens": 1174814.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.1321211218833924,
+      "epoch": 0.8439646712463199,
+      "grad_norm": 2.925628662109375,
+      "learning_rate": 6.67871852465281e-05,
+      "loss": 0.2991,
+      "mean_token_accuracy": 0.8839828908443451,
+      "num_tokens": 1188967.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.1208232760429382,
+      "epoch": 0.8537782139352306,
+      "grad_norm": 3.0828044414520264,
+      "learning_rate": 6.602074367865794e-05,
+      "loss": 0.2756,
+      "mean_token_accuracy": 0.8958942711353302,
+      "num_tokens": 1202907.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.1091869473457336,
+      "epoch": 0.8635917566241413,
+      "grad_norm": 2.6561832427978516,
+      "learning_rate": 6.525008357069e-05,
+      "loss": 0.2496,
+      "mean_token_accuracy": 0.9039375066757203,
+      "num_tokens": 1216865.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.1259613037109375,
+      "epoch": 0.873405299313052,
+      "grad_norm": 3.0927224159240723,
+      "learning_rate": 6.447540785081738e-05,
+      "loss": 0.2618,
+      "mean_token_accuracy": 0.8995936453342438,
+      "num_tokens": 1230849.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.1081516861915588,
+      "epoch": 0.8832188420019627,
+      "grad_norm": 2.8593716621398926,
+      "learning_rate": 6.36969205046136e-05,
+      "loss": 0.2669,
+      "mean_token_accuracy": 0.8947809040546417,
+      "num_tokens": 1243965.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.1007655382156372,
+      "epoch": 0.8930323846908734,
+      "grad_norm": 3.1711161136627197,
+      "learning_rate": 6.291482652131955e-05,
+      "loss": 0.251,
+      "mean_token_accuracy": 0.902769410610199,
+      "num_tokens": 1257918.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.09490567445755,
+      "epoch": 0.9028459273797841,
+      "grad_norm": 2.8350908756256104,
+      "learning_rate": 6.212933183986634e-05,
+      "loss": 0.2745,
+      "mean_token_accuracy": 0.895069706439972,
+      "num_tokens": 1271980.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.0979399681091309,
+      "epoch": 0.9126594700686947,
+      "grad_norm": 3.4391095638275146,
+      "learning_rate": 6.1340643294648e-05,
+      "loss": 0.2533,
+      "mean_token_accuracy": 0.8967029035091401,
+      "num_tokens": 1286287.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.1099019050598145,
+      "epoch": 0.9224730127576055,
+      "grad_norm": 2.3745553493499756,
+      "learning_rate": 6.054896856105834e-05,
+      "loss": 0.2639,
+      "mean_token_accuracy": 0.8974069118499756,
+      "num_tokens": 1300381.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.0974381685256958,
+      "epoch": 0.9322865554465162,
+      "grad_norm": 3.702469825744629,
+      "learning_rate": 5.9754516100806423e-05,
+      "loss": 0.2405,
+      "mean_token_accuracy": 0.9070028126239776,
+      "num_tokens": 1314086.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.1083937525749206,
+      "epoch": 0.9421000981354269,
+      "grad_norm": 3.0780155658721924,
+      "learning_rate": 5.895749510702502e-05,
+      "loss": 0.2601,
+      "mean_token_accuracy": 0.8937717497348785,
+      "num_tokens": 1328434.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.1007688283920287,
+      "epoch": 0.9519136408243376,
+      "grad_norm": 2.4240477085113525,
+      "learning_rate": 5.815811544918641e-05,
+      "loss": 0.2601,
+      "mean_token_accuracy": 0.8979498445987701,
+      "num_tokens": 1341971.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.1114157795906068,
+      "epoch": 0.9617271835132483,
+      "grad_norm": 3.555633068084717,
+      "learning_rate": 5.7356587617840065e-05,
+      "loss": 0.226,
+      "mean_token_accuracy": 0.9160135686397552,
+      "num_tokens": 1356685.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.1007156252861023,
+      "epoch": 0.971540726202159,
+      "grad_norm": 4.091366767883301,
+      "learning_rate": 5.655312266918684e-05,
+      "loss": 0.2782,
+      "mean_token_accuracy": 0.8913241505622864,
+      "num_tokens": 1370541.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.078227972984314,
+      "epoch": 0.9813542688910697,
+      "grad_norm": 3.624323844909668,
+      "learning_rate": 5.574793216950428e-05,
+      "loss": 0.2256,
+      "mean_token_accuracy": 0.911854875087738,
+      "num_tokens": 1383962.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.0911343812942504,
+      "epoch": 0.9911678115799804,
+      "grad_norm": 3.2023518085479736,
+      "learning_rate": 5.494122813943742e-05,
+      "loss": 0.2243,
+      "mean_token_accuracy": 0.9113712906837463,
+      "num_tokens": 1397902.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.0845252752304078,
+      "epoch": 1.000981354268891,
+      "grad_norm": 3.0422451496124268,
+      "learning_rate": 5.413322299817019e-05,
+      "loss": 0.2373,
+      "mean_token_accuracy": 0.903719425201416,
+      "num_tokens": 1410184.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.0734800577163697,
+      "epoch": 1.0107948969578018,
+      "grad_norm": 2.707657814025879,
+      "learning_rate": 5.332412950749169e-05,
+      "loss": 0.1654,
+      "mean_token_accuracy": 0.9364395558834075,
+      "num_tokens": 1424018.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.075714075565338,
+      "epoch": 1.0206084396467126,
+      "grad_norm": 2.75673508644104,
+      "learning_rate": 5.251416071577232e-05,
+      "loss": 0.1624,
+      "mean_token_accuracy": 0.9354286074638367,
+      "num_tokens": 1438068.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.0505158841609954,
+      "epoch": 1.030421982335623,
+      "grad_norm": 3.937579870223999,
+      "learning_rate": 5.170352990186441e-05,
+      "loss": 0.1513,
+      "mean_token_accuracy": 0.94420907497406,
+      "num_tokens": 1452019.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.0319998800754546,
+      "epoch": 1.0402355250245339,
+      "grad_norm": 4.000500202178955,
+      "learning_rate": 5.08924505189423e-05,
+      "loss": 0.1546,
+      "mean_token_accuracy": 0.9410330891609192,
+      "num_tokens": 1465698.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.0415607810020446,
+      "epoch": 1.0500490677134446,
+      "grad_norm": 3.269470691680908,
+      "learning_rate": 5.008113613829631e-05,
+      "loss": 0.1593,
+      "mean_token_accuracy": 0.9412495851516723,
+      "num_tokens": 1479631.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.0488685727119447,
+      "epoch": 1.0598626104023552,
+      "grad_norm": 4.048078536987305,
+      "learning_rate": 4.9269800393095747e-05,
+      "loss": 0.1712,
+      "mean_token_accuracy": 0.9371578395366669,
+      "num_tokens": 1492799.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.0499681234359741,
+      "epoch": 1.069676153091266,
+      "grad_norm": 3.534576416015625,
+      "learning_rate": 4.845865692213555e-05,
+      "loss": 0.15,
+      "mean_token_accuracy": 0.944264805316925,
+      "num_tokens": 1506531.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.0402419328689576,
+      "epoch": 1.0794896957801767,
+      "grad_norm": 2.6752941608428955,
+      "learning_rate": 4.764791931358155e-05,
+      "loss": 0.143,
+      "mean_token_accuracy": 0.9475312113761902,
+      "num_tokens": 1520031.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.0501301765441895,
+      "epoch": 1.0893032384690873,
+      "grad_norm": 3.266214370727539,
+      "learning_rate": 4.683780104872894e-05,
+      "loss": 0.1505,
+      "mean_token_accuracy": 0.9476670563220978,
+      "num_tokens": 1534436.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.0541617274284363,
+      "epoch": 1.099116781157998,
+      "grad_norm": 2.496053695678711,
+      "learning_rate": 4.6028515445788937e-05,
+      "loss": 0.1707,
+      "mean_token_accuracy": 0.9338534295558929,
+      "num_tokens": 1547962.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.034402847290039,
+      "epoch": 1.1089303238469088,
+      "grad_norm": 2.7755892276763916,
+      "learning_rate": 4.5220275603718474e-05,
+      "loss": 0.1591,
+      "mean_token_accuracy": 0.9391697168350219,
+      "num_tokens": 1561849.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.0240708827972411,
+      "epoch": 1.1187438665358194,
+      "grad_norm": 3.69889760017395,
+      "learning_rate": 4.4413294346107394e-05,
+      "loss": 0.1432,
+      "mean_token_accuracy": 0.9476984679698944,
+      "num_tokens": 1575503.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.0393796622753144,
+      "epoch": 1.1285574092247301,
+      "grad_norm": 3.421630382537842,
+      "learning_rate": 4.3607784165138474e-05,
+      "loss": 0.1497,
+      "mean_token_accuracy": 0.9402693152427674,
+      "num_tokens": 1589893.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.037726628780365,
+      "epoch": 1.138370951913641,
+      "grad_norm": 2.53251051902771,
+      "learning_rate": 4.280395716563441e-05,
+      "loss": 0.1339,
+      "mean_token_accuracy": 0.949617725610733,
+      "num_tokens": 1603816.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.046052622795105,
+      "epoch": 1.1481844946025515,
+      "grad_norm": 3.206963062286377,
+      "learning_rate": 4.2002025009206994e-05,
+      "loss": 0.1484,
+      "mean_token_accuracy": 0.9449362874031066,
+      "num_tokens": 1617087.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.0372208595275878,
+      "epoch": 1.1579980372914622,
+      "grad_norm": 3.3810195922851562,
+      "learning_rate": 4.120219885852292e-05,
+      "loss": 0.142,
+      "mean_token_accuracy": 0.9433592081069946,
+      "num_tokens": 1631558.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.047147536277771,
+      "epoch": 1.167811579980373,
+      "grad_norm": 3.2576894760131836,
+      "learning_rate": 4.040468932170097e-05,
+      "loss": 0.1488,
+      "mean_token_accuracy": 0.9437485873699188,
+      "num_tokens": 1645944.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.028389811515808,
+      "epoch": 1.1776251226692835,
+      "grad_norm": 3.1290345191955566,
+      "learning_rate": 3.9609706396855296e-05,
+      "loss": 0.1417,
+      "mean_token_accuracy": 0.9444807350635529,
+      "num_tokens": 1659636.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.0259510278701782,
+      "epoch": 1.1874386653581943,
+      "grad_norm": 3.0798492431640625,
+      "learning_rate": 3.881745941679917e-05,
+      "loss": 0.1455,
+      "mean_token_accuracy": 0.9441400289535522,
+      "num_tokens": 1673078.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.051153826713562,
+      "epoch": 1.197252208047105,
+      "grad_norm": 3.7906973361968994,
+      "learning_rate": 3.802815699392418e-05,
+      "loss": 0.1334,
+      "mean_token_accuracy": 0.9504075527191163,
+      "num_tokens": 1687154.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.037259477376938,
+      "epoch": 1.2070657507360156,
+      "grad_norm": 3.5961825847625732,
+      "learning_rate": 3.724200696526873e-05,
+      "loss": 0.1339,
+      "mean_token_accuracy": 0.9480992436408997,
+      "num_tokens": 1701533.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.0242123484611512,
+      "epoch": 1.2168792934249264,
+      "grad_norm": 4.319852352142334,
+      "learning_rate": 3.6459216337791124e-05,
+      "loss": 0.1459,
+      "mean_token_accuracy": 0.9491089105606079,
+      "num_tokens": 1714722.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.0142346739768981,
+      "epoch": 1.2266928361138372,
+      "grad_norm": 3.67496395111084,
+      "learning_rate": 3.567999123386103e-05,
+      "loss": 0.1484,
+      "mean_token_accuracy": 0.9443210601806641,
+      "num_tokens": 1727989.0,
+      "step": 1250
+    },
+    {
+      "entropy": 1.0340947270393372,
+      "epoch": 1.2365063788027477,
+      "grad_norm": 3.6783833503723145,
+      "learning_rate": 3.490453683698376e-05,
+      "loss": 0.1362,
+      "mean_token_accuracy": 0.9488137662410736,
+      "num_tokens": 1742532.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.0295516014099122,
+      "epoch": 1.2463199214916585,
+      "grad_norm": 3.1686367988586426,
+      "learning_rate": 3.4133057337771995e-05,
+      "loss": 0.1317,
+      "mean_token_accuracy": 0.9486495316028595,
+      "num_tokens": 1755690.0,
+      "step": 1270
+    },
+    {
+      "entropy": 1.017775797843933,
+      "epoch": 1.2561334641805693,
+      "grad_norm": 3.237813949584961,
+      "learning_rate": 3.336575588017881e-05,
+      "loss": 0.1389,
+      "mean_token_accuracy": 0.9473410189151764,
+      "num_tokens": 1769918.0,
+      "step": 1280
+    },
+    {
+      "entropy": 1.027511203289032,
+      "epoch": 1.2659470068694798,
+      "grad_norm": 4.21461296081543,
+      "learning_rate": 3.260283450800634e-05,
+      "loss": 0.1273,
+      "mean_token_accuracy": 0.9546197652816772,
+      "num_tokens": 1783944.0,
+      "step": 1290
+    },
+    {
+      "entropy": 1.0197508096694947,
+      "epoch": 1.2757605495583906,
+      "grad_norm": 3.6392710208892822,
+      "learning_rate": 3.184449411170416e-05,
+      "loss": 0.1195,
+      "mean_token_accuracy": 0.9533522009849549,
+      "num_tokens": 1797715.0,
+      "step": 1300
+    },
+    {
+      "entropy": 1.019310063123703,
+      "epoch": 1.2855740922473013,
+      "grad_norm": 2.6170411109924316,
+      "learning_rate": 3.109093437547128e-05,
+      "loss": 0.1549,
+      "mean_token_accuracy": 0.9430364489555358,
+      "num_tokens": 1811354.0,
+      "step": 1310
+    },
+    {
+      "entropy": 1.0234230160713196,
+      "epoch": 1.295387634936212,
+      "grad_norm": 3.7146430015563965,
+      "learning_rate": 3.0342353724675966e-05,
+      "loss": 0.1386,
+      "mean_token_accuracy": 0.947895622253418,
+      "num_tokens": 1825286.0,
+      "step": 1320
+    },
+    {
+      "entropy": 1.0157096564769745,
+      "epoch": 1.3052011776251227,
+      "grad_norm": 4.59371280670166,
+      "learning_rate": 2.9598949273606702e-05,
+      "loss": 0.1344,
+      "mean_token_accuracy": 0.9511359512805939,
+      "num_tokens": 1839043.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.9966915547847748,
+      "epoch": 1.3150147203140334,
+      "grad_norm": 2.9230849742889404,
+      "learning_rate": 2.8860916773568703e-05,
+      "loss": 0.1152,
+      "mean_token_accuracy": 0.9593356549739838,
+      "num_tokens": 1852342.0,
+      "step": 1340
+    },
+    {
+      "entropy": 1.0024271368980409,
+      "epoch": 1.324828263002944,
+      "grad_norm": 4.6238627433776855,
+      "learning_rate": 2.8128450561339193e-05,
+      "loss": 0.1385,
+      "mean_token_accuracy": 0.9509434998035431,
+      "num_tokens": 1866189.0,
+      "step": 1350
+    },
+    {
+      "entropy": 1.0314253211021422,
+      "epoch": 1.3346418056918548,
+      "grad_norm": 3.3522119522094727,
+      "learning_rate": 2.740174350799507e-05,
+      "loss": 0.1299,
+      "mean_token_accuracy": 0.9510968923568726,
+      "num_tokens": 1880098.0,
+      "step": 1360
+    },
+    {
+      "entropy": 1.0071939885616303,
+      "epoch": 1.3444553483807655,
+      "grad_norm": 3.5018398761749268,
+      "learning_rate": 2.668098696812675e-05,
+      "loss": 0.1234,
+      "mean_token_accuracy": 0.9565033853054047,
+      "num_tokens": 1893007.0,
+      "step": 1370
+    },
+    {
+      "entropy": 1.0181950151920318,
+      "epoch": 1.354268891069676,
+      "grad_norm": 3.1666171550750732,
+      "learning_rate": 2.596637072945099e-05,
+      "loss": 0.107,
+      "mean_token_accuracy": 0.9608981430530548,
+      "num_tokens": 1906405.0,
+      "step": 1380
+    },
+    {
+      "entropy": 1.0229911267757417,
+      "epoch": 1.3640824337585868,
+      "grad_norm": 4.901073455810547,
+      "learning_rate": 2.5258082962836614e-05,
+      "loss": 0.1477,
+      "mean_token_accuracy": 0.9456415474414825,
+      "num_tokens": 1920751.0,
+      "step": 1390
+    },
+    {
+      "entropy": 1.0236889183521272,
+      "epoch": 1.3738959764474976,
+      "grad_norm": 5.06928014755249,
+      "learning_rate": 2.4556310172755898e-05,
+      "loss": 0.1501,
+      "mean_token_accuracy": 0.9451291382312774,
+      "num_tokens": 1934411.0,
+      "step": 1400
+    },
+    {
+      "entropy": 1.0231754660606385,
+      "epoch": 1.3837095191364082,
+      "grad_norm": 4.104011535644531,
+      "learning_rate": 2.3861237148174474e-05,
+      "loss": 0.1314,
+      "mean_token_accuracy": 0.9483218967914582,
+      "num_tokens": 1948013.0,
+      "step": 1410
+    },
+    {
+      "entropy": 1.0202842891216277,
+      "epoch": 1.393523061825319,
+      "grad_norm": 3.319493532180786,
+      "learning_rate": 2.3173046913893502e-05,
+      "loss": 0.1164,
+      "mean_token_accuracy": 0.9566279232501984,
+      "num_tokens": 1962274.0,
+      "step": 1420
+    },
+    {
+      "entropy": 1.0396390914916993,
+      "epoch": 1.4033366045142297,
+      "grad_norm": 4.384134292602539,
+      "learning_rate": 2.249192068235581e-05,
+      "loss": 0.1286,
+      "mean_token_accuracy": 0.9555740237236023,
+      "num_tokens": 1976604.0,
+      "step": 1430
+    },
+    {
+      "entropy": 1.0360827267169952,
+      "epoch": 1.4131501472031402,
+      "grad_norm": 3.7225964069366455,
+      "learning_rate": 2.181803780592959e-05,
+      "loss": 0.1231,
+      "mean_token_accuracy": 0.9553512513637543,
+      "num_tokens": 1991136.0,
+      "step": 1440
+    },
+    {
+      "entropy": 1.0296751737594605,
+      "epoch": 1.422963689892051,
+      "grad_norm": 4.736987113952637,
+      "learning_rate": 2.1151575729681883e-05,
+      "loss": 0.1352,
+      "mean_token_accuracy": 0.9476152658462524,
+      "num_tokens": 2005289.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.0167883574962615,
+      "epoch": 1.4327772325809618,
+      "grad_norm": 2.6278562545776367,
+      "learning_rate": 2.049270994465407e-05,
+      "loss": 0.1194,
+      "mean_token_accuracy": 0.9533245801925659,
+      "num_tokens": 2018916.0,
+      "step": 1460
+    },
+    {
+      "entropy": 1.0144045293331145,
+      "epoch": 1.4425907752698723,
+      "grad_norm": 3.189089775085449,
+      "learning_rate": 1.9841613941652144e-05,
+      "loss": 0.1196,
+      "mean_token_accuracy": 0.9550191342830658,
+      "num_tokens": 2032597.0,
+      "step": 1470
+    },
+    {
+      "entropy": 1.016832709312439,
+      "epoch": 1.452404317958783,
+      "grad_norm": 2.6634836196899414,
+      "learning_rate": 1.9198459165563503e-05,
+      "loss": 0.1145,
+      "mean_token_accuracy": 0.9568855941295624,
+      "num_tokens": 2046720.0,
+      "step": 1480
+    },
+    {
+      "entropy": 1.0291604459285737,
+      "epoch": 1.4622178606476939,
+      "grad_norm": 3.8889691829681396,
+      "learning_rate": 1.8563414970212555e-05,
+      "loss": 0.1138,
+      "mean_token_accuracy": 0.9593892574310303,
+      "num_tokens": 2060943.0,
+      "step": 1490
+    },
+    {
+      "entropy": 1.0184860706329346,
+      "epoch": 1.4720314033366044,
+      "grad_norm": 3.682466506958008,
+      "learning_rate": 1.7936648573766957e-05,
+      "loss": 0.1236,
+      "mean_token_accuracy": 0.9525898098945618,
+      "num_tokens": 2074710.0,
+      "step": 1500
+    },
+    {
+      "entropy": 1.0316950976848602,
+      "epoch": 1.4818449460255152,
+      "grad_norm": 3.6130685806274414,
+      "learning_rate": 1.731832501470606e-05,
+      "loss": 0.1283,
+      "mean_token_accuracy": 0.950767320394516,
+      "num_tokens": 2088707.0,
+      "step": 1510
+    },
+    {
+      "entropy": 1.0129083931446075,
+      "epoch": 1.491658488714426,
+      "grad_norm": 3.51699161529541,
+      "learning_rate": 1.670860710836355e-05,
+      "loss": 0.1085,
+      "mean_token_accuracy": 0.9615793049335479,
+      "num_tokens": 2101428.0,
+      "step": 1520
+    },
+    {
+      "entropy": 1.0210984647274017,
+      "epoch": 1.5014720314033365,
+      "grad_norm": 2.585371971130371,
+      "learning_rate": 1.6107655404055223e-05,
+      "loss": 0.13,
+      "mean_token_accuracy": 0.9534424245357513,
+      "num_tokens": 2115066.0,
+      "step": 1530
+    },
+    {
+      "entropy": 1.0233879208564758,
+      "epoch": 1.5112855740922473,
+      "grad_norm": 4.09867000579834,
+      "learning_rate": 1.5515628142803562e-05,
+      "loss": 0.125,
+      "mean_token_accuracy": 0.9500893294811249,
+      "num_tokens": 2128953.0,
+      "step": 1540
+    },
+    {
+      "entropy": 1.022993516921997,
+      "epoch": 1.521099116781158,
+      "grad_norm": 3.7925915718078613,
+      "learning_rate": 1.493268121567028e-05,
+      "loss": 0.1357,
+      "mean_token_accuracy": 0.9527780055999756,
+      "num_tokens": 2142756.0,
+      "step": 1550
+    },
+    {
+      "entropy": 1.0126155316829681,
+      "epoch": 1.5309126594700686,
+      "grad_norm": 3.4878346920013428,
+      "learning_rate": 1.435896812270729e-05,
+      "loss": 0.1223,
+      "mean_token_accuracy": 0.9558422327041626,
+      "num_tokens": 2156108.0,
+      "step": 1560
+    },
+    {
+      "entropy": 1.019024622440338,
+      "epoch": 1.5407262021589794,
+      "grad_norm": 4.254006385803223,
+      "learning_rate": 1.3794639932537645e-05,
+      "loss": 0.1286,
+      "mean_token_accuracy": 0.9567427575588227,
+      "num_tokens": 2169356.0,
+      "step": 1570
+    },
+    {
+      "entropy": 1.0355127692222594,
+      "epoch": 1.5505397448478901,
+      "grad_norm": 3.0657994747161865,
+      "learning_rate": 1.323984524257641e-05,
+      "loss": 0.1152,
+      "mean_token_accuracy": 0.9599360525608063,
+      "num_tokens": 2183713.0,
+      "step": 1580
+    },
+    {
+      "entropy": 1.0317145586013794,
+      "epoch": 1.5603532875368007,
+      "grad_norm": 4.222753047943115,
+      "learning_rate": 1.2694730139902522e-05,
+      "loss": 0.1154,
+      "mean_token_accuracy": 0.955812793970108,
+      "num_tokens": 2198089.0,
+      "step": 1590
+    },
+    {
+      "entropy": 1.0375869631767274,
+      "epoch": 1.5701668302257115,
+      "grad_norm": 2.8060295581817627,
+      "learning_rate": 1.2159438162791348e-05,
+      "loss": 0.1241,
+      "mean_token_accuracy": 0.9532338082790375,
+      "num_tokens": 2213070.0,
+      "step": 1600
+    },
+    {
+      "entropy": 1.0177474081516267,
+      "epoch": 1.5799803729146222,
+      "grad_norm": 2.6118738651275635,
+      "learning_rate": 1.1634110262918718e-05,
+      "loss": 0.1195,
+      "mean_token_accuracy": 0.9579388201236725,
+      "num_tokens": 2227108.0,
+      "step": 1610
+    },
+    {
+      "entropy": 1.0259399175643922,
+      "epoch": 1.5897939156035328,
+      "grad_norm": 4.264151096343994,
+      "learning_rate": 1.1118884768245863e-05,
+      "loss": 0.1067,
+      "mean_token_accuracy": 0.9573590457439423,
+      "num_tokens": 2240704.0,
+      "step": 1620
+    },
+    {
+      "entropy": 1.0328880190849303,
+      "epoch": 1.5996074582924436,
+      "grad_norm": 2.685889959335327,
+      "learning_rate": 1.0613897346595203e-05,
+      "loss": 0.126,
+      "mean_token_accuracy": 0.9531948506832123,
+      "num_tokens": 2254968.0,
+      "step": 1630
+    },
+    {
+      "entropy": 1.0358797311782837,
+      "epoch": 1.6094210009813543,
+      "grad_norm": 3.4554998874664307,
+      "learning_rate": 1.011928096992672e-05,
+      "loss": 0.1153,
+      "mean_token_accuracy": 0.9544391334056854,
+      "num_tokens": 2269175.0,
+      "step": 1640
+    },
+    {
+      "entropy": 1.021525502204895,
+      "epoch": 1.6192345436702649,
+      "grad_norm": 3.818470001220703,
+      "learning_rate": 9.635165879324083e-06,
+      "loss": 0.1162,
+      "mean_token_accuracy": 0.9583705246448517,
+      "num_tokens": 2283760.0,
+      "step": 1650
+    },
+    {
+      "entropy": 1.02712362408638,
+      "epoch": 1.6290480863591756,
+      "grad_norm": 2.6576719284057617,
+      "learning_rate": 9.161679550699936e-06,
+      "loss": 0.1067,
+      "mean_token_accuracy": 0.9606935262680054,
+      "num_tokens": 2297542.0,
+      "step": 1660
+    },
+    {
+      "entropy": 1.0276360750198363,
+      "epoch": 1.6388616290480864,
+      "grad_norm": 3.416405200958252,
+      "learning_rate": 8.698946661229262e-06,
+      "loss": 0.115,
+      "mean_token_accuracy": 0.9588954031467438,
+      "num_tokens": 2311381.0,
+      "step": 1670
+    },
+    {
+      "entropy": 1.0020666241645813,
+      "epoch": 1.648675171736997,
+      "grad_norm": 2.495941638946533,
+      "learning_rate": 8.24708905651972e-06,
+      "loss": 0.1353,
+      "mean_token_accuracy": 0.947125256061554,
+      "num_tokens": 2324526.0,
+      "step": 1680
+    },
+    {
+      "entropy": 1.0160171449184419,
+      "epoch": 1.6584887144259077,
+      "grad_norm": 2.680049419403076,
+      "learning_rate": 7.80622571852761e-06,
+      "loss": 0.1161,
+      "mean_token_accuracy": 0.9561552762985229,
+      "num_tokens": 2337382.0,
+      "step": 1690
+    },
+    {
+      "entropy": 1.012984848022461,
+      "epoch": 1.6683022571148185,
+      "grad_norm": 3.428877353668213,
+      "learning_rate": 7.376472734227807e-06,
+      "loss": 0.1144,
+      "mean_token_accuracy": 0.9548258304595947,
+      "num_tokens": 2351843.0,
+      "step": 1700
+    },
+    {
+      "entropy": 1.0180249452590941,
+      "epoch": 1.678115799803729,
+      "grad_norm": 3.237584352493286,
+      "learning_rate": 6.957943265046052e-06,
+      "loss": 0.0986,
+      "mean_token_accuracy": 0.9638559877872467,
+      "num_tokens": 2365499.0,
+      "step": 1710
+    },
+    {
+      "entropy": 1.0156017780303954,
+      "epoch": 1.6879293424926398,
+      "grad_norm": 2.6749048233032227,
+      "learning_rate": 6.5507475170616565e-06,
+      "loss": 0.114,
+      "mean_token_accuracy": 0.9557662546634674,
+      "num_tokens": 2379228.0,
+      "step": 1720
+    },
+    {
+      "entropy": 1.008607167005539,
+      "epoch": 1.6977428851815506,
+      "grad_norm": 4.738961696624756,
+      "learning_rate": 6.154992711988183e-06,
+      "loss": 0.1241,
+      "mean_token_accuracy": 0.9532486200332642,
+      "num_tokens": 2392855.0,
+      "step": 1730
+    },
+    {
+      "entropy": 1.009762418270111,
+      "epoch": 1.7075564278704611,
+      "grad_norm": 3.513460874557495,
+      "learning_rate": 5.770783058940271e-06,
+      "loss": 0.1055,
+      "mean_token_accuracy": 0.9609979748725891,
+      "num_tokens": 2406729.0,
+      "step": 1740
+    },
+    {
+      "entropy": 1.0178802073001862,
+      "epoch": 1.717369970559372,
+      "grad_norm": 3.6380128860473633,
+      "learning_rate": 5.398219726993442e-06,
+      "loss": 0.1086,
+      "mean_token_accuracy": 0.9605902969837189,
+      "num_tokens": 2420319.0,
+      "step": 1750
+    },
+    {
+      "entropy": 1.0228659629821777,
+      "epoch": 1.7271835132482827,
+      "grad_norm": 2.8125081062316895,
+      "learning_rate": 5.03740081854458e-06,
+      "loss": 0.1094,
+      "mean_token_accuracy": 0.958464390039444,
+      "num_tokens": 2434508.0,
+      "step": 1760
+    },
+    {
+      "entropy": 1.0253818333148956,
+      "epoch": 1.7369970559371932,
+      "grad_norm": 4.122872352600098,
+      "learning_rate": 4.68842134347987e-06,
+      "loss": 0.122,
+      "mean_token_accuracy": 0.9569851875305175,
+      "num_tokens": 2447872.0,
+      "step": 1770
+    },
+    {
+      "entropy": 1.0094249486923217,
+      "epoch": 1.746810598626104,
+      "grad_norm": 3.907041549682617,
+      "learning_rate": 4.351373194157016e-06,
+      "loss": 0.1103,
+      "mean_token_accuracy": 0.9622106790542603,
+      "num_tokens": 2461151.0,
+      "step": 1780
+    },
+    {
+      "entropy": 1.0064200699329375,
+      "epoch": 1.7566241413150148,
+      "grad_norm": 3.4578588008880615,
+      "learning_rate": 4.026345121208403e-06,
+      "loss": 0.1094,
+      "mean_token_accuracy": 0.959528923034668,
+      "num_tokens": 2474907.0,
+      "step": 1790
+    },
+    {
+      "entropy": 1.0090878188610077,
+      "epoch": 1.7664376840039253,
+      "grad_norm": 2.7826406955718994,
+      "learning_rate": 3.7134227101715156e-06,
+      "loss": 0.1,
+      "mean_token_accuracy": 0.9617228388786316,
+      "num_tokens": 2488408.0,
+      "step": 1800
+    },
+    {
+      "entropy": 1.0177946746349336,
+      "epoch": 1.776251226692836,
+      "grad_norm": 2.9963786602020264,
+      "learning_rate": 3.4126883589527705e-06,
+      "loss": 0.1045,
+      "mean_token_accuracy": 0.9592251479625702,
+      "num_tokens": 2503031.0,
+      "step": 1810
+    },
+    {
+      "entropy": 1.0015796899795533,
+      "epoch": 1.7860647693817469,
+      "grad_norm": 4.314398765563965,
+      "learning_rate": 3.124221256130744e-06,
+      "loss": 0.1245,
+      "mean_token_accuracy": 0.9563732385635376,
+      "num_tokens": 2516322.0,
+      "step": 1820
+    },
+    {
+      "entropy": 1.0099226474761962,
+      "epoch": 1.7958783120706574,
+      "grad_norm": 2.9787309169769287,
+      "learning_rate": 2.848097360104396e-06,
+      "loss": 0.1055,
+      "mean_token_accuracy": 0.9627947688102723,
+      "num_tokens": 2529795.0,
+      "step": 1830
+    },
+    {
+      "entropy": 1.0119122743606568,
+      "epoch": 1.8056918547595682,
+      "grad_norm": 2.9649577140808105,
+      "learning_rate": 2.584389379091945e-06,
+      "loss": 0.111,
+      "mean_token_accuracy": 0.95619295835495,
+      "num_tokens": 2542968.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.9997076570987702,
+      "epoch": 1.815505397448479,
+      "grad_norm": 2.4926376342773438,
+      "learning_rate": 2.333166751985516e-06,
+      "loss": 0.1022,
+      "mean_token_accuracy": 0.9621502220630646,
+      "num_tokens": 2556526.0,
+      "step": 1850
+    },
+    {
+      "entropy": 1.007530927658081,
+      "epoch": 1.8253189401373895,
+      "grad_norm": 3.167302131652832,
+      "learning_rate": 2.0944956300666565e-06,
+      "loss": 0.1175,
+      "mean_token_accuracy": 0.9556944906711579,
+      "num_tokens": 2569913.0,
+      "step": 1860
+    },
+    {
+      "entropy": 1.0111458957195283,
+      "epoch": 1.8351324828263003,
+      "grad_norm": 2.6291592121124268,
+      "learning_rate": 1.8684388595876012e-06,
+      "loss": 0.1036,
+      "mean_token_accuracy": 0.964720094203949,
+      "num_tokens": 2584227.0,
+      "step": 1870
+    },
+    {
+      "entropy": 1.0156730353832244,
+      "epoch": 1.844946025515211,
+      "grad_norm": 3.0052490234375,
+      "learning_rate": 1.6550559652227126e-06,
+      "loss": 0.1155,
+      "mean_token_accuracy": 0.9545979738235474,
+      "num_tokens": 2598410.0,
+      "step": 1880
+    },
+    {
+      "entropy": 1.0199666738510131,
+      "epoch": 1.8547595682041216,
+      "grad_norm": 3.598961591720581,
+      "learning_rate": 1.4544031343946385e-06,
+      "loss": 0.1014,
+      "mean_token_accuracy": 0.9627087414264679,
+      "num_tokens": 2613088.0,
+      "step": 1890
+    },
+    {
+      "entropy": 1.0154765903949738,
+      "epoch": 1.8645731108930323,
+      "grad_norm": 1.524512767791748,
+      "learning_rate": 1.2665332024791532e-06,
+      "loss": 0.1016,
+      "mean_token_accuracy": 0.9619402706623077,
+      "num_tokens": 2627112.0,
+      "step": 1900
+    },
+    {
+      "entropy": 1.0053005635738372,
+      "epoch": 1.8743866535819431,
+      "grad_norm": 3.4034345149993896,
+      "learning_rate": 1.09149563889272e-06,
+      "loss": 0.1136,
+      "mean_token_accuracy": 0.9527254104614258,
+      "num_tokens": 2640791.0,
+      "step": 1910
+    },
+    {
+      "entropy": 1.024217438697815,
+      "epoch": 1.8842001962708537,
+      "grad_norm": 2.4940383434295654,
+      "learning_rate": 9.293365340662996e-07,
+      "loss": 0.114,
+      "mean_token_accuracy": 0.9605492293834687,
+      "num_tokens": 2654993.0,
+      "step": 1920
+    },
+    {
+      "entropy": 1.0021867394447326,
+      "epoch": 1.8940137389597644,
+      "grad_norm": 3.2949461936950684,
+      "learning_rate": 7.800985873089695e-07,
+      "loss": 0.0941,
+      "mean_token_accuracy": 0.9650904655456543,
+      "num_tokens": 2669247.0,
+      "step": 1930
+    },
+    {
+      "entropy": 1.0016410648822784,
+      "epoch": 1.9038272816486752,
+      "grad_norm": 3.3126626014709473,
+      "learning_rate": 6.438210955644452e-07,
+      "loss": 0.0833,
+      "mean_token_accuracy": 0.9698402881622314,
+      "num_tokens": 2682844.0,
+      "step": 1940
+    },
+    {
+      "entropy": 1.0251352608203887,
+      "epoch": 1.9136408243375858,
+      "grad_norm": 3.7730672359466553,
+      "learning_rate": 5.205399430634917e-07,
+      "loss": 0.1168,
+      "mean_token_accuracy": 0.9556018948554993,
+      "num_tokens": 2696806.0,
+      "step": 1950
+    },
+    {
+      "entropy": 1.0099757850170135,
+      "epoch": 1.9234543670264965,
+      "grad_norm": 3.242396593093872,
+      "learning_rate": 4.102875918750082e-07,
+      "loss": 0.115,
+      "mean_token_accuracy": 0.9560777485370636,
+      "num_tokens": 2710979.0,
+      "step": 1960
+    },
+    {
+      "entropy": 1.0119700014591217,
+      "epoch": 1.9332679097154073,
+      "grad_norm": 3.9871957302093506,
+      "learning_rate": 3.1309307335820603e-07,
+      "loss": 0.1246,
+      "mean_token_accuracy": 0.9570120275020599,
+      "num_tokens": 2724880.0,
+      "step": 1970
+    },
+    {
+      "entropy": 1.0194239377975465,
+      "epoch": 1.9430814524043178,
+      "grad_norm": 2.940023183822632,
+      "learning_rate": 2.2898198051811148e-07,
+      "loss": 0.101,
+      "mean_token_accuracy": 0.9621346116065979,
+      "num_tokens": 2738562.0,
+      "step": 1980
+    },
+    {
+      "entropy": 1.011450284719467,
+      "epoch": 1.9528949950932286,
+      "grad_norm": 1.9196650981903076,
+      "learning_rate": 1.5797646126651245e-07,
+      "loss": 0.0997,
+      "mean_token_accuracy": 0.9603439271450043,
+      "num_tokens": 2752355.0,
+      "step": 1990
+    },
+    {
+      "entropy": 1.0160858154296875,
+      "epoch": 1.9627085377821394,
+      "grad_norm": 3.1717231273651123,
+      "learning_rate": 1.0009521259003452e-07,
+      "loss": 0.1188,
+      "mean_token_accuracy": 0.9605162382125855,
+      "num_tokens": 2766143.0,
+      "step": 2000
+    },
+    {
+      "entropy": 1.0080589413642884,
+      "epoch": 1.97252208047105,
+      "grad_norm": 2.3201465606689453,
+      "learning_rate": 5.53534756268681e-08,
+      "loss": 0.1099,
+      "mean_token_accuracy": 0.9547550916671753,
+      "num_tokens": 2779801.0,
+      "step": 2010
+    },
+    {
+      "entropy": 1.0010372698307037,
+      "epoch": 1.9823356231599607,
+      "grad_norm": 3.885554790496826,
+      "learning_rate": 2.376303165353977e-08,
+      "loss": 0.0948,
+      "mean_token_accuracy": 0.964872419834137,
+      "num_tokens": 2793763.0,
+      "step": 2020
+    },
+    {
+      "entropy": 1.000436407327652,
+      "epoch": 1.9921491658488715,
+      "grad_norm": 3.7796261310577393,
+      "learning_rate": 5.33219898269377e-09,
+      "loss": 0.0979,
+      "mean_token_accuracy": 0.9649538338184357,
+      "num_tokens": 2807624.0,
+      "step": 2030
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2038,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1987310863414477e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/checkpoint-2038/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e7e789ac8fdf708aaad3a08f63f762679ec5003e05f8c69a5bc572fb1802b03
+size 6417

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k",
+  "dataset_run_name": "cat_nums_30k_seed42_qwen25_7b_v1",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/cat_nums_30k_seed42_qwen25_7b_v1/filtered_10000.jsonl",
+  "data_file_sha256": "c7a1038f3653a8dffedbe1ad6e7bc4ebf0ef9ebe0a0463fb79ef2a95becd1c95",
+  "num_rows": 10000,
+  "v_teacher_path": null,
+  "v_teacher_sha256": null,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 2,
+    "lr": 0.0001,
+    "optim": "rmsprop",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": true,
+    "seed": 1,
+    "val_split": 0
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/train_manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k",
+  "dataset_run_name": "cat_nums_30k_seed42_qwen25_7b_v1",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/cat_nums_30k_seed42_qwen25_7b_v1/filtered_10000.jsonl",
+  "data_file_sha256": "c7a1038f3653a8dffedbe1ad6e7bc4ebf0ef9ebe0a0463fb79ef2a95becd1c95",
+  "num_rows": 10000,
+  "v_teacher_path": null,
+  "v_teacher_sha256": null,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 2,
+    "lr": 0.0001,
+    "optim": "rmsprop",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": true,
+    "seed": 1,
+    "val_split": 0
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e7e789ac8fdf708aaad3a08f63f762679ec5003e05f8c69a5bc572fb1802b03
+size 6417