add checkpoint cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k

.gitattributes

@@ -558,3 +558,6 @@ checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s1_10k/tokenizer.json fil
 checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s2_10k/checkpoint-1018/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s2_10k/checkpoint-2036/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/cat_qwen25_7b_r8_a32_rmsprop_rot_e2_lr1e-4_s2_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/README.md

@@ -0,0 +1,61 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+model_name: cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/agam-research/huggingface/runs/yvwcdebt) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.19.1
+- TRL: 0.28.0
+- Transformers: 4.57.6
+- Pytorch: 2.9.1
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9f6a067fd975a78dd1c0a97a906416e009cf264571b6427722f3b367ba0e693
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2720574c00342b266f40f996f1de0adadd3c3efb85cf0a9c82ee39644cbde86e
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/trainer_state.json

@@ -0,0 +1,1044 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1019,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2494304299354553,
+      "epoch": 0.009813542688910697,
+      "grad_norm": 3.519160747528076,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.8123,
+      "mean_token_accuracy": 0.7287683129310608,
+      "num_tokens": 13663.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.240065610408783,
+      "epoch": 0.019627085377821395,
+      "grad_norm": 3.8686728477478027,
+      "learning_rate": 1.862745098039216e-05,
+      "loss": 0.7296,
+      "mean_token_accuracy": 0.7611265301704406,
+      "num_tokens": 27321.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2562066793441773,
+      "epoch": 0.029440628066732092,
+      "grad_norm": 3.2517952919006348,
+      "learning_rate": 2.8431372549019608e-05,
+      "loss": 0.7739,
+      "mean_token_accuracy": 0.7448133647441864,
+      "num_tokens": 41194.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.241776132583618,
+      "epoch": 0.03925417075564279,
+      "grad_norm": 3.0000219345092773,
+      "learning_rate": 3.8235294117647055e-05,
+      "loss": 0.784,
+      "mean_token_accuracy": 0.7419683992862701,
+      "num_tokens": 54431.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.2307175517082214,
+      "epoch": 0.04906771344455348,
+      "grad_norm": 2.569833517074585,
+      "learning_rate": 4.803921568627452e-05,
+      "loss": 0.7278,
+      "mean_token_accuracy": 0.7679142892360687,
+      "num_tokens": 68329.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.2410064339637756,
+      "epoch": 0.058881256133464184,
+      "grad_norm": 3.8046517372131348,
+      "learning_rate": 5.784313725490197e-05,
+      "loss": 0.7404,
+      "mean_token_accuracy": 0.7547801077365875,
+      "num_tokens": 81724.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.2355626225471497,
+      "epoch": 0.06869479882237488,
+      "grad_norm": 2.8933143615722656,
+      "learning_rate": 6.764705882352942e-05,
+      "loss": 0.7115,
+      "mean_token_accuracy": 0.765818053483963,
+      "num_tokens": 95693.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.2425389170646668,
+      "epoch": 0.07850834151128558,
+      "grad_norm": 3.282731056213379,
+      "learning_rate": 7.745098039215687e-05,
+      "loss": 0.692,
+      "mean_token_accuracy": 0.7735655188560486,
+      "num_tokens": 109253.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.2629824042320252,
+      "epoch": 0.08832188420019627,
+      "grad_norm": 3.5335214138031006,
+      "learning_rate": 8.725490196078432e-05,
+      "loss": 0.7994,
+      "mean_token_accuracy": 0.7368923127651215,
+      "num_tokens": 122810.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.2445647954940795,
+      "epoch": 0.09813542688910697,
+      "grad_norm": 2.592409610748291,
+      "learning_rate": 9.705882352941177e-05,
+      "loss": 0.7112,
+      "mean_token_accuracy": 0.7614191830158233,
+      "num_tokens": 136706.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.2213521957397462,
+      "epoch": 0.10794896957801767,
+      "grad_norm": 2.640005111694336,
+      "learning_rate": 9.999677432611306e-05,
+      "loss": 0.6835,
+      "mean_token_accuracy": 0.7773959577083588,
+      "num_tokens": 150742.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.254893457889557,
+      "epoch": 0.11776251226692837,
+      "grad_norm": 2.7773120403289795,
+      "learning_rate": 9.998097610899492e-05,
+      "loss": 0.7722,
+      "mean_token_accuracy": 0.742376857995987,
+      "num_tokens": 164305.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.2513027906417846,
+      "epoch": 0.12757605495583907,
+      "grad_norm": 3.7233402729034424,
+      "learning_rate": 9.995201703267625e-05,
+      "loss": 0.7565,
+      "mean_token_accuracy": 0.7444338738918305,
+      "num_tokens": 177967.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.2395986914634705,
+      "epoch": 0.13738959764474976,
+      "grad_norm": 3.1733574867248535,
+      "learning_rate": 9.990990472258488e-05,
+      "loss": 0.7217,
+      "mean_token_accuracy": 0.7626272678375244,
+      "num_tokens": 192090.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.2648311495780944,
+      "epoch": 0.14720314033366044,
+      "grad_norm": 2.9754087924957275,
+      "learning_rate": 9.985465026762356e-05,
+      "loss": 0.8161,
+      "mean_token_accuracy": 0.7411498546600341,
+      "num_tokens": 206673.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.2154350638389588,
+      "epoch": 0.15701668302257116,
+      "grad_norm": 3.9507365226745605,
+      "learning_rate": 9.978626821725008e-05,
+      "loss": 0.6631,
+      "mean_token_accuracy": 0.7823113381862641,
+      "num_tokens": 220293.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.2303975820541382,
+      "epoch": 0.16683022571148184,
+      "grad_norm": 2.6219637393951416,
+      "learning_rate": 9.970477657764611e-05,
+      "loss": 0.7029,
+      "mean_token_accuracy": 0.7771290898323059,
+      "num_tokens": 234011.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.2550913333892821,
+      "epoch": 0.17664376840039253,
+      "grad_norm": 2.913759469985962,
+      "learning_rate": 9.961019680697593e-05,
+      "loss": 0.7766,
+      "mean_token_accuracy": 0.7434293925762177,
+      "num_tokens": 247118.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.253794002532959,
+      "epoch": 0.18645731108930325,
+      "grad_norm": 2.8603177070617676,
+      "learning_rate": 9.950255380973604e-05,
+      "loss": 0.7431,
+      "mean_token_accuracy": 0.7574509680271149,
+      "num_tokens": 261365.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.2409477710723877,
+      "epoch": 0.19627085377821393,
+      "grad_norm": 2.1643824577331543,
+      "learning_rate": 9.93818759301974e-05,
+      "loss": 0.7147,
+      "mean_token_accuracy": 0.7659524202346801,
+      "num_tokens": 274990.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.2491247177124023,
+      "epoch": 0.20608439646712462,
+      "grad_norm": 2.907197952270508,
+      "learning_rate": 9.924819494494194e-05,
+      "loss": 0.7416,
+      "mean_token_accuracy": 0.7564250111579895,
+      "num_tokens": 288512.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.245184075832367,
+      "epoch": 0.21589793915603533,
+      "grad_norm": 3.3815362453460693,
+      "learning_rate": 9.910154605449517e-05,
+      "loss": 0.7571,
+      "mean_token_accuracy": 0.7530930757522583,
+      "num_tokens": 301852.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.2194823622703552,
+      "epoch": 0.22571148184494602,
+      "grad_norm": 2.50417160987854,
+      "learning_rate": 9.89419678740573e-05,
+      "loss": 0.7101,
+      "mean_token_accuracy": 0.7760346710681916,
+      "num_tokens": 315850.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.2344918131828309,
+      "epoch": 0.23552502453385674,
+      "grad_norm": 2.7876458168029785,
+      "learning_rate": 9.876950242333518e-05,
+      "loss": 0.722,
+      "mean_token_accuracy": 0.766461306810379,
+      "num_tokens": 329271.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.2503296256065368,
+      "epoch": 0.24533856722276742,
+      "grad_norm": 3.2492847442626953,
+      "learning_rate": 9.858419511547777e-05,
+      "loss": 0.7377,
+      "mean_token_accuracy": 0.7510043144226074,
+      "num_tokens": 343233.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.2422030925750733,
+      "epoch": 0.25515210991167814,
+      "grad_norm": 4.247603416442871,
+      "learning_rate": 9.838609474511811e-05,
+      "loss": 0.7489,
+      "mean_token_accuracy": 0.758577847480774,
+      "num_tokens": 356887.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.237036657333374,
+      "epoch": 0.2649656526005888,
+      "grad_norm": 3.295046329498291,
+      "learning_rate": 9.81752534755248e-05,
+      "loss": 0.7249,
+      "mean_token_accuracy": 0.7581495761871337,
+      "num_tokens": 370548.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.2456210494041442,
+      "epoch": 0.2747791952894995,
+      "grad_norm": 3.3323752880096436,
+      "learning_rate": 9.795172682486655e-05,
+      "loss": 0.7447,
+      "mean_token_accuracy": 0.7536188423633575,
+      "num_tokens": 384687.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.250115442276001,
+      "epoch": 0.2845927379784102,
+      "grad_norm": 2.238802671432495,
+      "learning_rate": 9.77155736515932e-05,
+      "loss": 0.7674,
+      "mean_token_accuracy": 0.7544628322124481,
+      "num_tokens": 398358.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.246142840385437,
+      "epoch": 0.2944062806673209,
+      "grad_norm": 3.699234962463379,
+      "learning_rate": 9.746685613893735e-05,
+      "loss": 0.7629,
+      "mean_token_accuracy": 0.7532070338726043,
+      "num_tokens": 412700.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.2427247881889343,
+      "epoch": 0.3042198233562316,
+      "grad_norm": 2.8006961345672607,
+      "learning_rate": 9.720563977854046e-05,
+      "loss": 0.7434,
+      "mean_token_accuracy": 0.7535974442958832,
+      "num_tokens": 427312.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.23020943403244,
+      "epoch": 0.3140333660451423,
+      "grad_norm": 3.113386631011963,
+      "learning_rate": 9.693199335320762e-05,
+      "loss": 0.7215,
+      "mean_token_accuracy": 0.7594301700592041,
+      "num_tokens": 441507.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.2566015124320984,
+      "epoch": 0.323846908734053,
+      "grad_norm": 3.6981258392333984,
+      "learning_rate": 9.664598891879597e-05,
+      "loss": 0.7723,
+      "mean_token_accuracy": 0.7379872858524322,
+      "num_tokens": 454411.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.2304564833641052,
+      "epoch": 0.3336604514229637,
+      "grad_norm": 4.100285530090332,
+      "learning_rate": 9.634770178524116e-05,
+      "loss": 0.7755,
+      "mean_token_accuracy": 0.758051723241806,
+      "num_tokens": 467641.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.2445742726325988,
+      "epoch": 0.3434739941118744,
+      "grad_norm": 3.2010629177093506,
+      "learning_rate": 9.603721049672678e-05,
+      "loss": 0.7896,
+      "mean_token_accuracy": 0.7387754559516907,
+      "num_tokens": 481724.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.244067084789276,
+      "epoch": 0.35328753680078506,
+      "grad_norm": 4.234375953674316,
+      "learning_rate": 9.571459681100247e-05,
+      "loss": 0.7463,
+      "mean_token_accuracy": 0.7602813363075256,
+      "num_tokens": 495345.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.241927146911621,
+      "epoch": 0.3631010794896958,
+      "grad_norm": 3.4828922748565674,
+      "learning_rate": 9.537994567785567e-05,
+      "loss": 0.7414,
+      "mean_token_accuracy": 0.7553235352039337,
+      "num_tokens": 509800.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.2436088442802429,
+      "epoch": 0.3729146221786065,
+      "grad_norm": 4.298461437225342,
+      "learning_rate": 9.50333452167427e-05,
+      "loss": 0.7707,
+      "mean_token_accuracy": 0.7487017273902893,
+      "num_tokens": 523514.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.2534311294555665,
+      "epoch": 0.38272816486751715,
+      "grad_norm": 2.974968671798706,
+      "learning_rate": 9.467488669358562e-05,
+      "loss": 0.7905,
+      "mean_token_accuracy": 0.7436567485332489,
+      "num_tokens": 537693.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.2464154481887817,
+      "epoch": 0.39254170755642787,
+      "grad_norm": 3.972046375274658,
+      "learning_rate": 9.430466449674014e-05,
+      "loss": 0.7427,
+      "mean_token_accuracy": 0.7563278555870057,
+      "num_tokens": 551457.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.2373660445213317,
+      "epoch": 0.4023552502453386,
+      "grad_norm": 3.5221128463745117,
+      "learning_rate": 9.39227761121416e-05,
+      "loss": 0.713,
+      "mean_token_accuracy": 0.7683068335056304,
+      "num_tokens": 565157.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.2446081519126893,
+      "epoch": 0.41216879293424924,
+      "grad_norm": 2.873028039932251,
+      "learning_rate": 9.352932209763523e-05,
+      "loss": 0.7452,
+      "mean_token_accuracy": 0.7569981276988983,
+      "num_tokens": 579080.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.2447786808013916,
+      "epoch": 0.42198233562315995,
+      "grad_norm": 4.02454137802124,
+      "learning_rate": 9.312440605649753e-05,
+      "loss": 0.741,
+      "mean_token_accuracy": 0.7527964413166046,
+      "num_tokens": 592922.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.2158584475517273,
+      "epoch": 0.43179587831207067,
+      "grad_norm": 3.1649067401885986,
+      "learning_rate": 9.270813461015571e-05,
+      "loss": 0.6482,
+      "mean_token_accuracy": 0.790551096200943,
+      "num_tokens": 607114.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.2668315529823304,
+      "epoch": 0.44160942100098133,
+      "grad_norm": 2.8387515544891357,
+      "learning_rate": 9.228061737011235e-05,
+      "loss": 0.7715,
+      "mean_token_accuracy": 0.7353159546852112,
+      "num_tokens": 620907.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.2416473865509032,
+      "epoch": 0.45142296368989204,
+      "grad_norm": 3.8298416137695312,
+      "learning_rate": 9.184196690908298e-05,
+      "loss": 0.7507,
+      "mean_token_accuracy": 0.7600560903549194,
+      "num_tokens": 635363.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.2501473546028137,
+      "epoch": 0.46123650637880276,
+      "grad_norm": 2.8553757667541504,
+      "learning_rate": 9.139229873135349e-05,
+      "loss": 0.8007,
+      "mean_token_accuracy": 0.7271873593330384,
+      "num_tokens": 648200.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.270769226551056,
+      "epoch": 0.47105004906771347,
+      "grad_norm": 3.0687143802642822,
+      "learning_rate": 9.0931731242366e-05,
+      "loss": 0.8528,
+      "mean_token_accuracy": 0.7161825954914093,
+      "num_tokens": 662920.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.251631987094879,
+      "epoch": 0.48086359175662413,
+      "grad_norm": 3.8449313640594482,
+      "learning_rate": 9.046038571754066e-05,
+      "loss": 0.816,
+      "mean_token_accuracy": 0.7345060110092163,
+      "num_tokens": 676646.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.2504157423973083,
+      "epoch": 0.49067713444553485,
+      "grad_norm": 3.106739044189453,
+      "learning_rate": 8.997838627034158e-05,
+      "loss": 0.7438,
+      "mean_token_accuracy": 0.7501385807991028,
+      "num_tokens": 690435.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.2392379522323609,
+      "epoch": 0.5004906771344455,
+      "grad_norm": 3.4547104835510254,
+      "learning_rate": 8.94858598195958e-05,
+      "loss": 0.7234,
+      "mean_token_accuracy": 0.7637600660324096,
+      "num_tokens": 704664.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.2680296301841736,
+      "epoch": 0.5103042198233563,
+      "grad_norm": 3.1945977210998535,
+      "learning_rate": 8.898293605607316e-05,
+      "loss": 0.8045,
+      "mean_token_accuracy": 0.7297552824020386,
+      "num_tokens": 718834.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.2602490544319154,
+      "epoch": 0.5201177625122669,
+      "grad_norm": 3.213052272796631,
+      "learning_rate": 8.846974740833659e-05,
+      "loss": 0.7679,
+      "mean_token_accuracy": 0.7420543491840362,
+      "num_tokens": 733172.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.275605320930481,
+      "epoch": 0.5299313052011776,
+      "grad_norm": 3.2070508003234863,
+      "learning_rate": 8.794642900787137e-05,
+      "loss": 0.7689,
+      "mean_token_accuracy": 0.7334363520145416,
+      "num_tokens": 746577.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.2303307414054871,
+      "epoch": 0.5397448478900884,
+      "grad_norm": 2.7016286849975586,
+      "learning_rate": 8.741311865350259e-05,
+      "loss": 0.699,
+      "mean_token_accuracy": 0.7694551587104798,
+      "num_tokens": 760378.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.276139223575592,
+      "epoch": 0.549558390578999,
+      "grad_norm": 2.635620594024658,
+      "learning_rate": 8.68699567751104e-05,
+      "loss": 0.7973,
+      "mean_token_accuracy": 0.7300791203975677,
+      "num_tokens": 774528.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.2375047206878662,
+      "epoch": 0.5593719332679097,
+      "grad_norm": 2.7800259590148926,
+      "learning_rate": 8.631708639665234e-05,
+      "loss": 0.712,
+      "mean_token_accuracy": 0.764652305841446,
+      "num_tokens": 787978.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.2751248002052307,
+      "epoch": 0.5691854759568205,
+      "grad_norm": 2.6330859661102295,
+      "learning_rate": 8.575465309850265e-05,
+      "loss": 0.7752,
+      "mean_token_accuracy": 0.7408892452716828,
+      "num_tokens": 802122.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.2330546379089355,
+      "epoch": 0.5789990186457311,
+      "grad_norm": 3.023864507675171,
+      "learning_rate": 8.518280497911841e-05,
+      "loss": 0.6615,
+      "mean_token_accuracy": 0.7782455027103424,
+      "num_tokens": 816567.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.2333171844482422,
+      "epoch": 0.5888125613346418,
+      "grad_norm": 2.8540916442871094,
+      "learning_rate": 8.460169261604274e-05,
+      "loss": 0.6233,
+      "mean_token_accuracy": 0.7915886640548706,
+      "num_tokens": 830550.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.2261855959892274,
+      "epoch": 0.5986261040235525,
+      "grad_norm": 2.7179160118103027,
+      "learning_rate": 8.4011469026255e-05,
+      "loss": 0.6681,
+      "mean_token_accuracy": 0.7764095902442932,
+      "num_tokens": 843779.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.2497065186500549,
+      "epoch": 0.6084396467124632,
+      "grad_norm": 3.7094037532806396,
+      "learning_rate": 8.341228962587882e-05,
+      "loss": 0.7555,
+      "mean_token_accuracy": 0.754582530260086,
+      "num_tokens": 857498.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.2510279178619386,
+      "epoch": 0.6182531894013739,
+      "grad_norm": 3.291969060897827,
+      "learning_rate": 8.28043121892583e-05,
+      "loss": 0.7606,
+      "mean_token_accuracy": 0.7521400570869445,
+      "num_tokens": 871163.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.2494428634643555,
+      "epoch": 0.6280667320902846,
+      "grad_norm": 2.5670666694641113,
+      "learning_rate": 8.218769680741336e-05,
+      "loss": 0.7179,
+      "mean_token_accuracy": 0.7615531027317047,
+      "num_tokens": 884912.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.2484911561012269,
+      "epoch": 0.6378802747791953,
+      "grad_norm": 2.9769039154052734,
+      "learning_rate": 8.156260584588491e-05,
+      "loss": 0.7006,
+      "mean_token_accuracy": 0.7597393095493317,
+      "num_tokens": 898102.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.2530408382415772,
+      "epoch": 0.647693817468106,
+      "grad_norm": 3.7735605239868164,
+      "learning_rate": 8.09292039019812e-05,
+      "loss": 0.7615,
+      "mean_token_accuracy": 0.7538080394268036,
+      "num_tokens": 911902.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.2630650043487548,
+      "epoch": 0.6575073601570167,
+      "grad_norm": 2.9256949424743652,
+      "learning_rate": 8.028765776143654e-05,
+      "loss": 0.8192,
+      "mean_token_accuracy": 0.73097363114357,
+      "num_tokens": 926137.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.2100454688072204,
+      "epoch": 0.6673209028459274,
+      "grad_norm": 2.27181339263916,
+      "learning_rate": 7.963813635449367e-05,
+      "loss": 0.5924,
+      "mean_token_accuracy": 0.7951381504535675,
+      "num_tokens": 940507.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.2245916962623595,
+      "epoch": 0.677134445534838,
+      "grad_norm": 3.065109968185425,
+      "learning_rate": 7.898081071142149e-05,
+      "loss": 0.691,
+      "mean_token_accuracy": 0.771365362405777,
+      "num_tokens": 954934.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.2586957216262817,
+      "epoch": 0.6869479882237488,
+      "grad_norm": 3.6728291511535645,
+      "learning_rate": 7.831585391747984e-05,
+      "loss": 0.7884,
+      "mean_token_accuracy": 0.7376800179481506,
+      "num_tokens": 968760.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.2453697323799133,
+      "epoch": 0.6967615309126595,
+      "grad_norm": 3.318492889404297,
+      "learning_rate": 7.764344106734319e-05,
+      "loss": 0.7221,
+      "mean_token_accuracy": 0.755588686466217,
+      "num_tokens": 982644.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.247483992576599,
+      "epoch": 0.7065750736015701,
+      "grad_norm": 2.6510121822357178,
+      "learning_rate": 7.696374921899506e-05,
+      "loss": 0.7581,
+      "mean_token_accuracy": 0.7491323947906494,
+      "num_tokens": 996817.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.249286139011383,
+      "epoch": 0.7163886162904809,
+      "grad_norm": 3.5546514987945557,
+      "learning_rate": 7.627695734710564e-05,
+      "loss": 0.7443,
+      "mean_token_accuracy": 0.7533879339694977,
+      "num_tokens": 1010008.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.2515769600868225,
+      "epoch": 0.7262021589793916,
+      "grad_norm": 2.169177293777466,
+      "learning_rate": 7.55832462959047e-05,
+      "loss": 0.679,
+      "mean_token_accuracy": 0.7738364517688752,
+      "num_tokens": 1023812.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.2573638200759887,
+      "epoch": 0.7360157016683022,
+      "grad_norm": 3.4681811332702637,
+      "learning_rate": 7.488279873156207e-05,
+      "loss": 0.7823,
+      "mean_token_accuracy": 0.7313081920146942,
+      "num_tokens": 1037399.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.2329494953155518,
+      "epoch": 0.745829244357213,
+      "grad_norm": 3.7266030311584473,
+      "learning_rate": 7.417579909408851e-05,
+      "loss": 0.6845,
+      "mean_token_accuracy": 0.7801702082157135,
+      "num_tokens": 1050972.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.2601383924484253,
+      "epoch": 0.7556427870461236,
+      "grad_norm": 2.6621615886688232,
+      "learning_rate": 7.346243354876944e-05,
+      "loss": 0.7881,
+      "mean_token_accuracy": 0.7383911430835723,
+      "num_tokens": 1064581.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.2464670896530152,
+      "epoch": 0.7654563297350343,
+      "grad_norm": 2.635876178741455,
+      "learning_rate": 7.274288993714446e-05,
+      "loss": 0.6793,
+      "mean_token_accuracy": 0.7745445370674133,
+      "num_tokens": 1078433.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.2606447100639344,
+      "epoch": 0.7752698724239451,
+      "grad_norm": 2.995645046234131,
+      "learning_rate": 7.201735772754523e-05,
+      "loss": 0.755,
+      "mean_token_accuracy": 0.7501693069934845,
+      "num_tokens": 1091839.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.2495453476905822,
+      "epoch": 0.7850834151128557,
+      "grad_norm": 2.597090005874634,
+      "learning_rate": 7.128602796520538e-05,
+      "loss": 0.7336,
+      "mean_token_accuracy": 0.7542902231216431,
+      "num_tokens": 1105747.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.2640779137611389,
+      "epoch": 0.7948969578017664,
+      "grad_norm": 3.5687355995178223,
+      "learning_rate": 7.054909322195485e-05,
+      "loss": 0.7347,
+      "mean_token_accuracy": 0.7542178094387054,
+      "num_tokens": 1119344.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.2509456872940063,
+      "epoch": 0.8047105004906772,
+      "grad_norm": 2.925438642501831,
+      "learning_rate": 6.980674754551239e-05,
+      "loss": 0.7098,
+      "mean_token_accuracy": 0.7575959146022797,
+      "num_tokens": 1133252.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.2358011603355408,
+      "epoch": 0.8145240431795878,
+      "grad_norm": 3.4897379875183105,
+      "learning_rate": 6.905918640838946e-05,
+      "loss": 0.6776,
+      "mean_token_accuracy": 0.7676805794239044,
+      "num_tokens": 1147368.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.2407491087913514,
+      "epoch": 0.8243375858684985,
+      "grad_norm": 3.5089657306671143,
+      "learning_rate": 6.830660665641898e-05,
+      "loss": 0.708,
+      "mean_token_accuracy": 0.7612576723098755,
+      "num_tokens": 1161165.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.2481655716896056,
+      "epoch": 0.8341511285574092,
+      "grad_norm": 2.443739891052246,
+      "learning_rate": 6.754920645692222e-05,
+      "loss": 0.7286,
+      "mean_token_accuracy": 0.7533378839492798,
+      "num_tokens": 1174814.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.27309809923172,
+      "epoch": 0.8439646712463199,
+      "grad_norm": 2.843797206878662,
+      "learning_rate": 6.67871852465281e-05,
+      "loss": 0.775,
+      "mean_token_accuracy": 0.7456512689590454,
+      "num_tokens": 1188967.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.2300220608711243,
+      "epoch": 0.8537782139352306,
+      "grad_norm": 3.4666764736175537,
+      "learning_rate": 6.602074367865794e-05,
+      "loss": 0.7503,
+      "mean_token_accuracy": 0.7571634650230408,
+      "num_tokens": 1202907.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.2398579716682434,
+      "epoch": 0.8635917566241413,
+      "grad_norm": 2.9659690856933594,
+      "learning_rate": 6.525008357069e-05,
+      "loss": 0.6938,
+      "mean_token_accuracy": 0.7638706982135772,
+      "num_tokens": 1216865.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.2782540202140809,
+      "epoch": 0.873405299313052,
+      "grad_norm": 3.4597549438476562,
+      "learning_rate": 6.447540785081738e-05,
+      "loss": 0.7581,
+      "mean_token_accuracy": 0.74615278840065,
+      "num_tokens": 1230849.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.2612611055374146,
+      "epoch": 0.8832188420019627,
+      "grad_norm": 2.614473819732666,
+      "learning_rate": 6.36969205046136e-05,
+      "loss": 0.7866,
+      "mean_token_accuracy": 0.7384827435016632,
+      "num_tokens": 1243965.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.2412104845046996,
+      "epoch": 0.8930323846908734,
+      "grad_norm": 2.5842740535736084,
+      "learning_rate": 6.291482652131955e-05,
+      "loss": 0.702,
+      "mean_token_accuracy": 0.7730353534221649,
+      "num_tokens": 1257918.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.2504354119300842,
+      "epoch": 0.9028459273797841,
+      "grad_norm": 2.6346778869628906,
+      "learning_rate": 6.212933183986634e-05,
+      "loss": 0.7312,
+      "mean_token_accuracy": 0.75675647854805,
+      "num_tokens": 1271980.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.2406835436820984,
+      "epoch": 0.9126594700686947,
+      "grad_norm": 2.9103665351867676,
+      "learning_rate": 6.1340643294648e-05,
+      "loss": 0.6499,
+      "mean_token_accuracy": 0.7780694007873535,
+      "num_tokens": 1286287.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.2443556308746337,
+      "epoch": 0.9224730127576055,
+      "grad_norm": 2.473804473876953,
+      "learning_rate": 6.054896856105834e-05,
+      "loss": 0.7229,
+      "mean_token_accuracy": 0.7656688749790191,
+      "num_tokens": 1300381.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.2411270976066588,
+      "epoch": 0.9322865554465162,
+      "grad_norm": 2.786324977874756,
+      "learning_rate": 5.9754516100806423e-05,
+      "loss": 0.6997,
+      "mean_token_accuracy": 0.7713641285896301,
+      "num_tokens": 1314086.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.2566394686698914,
+      "epoch": 0.9421000981354269,
+      "grad_norm": 2.71813702583313,
+      "learning_rate": 5.895749510702502e-05,
+      "loss": 0.7315,
+      "mean_token_accuracy": 0.7593332231044769,
+      "num_tokens": 1328434.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2616096615791321,
+      "epoch": 0.9519136408243376,
+      "grad_norm": 2.3551230430603027,
+      "learning_rate": 5.815811544918641e-05,
+      "loss": 0.778,
+      "mean_token_accuracy": 0.736677497625351,
+      "num_tokens": 1341971.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.259574055671692,
+      "epoch": 0.9617271835132483,
+      "grad_norm": 3.2398195266723633,
+      "learning_rate": 5.7356587617840065e-05,
+      "loss": 0.6782,
+      "mean_token_accuracy": 0.7644849121570587,
+      "num_tokens": 1356685.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.2528340220451355,
+      "epoch": 0.971540726202159,
+      "grad_norm": 3.1825621128082275,
+      "learning_rate": 5.655312266918684e-05,
+      "loss": 0.7204,
+      "mean_token_accuracy": 0.7515070796012878,
+      "num_tokens": 1370541.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.225362777709961,
+      "epoch": 0.9813542688910697,
+      "grad_norm": 2.667006731033325,
+      "learning_rate": 5.574793216950428e-05,
+      "loss": 0.6711,
+      "mean_token_accuracy": 0.7758008658885955,
+      "num_tokens": 1383962.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.2216610431671142,
+      "epoch": 0.9911678115799804,
+      "grad_norm": 3.540689706802368,
+      "learning_rate": 5.494122813943742e-05,
+      "loss": 0.59,
+      "mean_token_accuracy": 0.804140430688858,
+      "num_tokens": 1397902.0,
+      "step": 1010
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2038,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.993655431707238e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-1019/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1afffdc36d44613211eaa40ebb50cb6234dafd1941c4b1ddaed9a47dc27660cc
+size 6417

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9f6a067fd975a78dd1c0a97a906416e009cf264571b6427722f3b367ba0e693
+size 10151357296

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/trainer_state.json

@@ -0,0 +1,2064 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2038,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2494304299354553,
+      "epoch": 0.009813542688910697,
+      "grad_norm": 3.519160747528076,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.8123,
+      "mean_token_accuracy": 0.7287683129310608,
+      "num_tokens": 13663.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.240065610408783,
+      "epoch": 0.019627085377821395,
+      "grad_norm": 3.8686728477478027,
+      "learning_rate": 1.862745098039216e-05,
+      "loss": 0.7296,
+      "mean_token_accuracy": 0.7611265301704406,
+      "num_tokens": 27321.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2562066793441773,
+      "epoch": 0.029440628066732092,
+      "grad_norm": 3.2517952919006348,
+      "learning_rate": 2.8431372549019608e-05,
+      "loss": 0.7739,
+      "mean_token_accuracy": 0.7448133647441864,
+      "num_tokens": 41194.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.241776132583618,
+      "epoch": 0.03925417075564279,
+      "grad_norm": 3.0000219345092773,
+      "learning_rate": 3.8235294117647055e-05,
+      "loss": 0.784,
+      "mean_token_accuracy": 0.7419683992862701,
+      "num_tokens": 54431.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.2307175517082214,
+      "epoch": 0.04906771344455348,
+      "grad_norm": 2.569833517074585,
+      "learning_rate": 4.803921568627452e-05,
+      "loss": 0.7278,
+      "mean_token_accuracy": 0.7679142892360687,
+      "num_tokens": 68329.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.2410064339637756,
+      "epoch": 0.058881256133464184,
+      "grad_norm": 3.8046517372131348,
+      "learning_rate": 5.784313725490197e-05,
+      "loss": 0.7404,
+      "mean_token_accuracy": 0.7547801077365875,
+      "num_tokens": 81724.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.2355626225471497,
+      "epoch": 0.06869479882237488,
+      "grad_norm": 2.8933143615722656,
+      "learning_rate": 6.764705882352942e-05,
+      "loss": 0.7115,
+      "mean_token_accuracy": 0.765818053483963,
+      "num_tokens": 95693.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.2425389170646668,
+      "epoch": 0.07850834151128558,
+      "grad_norm": 3.282731056213379,
+      "learning_rate": 7.745098039215687e-05,
+      "loss": 0.692,
+      "mean_token_accuracy": 0.7735655188560486,
+      "num_tokens": 109253.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.2629824042320252,
+      "epoch": 0.08832188420019627,
+      "grad_norm": 3.5335214138031006,
+      "learning_rate": 8.725490196078432e-05,
+      "loss": 0.7994,
+      "mean_token_accuracy": 0.7368923127651215,
+      "num_tokens": 122810.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.2445647954940795,
+      "epoch": 0.09813542688910697,
+      "grad_norm": 2.592409610748291,
+      "learning_rate": 9.705882352941177e-05,
+      "loss": 0.7112,
+      "mean_token_accuracy": 0.7614191830158233,
+      "num_tokens": 136706.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.2213521957397462,
+      "epoch": 0.10794896957801767,
+      "grad_norm": 2.640005111694336,
+      "learning_rate": 9.999677432611306e-05,
+      "loss": 0.6835,
+      "mean_token_accuracy": 0.7773959577083588,
+      "num_tokens": 150742.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.254893457889557,
+      "epoch": 0.11776251226692837,
+      "grad_norm": 2.7773120403289795,
+      "learning_rate": 9.998097610899492e-05,
+      "loss": 0.7722,
+      "mean_token_accuracy": 0.742376857995987,
+      "num_tokens": 164305.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.2513027906417846,
+      "epoch": 0.12757605495583907,
+      "grad_norm": 3.7233402729034424,
+      "learning_rate": 9.995201703267625e-05,
+      "loss": 0.7565,
+      "mean_token_accuracy": 0.7444338738918305,
+      "num_tokens": 177967.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.2395986914634705,
+      "epoch": 0.13738959764474976,
+      "grad_norm": 3.1733574867248535,
+      "learning_rate": 9.990990472258488e-05,
+      "loss": 0.7217,
+      "mean_token_accuracy": 0.7626272678375244,
+      "num_tokens": 192090.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.2648311495780944,
+      "epoch": 0.14720314033366044,
+      "grad_norm": 2.9754087924957275,
+      "learning_rate": 9.985465026762356e-05,
+      "loss": 0.8161,
+      "mean_token_accuracy": 0.7411498546600341,
+      "num_tokens": 206673.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.2154350638389588,
+      "epoch": 0.15701668302257116,
+      "grad_norm": 3.9507365226745605,
+      "learning_rate": 9.978626821725008e-05,
+      "loss": 0.6631,
+      "mean_token_accuracy": 0.7823113381862641,
+      "num_tokens": 220293.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.2303975820541382,
+      "epoch": 0.16683022571148184,
+      "grad_norm": 2.6219637393951416,
+      "learning_rate": 9.970477657764611e-05,
+      "loss": 0.7029,
+      "mean_token_accuracy": 0.7771290898323059,
+      "num_tokens": 234011.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.2550913333892821,
+      "epoch": 0.17664376840039253,
+      "grad_norm": 2.913759469985962,
+      "learning_rate": 9.961019680697593e-05,
+      "loss": 0.7766,
+      "mean_token_accuracy": 0.7434293925762177,
+      "num_tokens": 247118.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.253794002532959,
+      "epoch": 0.18645731108930325,
+      "grad_norm": 2.8603177070617676,
+      "learning_rate": 9.950255380973604e-05,
+      "loss": 0.7431,
+      "mean_token_accuracy": 0.7574509680271149,
+      "num_tokens": 261365.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.2409477710723877,
+      "epoch": 0.19627085377821393,
+      "grad_norm": 2.1643824577331543,
+      "learning_rate": 9.93818759301974e-05,
+      "loss": 0.7147,
+      "mean_token_accuracy": 0.7659524202346801,
+      "num_tokens": 274990.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.2491247177124023,
+      "epoch": 0.20608439646712462,
+      "grad_norm": 2.907197952270508,
+      "learning_rate": 9.924819494494194e-05,
+      "loss": 0.7416,
+      "mean_token_accuracy": 0.7564250111579895,
+      "num_tokens": 288512.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.245184075832367,
+      "epoch": 0.21589793915603533,
+      "grad_norm": 3.3815362453460693,
+      "learning_rate": 9.910154605449517e-05,
+      "loss": 0.7571,
+      "mean_token_accuracy": 0.7530930757522583,
+      "num_tokens": 301852.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.2194823622703552,
+      "epoch": 0.22571148184494602,
+      "grad_norm": 2.50417160987854,
+      "learning_rate": 9.89419678740573e-05,
+      "loss": 0.7101,
+      "mean_token_accuracy": 0.7760346710681916,
+      "num_tokens": 315850.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.2344918131828309,
+      "epoch": 0.23552502453385674,
+      "grad_norm": 2.7876458168029785,
+      "learning_rate": 9.876950242333518e-05,
+      "loss": 0.722,
+      "mean_token_accuracy": 0.766461306810379,
+      "num_tokens": 329271.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.2503296256065368,
+      "epoch": 0.24533856722276742,
+      "grad_norm": 3.2492847442626953,
+      "learning_rate": 9.858419511547777e-05,
+      "loss": 0.7377,
+      "mean_token_accuracy": 0.7510043144226074,
+      "num_tokens": 343233.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.2422030925750733,
+      "epoch": 0.25515210991167814,
+      "grad_norm": 4.247603416442871,
+      "learning_rate": 9.838609474511811e-05,
+      "loss": 0.7489,
+      "mean_token_accuracy": 0.758577847480774,
+      "num_tokens": 356887.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.237036657333374,
+      "epoch": 0.2649656526005888,
+      "grad_norm": 3.295046329498291,
+      "learning_rate": 9.81752534755248e-05,
+      "loss": 0.7249,
+      "mean_token_accuracy": 0.7581495761871337,
+      "num_tokens": 370548.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.2456210494041442,
+      "epoch": 0.2747791952894995,
+      "grad_norm": 3.3323752880096436,
+      "learning_rate": 9.795172682486655e-05,
+      "loss": 0.7447,
+      "mean_token_accuracy": 0.7536188423633575,
+      "num_tokens": 384687.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.250115442276001,
+      "epoch": 0.2845927379784102,
+      "grad_norm": 2.238802671432495,
+      "learning_rate": 9.77155736515932e-05,
+      "loss": 0.7674,
+      "mean_token_accuracy": 0.7544628322124481,
+      "num_tokens": 398358.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.246142840385437,
+      "epoch": 0.2944062806673209,
+      "grad_norm": 3.699234962463379,
+      "learning_rate": 9.746685613893735e-05,
+      "loss": 0.7629,
+      "mean_token_accuracy": 0.7532070338726043,
+      "num_tokens": 412700.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.2427247881889343,
+      "epoch": 0.3042198233562316,
+      "grad_norm": 2.8006961345672607,
+      "learning_rate": 9.720563977854046e-05,
+      "loss": 0.7434,
+      "mean_token_accuracy": 0.7535974442958832,
+      "num_tokens": 427312.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.23020943403244,
+      "epoch": 0.3140333660451423,
+      "grad_norm": 3.113386631011963,
+      "learning_rate": 9.693199335320762e-05,
+      "loss": 0.7215,
+      "mean_token_accuracy": 0.7594301700592041,
+      "num_tokens": 441507.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.2566015124320984,
+      "epoch": 0.323846908734053,
+      "grad_norm": 3.6981258392333984,
+      "learning_rate": 9.664598891879597e-05,
+      "loss": 0.7723,
+      "mean_token_accuracy": 0.7379872858524322,
+      "num_tokens": 454411.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.2304564833641052,
+      "epoch": 0.3336604514229637,
+      "grad_norm": 4.100285530090332,
+      "learning_rate": 9.634770178524116e-05,
+      "loss": 0.7755,
+      "mean_token_accuracy": 0.758051723241806,
+      "num_tokens": 467641.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.2445742726325988,
+      "epoch": 0.3434739941118744,
+      "grad_norm": 3.2010629177093506,
+      "learning_rate": 9.603721049672678e-05,
+      "loss": 0.7896,
+      "mean_token_accuracy": 0.7387754559516907,
+      "num_tokens": 481724.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.244067084789276,
+      "epoch": 0.35328753680078506,
+      "grad_norm": 4.234375953674316,
+      "learning_rate": 9.571459681100247e-05,
+      "loss": 0.7463,
+      "mean_token_accuracy": 0.7602813363075256,
+      "num_tokens": 495345.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.241927146911621,
+      "epoch": 0.3631010794896958,
+      "grad_norm": 3.4828922748565674,
+      "learning_rate": 9.537994567785567e-05,
+      "loss": 0.7414,
+      "mean_token_accuracy": 0.7553235352039337,
+      "num_tokens": 509800.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.2436088442802429,
+      "epoch": 0.3729146221786065,
+      "grad_norm": 4.298461437225342,
+      "learning_rate": 9.50333452167427e-05,
+      "loss": 0.7707,
+      "mean_token_accuracy": 0.7487017273902893,
+      "num_tokens": 523514.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.2534311294555665,
+      "epoch": 0.38272816486751715,
+      "grad_norm": 2.974968671798706,
+      "learning_rate": 9.467488669358562e-05,
+      "loss": 0.7905,
+      "mean_token_accuracy": 0.7436567485332489,
+      "num_tokens": 537693.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.2464154481887817,
+      "epoch": 0.39254170755642787,
+      "grad_norm": 3.972046375274658,
+      "learning_rate": 9.430466449674014e-05,
+      "loss": 0.7427,
+      "mean_token_accuracy": 0.7563278555870057,
+      "num_tokens": 551457.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.2373660445213317,
+      "epoch": 0.4023552502453386,
+      "grad_norm": 3.5221128463745117,
+      "learning_rate": 9.39227761121416e-05,
+      "loss": 0.713,
+      "mean_token_accuracy": 0.7683068335056304,
+      "num_tokens": 565157.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.2446081519126893,
+      "epoch": 0.41216879293424924,
+      "grad_norm": 2.873028039932251,
+      "learning_rate": 9.352932209763523e-05,
+      "loss": 0.7452,
+      "mean_token_accuracy": 0.7569981276988983,
+      "num_tokens": 579080.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.2447786808013916,
+      "epoch": 0.42198233562315995,
+      "grad_norm": 4.02454137802124,
+      "learning_rate": 9.312440605649753e-05,
+      "loss": 0.741,
+      "mean_token_accuracy": 0.7527964413166046,
+      "num_tokens": 592922.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.2158584475517273,
+      "epoch": 0.43179587831207067,
+      "grad_norm": 3.1649067401885986,
+      "learning_rate": 9.270813461015571e-05,
+      "loss": 0.6482,
+      "mean_token_accuracy": 0.790551096200943,
+      "num_tokens": 607114.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.2668315529823304,
+      "epoch": 0.44160942100098133,
+      "grad_norm": 2.8387515544891357,
+      "learning_rate": 9.228061737011235e-05,
+      "loss": 0.7715,
+      "mean_token_accuracy": 0.7353159546852112,
+      "num_tokens": 620907.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.2416473865509032,
+      "epoch": 0.45142296368989204,
+      "grad_norm": 3.8298416137695312,
+      "learning_rate": 9.184196690908298e-05,
+      "loss": 0.7507,
+      "mean_token_accuracy": 0.7600560903549194,
+      "num_tokens": 635363.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.2501473546028137,
+      "epoch": 0.46123650637880276,
+      "grad_norm": 2.8553757667541504,
+      "learning_rate": 9.139229873135349e-05,
+      "loss": 0.8007,
+      "mean_token_accuracy": 0.7271873593330384,
+      "num_tokens": 648200.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.270769226551056,
+      "epoch": 0.47105004906771347,
+      "grad_norm": 3.0687143802642822,
+      "learning_rate": 9.0931731242366e-05,
+      "loss": 0.8528,
+      "mean_token_accuracy": 0.7161825954914093,
+      "num_tokens": 662920.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.251631987094879,
+      "epoch": 0.48086359175662413,
+      "grad_norm": 3.8449313640594482,
+      "learning_rate": 9.046038571754066e-05,
+      "loss": 0.816,
+      "mean_token_accuracy": 0.7345060110092163,
+      "num_tokens": 676646.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.2504157423973083,
+      "epoch": 0.49067713444553485,
+      "grad_norm": 3.106739044189453,
+      "learning_rate": 8.997838627034158e-05,
+      "loss": 0.7438,
+      "mean_token_accuracy": 0.7501385807991028,
+      "num_tokens": 690435.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.2392379522323609,
+      "epoch": 0.5004906771344455,
+      "grad_norm": 3.4547104835510254,
+      "learning_rate": 8.94858598195958e-05,
+      "loss": 0.7234,
+      "mean_token_accuracy": 0.7637600660324096,
+      "num_tokens": 704664.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.2680296301841736,
+      "epoch": 0.5103042198233563,
+      "grad_norm": 3.1945977210998535,
+      "learning_rate": 8.898293605607316e-05,
+      "loss": 0.8045,
+      "mean_token_accuracy": 0.7297552824020386,
+      "num_tokens": 718834.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.2602490544319154,
+      "epoch": 0.5201177625122669,
+      "grad_norm": 3.213052272796631,
+      "learning_rate": 8.846974740833659e-05,
+      "loss": 0.7679,
+      "mean_token_accuracy": 0.7420543491840362,
+      "num_tokens": 733172.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.275605320930481,
+      "epoch": 0.5299313052011776,
+      "grad_norm": 3.2070508003234863,
+      "learning_rate": 8.794642900787137e-05,
+      "loss": 0.7689,
+      "mean_token_accuracy": 0.7334363520145416,
+      "num_tokens": 746577.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.2303307414054871,
+      "epoch": 0.5397448478900884,
+      "grad_norm": 2.7016286849975586,
+      "learning_rate": 8.741311865350259e-05,
+      "loss": 0.699,
+      "mean_token_accuracy": 0.7694551587104798,
+      "num_tokens": 760378.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.276139223575592,
+      "epoch": 0.549558390578999,
+      "grad_norm": 2.635620594024658,
+      "learning_rate": 8.68699567751104e-05,
+      "loss": 0.7973,
+      "mean_token_accuracy": 0.7300791203975677,
+      "num_tokens": 774528.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.2375047206878662,
+      "epoch": 0.5593719332679097,
+      "grad_norm": 2.7800259590148926,
+      "learning_rate": 8.631708639665234e-05,
+      "loss": 0.712,
+      "mean_token_accuracy": 0.764652305841446,
+      "num_tokens": 787978.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.2751248002052307,
+      "epoch": 0.5691854759568205,
+      "grad_norm": 2.6330859661102295,
+      "learning_rate": 8.575465309850265e-05,
+      "loss": 0.7752,
+      "mean_token_accuracy": 0.7408892452716828,
+      "num_tokens": 802122.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.2330546379089355,
+      "epoch": 0.5789990186457311,
+      "grad_norm": 3.023864507675171,
+      "learning_rate": 8.518280497911841e-05,
+      "loss": 0.6615,
+      "mean_token_accuracy": 0.7782455027103424,
+      "num_tokens": 816567.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.2333171844482422,
+      "epoch": 0.5888125613346418,
+      "grad_norm": 2.8540916442871094,
+      "learning_rate": 8.460169261604274e-05,
+      "loss": 0.6233,
+      "mean_token_accuracy": 0.7915886640548706,
+      "num_tokens": 830550.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.2261855959892274,
+      "epoch": 0.5986261040235525,
+      "grad_norm": 2.7179160118103027,
+      "learning_rate": 8.4011469026255e-05,
+      "loss": 0.6681,
+      "mean_token_accuracy": 0.7764095902442932,
+      "num_tokens": 843779.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.2497065186500549,
+      "epoch": 0.6084396467124632,
+      "grad_norm": 3.7094037532806396,
+      "learning_rate": 8.341228962587882e-05,
+      "loss": 0.7555,
+      "mean_token_accuracy": 0.754582530260086,
+      "num_tokens": 857498.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.2510279178619386,
+      "epoch": 0.6182531894013739,
+      "grad_norm": 3.291969060897827,
+      "learning_rate": 8.28043121892583e-05,
+      "loss": 0.7606,
+      "mean_token_accuracy": 0.7521400570869445,
+      "num_tokens": 871163.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.2494428634643555,
+      "epoch": 0.6280667320902846,
+      "grad_norm": 2.5670666694641113,
+      "learning_rate": 8.218769680741336e-05,
+      "loss": 0.7179,
+      "mean_token_accuracy": 0.7615531027317047,
+      "num_tokens": 884912.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.2484911561012269,
+      "epoch": 0.6378802747791953,
+      "grad_norm": 2.9769039154052734,
+      "learning_rate": 8.156260584588491e-05,
+      "loss": 0.7006,
+      "mean_token_accuracy": 0.7597393095493317,
+      "num_tokens": 898102.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.2530408382415772,
+      "epoch": 0.647693817468106,
+      "grad_norm": 3.7735605239868164,
+      "learning_rate": 8.09292039019812e-05,
+      "loss": 0.7615,
+      "mean_token_accuracy": 0.7538080394268036,
+      "num_tokens": 911902.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.2630650043487548,
+      "epoch": 0.6575073601570167,
+      "grad_norm": 2.9256949424743652,
+      "learning_rate": 8.028765776143654e-05,
+      "loss": 0.8192,
+      "mean_token_accuracy": 0.73097363114357,
+      "num_tokens": 926137.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.2100454688072204,
+      "epoch": 0.6673209028459274,
+      "grad_norm": 2.27181339263916,
+      "learning_rate": 7.963813635449367e-05,
+      "loss": 0.5924,
+      "mean_token_accuracy": 0.7951381504535675,
+      "num_tokens": 940507.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.2245916962623595,
+      "epoch": 0.677134445534838,
+      "grad_norm": 3.065109968185425,
+      "learning_rate": 7.898081071142149e-05,
+      "loss": 0.691,
+      "mean_token_accuracy": 0.771365362405777,
+      "num_tokens": 954934.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.2586957216262817,
+      "epoch": 0.6869479882237488,
+      "grad_norm": 3.6728291511535645,
+      "learning_rate": 7.831585391747984e-05,
+      "loss": 0.7884,
+      "mean_token_accuracy": 0.7376800179481506,
+      "num_tokens": 968760.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.2453697323799133,
+      "epoch": 0.6967615309126595,
+      "grad_norm": 3.318492889404297,
+      "learning_rate": 7.764344106734319e-05,
+      "loss": 0.7221,
+      "mean_token_accuracy": 0.755588686466217,
+      "num_tokens": 982644.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.247483992576599,
+      "epoch": 0.7065750736015701,
+      "grad_norm": 2.6510121822357178,
+      "learning_rate": 7.696374921899506e-05,
+      "loss": 0.7581,
+      "mean_token_accuracy": 0.7491323947906494,
+      "num_tokens": 996817.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.249286139011383,
+      "epoch": 0.7163886162904809,
+      "grad_norm": 3.5546514987945557,
+      "learning_rate": 7.627695734710564e-05,
+      "loss": 0.7443,
+      "mean_token_accuracy": 0.7533879339694977,
+      "num_tokens": 1010008.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.2515769600868225,
+      "epoch": 0.7262021589793916,
+      "grad_norm": 2.169177293777466,
+      "learning_rate": 7.55832462959047e-05,
+      "loss": 0.679,
+      "mean_token_accuracy": 0.7738364517688752,
+      "num_tokens": 1023812.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.2573638200759887,
+      "epoch": 0.7360157016683022,
+      "grad_norm": 3.4681811332702637,
+      "learning_rate": 7.488279873156207e-05,
+      "loss": 0.7823,
+      "mean_token_accuracy": 0.7313081920146942,
+      "num_tokens": 1037399.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.2329494953155518,
+      "epoch": 0.745829244357213,
+      "grad_norm": 3.7266030311584473,
+      "learning_rate": 7.417579909408851e-05,
+      "loss": 0.6845,
+      "mean_token_accuracy": 0.7801702082157135,
+      "num_tokens": 1050972.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.2601383924484253,
+      "epoch": 0.7556427870461236,
+      "grad_norm": 2.6621615886688232,
+      "learning_rate": 7.346243354876944e-05,
+      "loss": 0.7881,
+      "mean_token_accuracy": 0.7383911430835723,
+      "num_tokens": 1064581.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.2464670896530152,
+      "epoch": 0.7654563297350343,
+      "grad_norm": 2.635876178741455,
+      "learning_rate": 7.274288993714446e-05,
+      "loss": 0.6793,
+      "mean_token_accuracy": 0.7745445370674133,
+      "num_tokens": 1078433.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.2606447100639344,
+      "epoch": 0.7752698724239451,
+      "grad_norm": 2.995645046234131,
+      "learning_rate": 7.201735772754523e-05,
+      "loss": 0.755,
+      "mean_token_accuracy": 0.7501693069934845,
+      "num_tokens": 1091839.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.2495453476905822,
+      "epoch": 0.7850834151128557,
+      "grad_norm": 2.597090005874634,
+      "learning_rate": 7.128602796520538e-05,
+      "loss": 0.7336,
+      "mean_token_accuracy": 0.7542902231216431,
+      "num_tokens": 1105747.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.2640779137611389,
+      "epoch": 0.7948969578017664,
+      "grad_norm": 3.5687355995178223,
+      "learning_rate": 7.054909322195485e-05,
+      "loss": 0.7347,
+      "mean_token_accuracy": 0.7542178094387054,
+      "num_tokens": 1119344.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.2509456872940063,
+      "epoch": 0.8047105004906772,
+      "grad_norm": 2.925438642501831,
+      "learning_rate": 6.980674754551239e-05,
+      "loss": 0.7098,
+      "mean_token_accuracy": 0.7575959146022797,
+      "num_tokens": 1133252.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.2358011603355408,
+      "epoch": 0.8145240431795878,
+      "grad_norm": 3.4897379875183105,
+      "learning_rate": 6.905918640838946e-05,
+      "loss": 0.6776,
+      "mean_token_accuracy": 0.7676805794239044,
+      "num_tokens": 1147368.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.2407491087913514,
+      "epoch": 0.8243375858684985,
+      "grad_norm": 3.5089657306671143,
+      "learning_rate": 6.830660665641898e-05,
+      "loss": 0.708,
+      "mean_token_accuracy": 0.7612576723098755,
+      "num_tokens": 1161165.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.2481655716896056,
+      "epoch": 0.8341511285574092,
+      "grad_norm": 2.443739891052246,
+      "learning_rate": 6.754920645692222e-05,
+      "loss": 0.7286,
+      "mean_token_accuracy": 0.7533378839492798,
+      "num_tokens": 1174814.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.27309809923172,
+      "epoch": 0.8439646712463199,
+      "grad_norm": 2.843797206878662,
+      "learning_rate": 6.67871852465281e-05,
+      "loss": 0.775,
+      "mean_token_accuracy": 0.7456512689590454,
+      "num_tokens": 1188967.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.2300220608711243,
+      "epoch": 0.8537782139352306,
+      "grad_norm": 3.4666764736175537,
+      "learning_rate": 6.602074367865794e-05,
+      "loss": 0.7503,
+      "mean_token_accuracy": 0.7571634650230408,
+      "num_tokens": 1202907.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.2398579716682434,
+      "epoch": 0.8635917566241413,
+      "grad_norm": 2.9659690856933594,
+      "learning_rate": 6.525008357069e-05,
+      "loss": 0.6938,
+      "mean_token_accuracy": 0.7638706982135772,
+      "num_tokens": 1216865.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.2782540202140809,
+      "epoch": 0.873405299313052,
+      "grad_norm": 3.4597549438476562,
+      "learning_rate": 6.447540785081738e-05,
+      "loss": 0.7581,
+      "mean_token_accuracy": 0.74615278840065,
+      "num_tokens": 1230849.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.2612611055374146,
+      "epoch": 0.8832188420019627,
+      "grad_norm": 2.614473819732666,
+      "learning_rate": 6.36969205046136e-05,
+      "loss": 0.7866,
+      "mean_token_accuracy": 0.7384827435016632,
+      "num_tokens": 1243965.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.2412104845046996,
+      "epoch": 0.8930323846908734,
+      "grad_norm": 2.5842740535736084,
+      "learning_rate": 6.291482652131955e-05,
+      "loss": 0.702,
+      "mean_token_accuracy": 0.7730353534221649,
+      "num_tokens": 1257918.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.2504354119300842,
+      "epoch": 0.9028459273797841,
+      "grad_norm": 2.6346778869628906,
+      "learning_rate": 6.212933183986634e-05,
+      "loss": 0.7312,
+      "mean_token_accuracy": 0.75675647854805,
+      "num_tokens": 1271980.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.2406835436820984,
+      "epoch": 0.9126594700686947,
+      "grad_norm": 2.9103665351867676,
+      "learning_rate": 6.1340643294648e-05,
+      "loss": 0.6499,
+      "mean_token_accuracy": 0.7780694007873535,
+      "num_tokens": 1286287.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.2443556308746337,
+      "epoch": 0.9224730127576055,
+      "grad_norm": 2.473804473876953,
+      "learning_rate": 6.054896856105834e-05,
+      "loss": 0.7229,
+      "mean_token_accuracy": 0.7656688749790191,
+      "num_tokens": 1300381.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.2411270976066588,
+      "epoch": 0.9322865554465162,
+      "grad_norm": 2.786324977874756,
+      "learning_rate": 5.9754516100806423e-05,
+      "loss": 0.6997,
+      "mean_token_accuracy": 0.7713641285896301,
+      "num_tokens": 1314086.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.2566394686698914,
+      "epoch": 0.9421000981354269,
+      "grad_norm": 2.71813702583313,
+      "learning_rate": 5.895749510702502e-05,
+      "loss": 0.7315,
+      "mean_token_accuracy": 0.7593332231044769,
+      "num_tokens": 1328434.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2616096615791321,
+      "epoch": 0.9519136408243376,
+      "grad_norm": 2.3551230430603027,
+      "learning_rate": 5.815811544918641e-05,
+      "loss": 0.778,
+      "mean_token_accuracy": 0.736677497625351,
+      "num_tokens": 1341971.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.259574055671692,
+      "epoch": 0.9617271835132483,
+      "grad_norm": 3.2398195266723633,
+      "learning_rate": 5.7356587617840065e-05,
+      "loss": 0.6782,
+      "mean_token_accuracy": 0.7644849121570587,
+      "num_tokens": 1356685.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.2528340220451355,
+      "epoch": 0.971540726202159,
+      "grad_norm": 3.1825621128082275,
+      "learning_rate": 5.655312266918684e-05,
+      "loss": 0.7204,
+      "mean_token_accuracy": 0.7515070796012878,
+      "num_tokens": 1370541.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.225362777709961,
+      "epoch": 0.9813542688910697,
+      "grad_norm": 2.667006731033325,
+      "learning_rate": 5.574793216950428e-05,
+      "loss": 0.6711,
+      "mean_token_accuracy": 0.7758008658885955,
+      "num_tokens": 1383962.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.2216610431671142,
+      "epoch": 0.9911678115799804,
+      "grad_norm": 3.540689706802368,
+      "learning_rate": 5.494122813943742e-05,
+      "loss": 0.59,
+      "mean_token_accuracy": 0.804140430688858,
+      "num_tokens": 1397902.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.2351887226104736,
+      "epoch": 1.000981354268891,
+      "grad_norm": 2.9440114498138428,
+      "learning_rate": 5.413322299817019e-05,
+      "loss": 0.7101,
+      "mean_token_accuracy": 0.7661577880382537,
+      "num_tokens": 1410184.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.2483286023139955,
+      "epoch": 1.0107948969578018,
+      "grad_norm": 2.8180136680603027,
+      "learning_rate": 5.332412950749169e-05,
+      "loss": 0.7161,
+      "mean_token_accuracy": 0.7508663415908814,
+      "num_tokens": 1424018.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.2543777227401733,
+      "epoch": 1.0206084396467126,
+      "grad_norm": 2.1813180446624756,
+      "learning_rate": 5.251416071577232e-05,
+      "loss": 0.7106,
+      "mean_token_accuracy": 0.7597975909709931,
+      "num_tokens": 1438068.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.2590750098228454,
+      "epoch": 1.030421982335623,
+      "grad_norm": 3.2160985469818115,
+      "learning_rate": 5.170352990186441e-05,
+      "loss": 0.7566,
+      "mean_token_accuracy": 0.7482528626918793,
+      "num_tokens": 1452019.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.2396368861198426,
+      "epoch": 1.0402355250245339,
+      "grad_norm": 3.486187219619751,
+      "learning_rate": 5.08924505189423e-05,
+      "loss": 0.6984,
+      "mean_token_accuracy": 0.7629899203777313,
+      "num_tokens": 1465698.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.2675235152244568,
+      "epoch": 1.0500490677134446,
+      "grad_norm": 2.38860821723938,
+      "learning_rate": 5.008113613829631e-05,
+      "loss": 0.7571,
+      "mean_token_accuracy": 0.7420541822910309,
+      "num_tokens": 1479631.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.2640196561813355,
+      "epoch": 1.0598626104023552,
+      "grad_norm": 3.6147372722625732,
+      "learning_rate": 4.9269800393095747e-05,
+      "loss": 0.7665,
+      "mean_token_accuracy": 0.7382259011268616,
+      "num_tokens": 1492799.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.2641969680786134,
+      "epoch": 1.069676153091266,
+      "grad_norm": 2.739342451095581,
+      "learning_rate": 4.845865692213555e-05,
+      "loss": 0.7607,
+      "mean_token_accuracy": 0.7497819185256958,
+      "num_tokens": 1506531.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.2317523956298828,
+      "epoch": 1.0794896957801767,
+      "grad_norm": 3.975865602493286,
+      "learning_rate": 4.764791931358155e-05,
+      "loss": 0.6988,
+      "mean_token_accuracy": 0.7681858658790588,
+      "num_tokens": 1520031.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.2541891574859618,
+      "epoch": 1.0893032384690873,
+      "grad_norm": 2.668766975402832,
+      "learning_rate": 4.683780104872894e-05,
+      "loss": 0.7518,
+      "mean_token_accuracy": 0.7494930386543274,
+      "num_tokens": 1534436.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.2712677121162415,
+      "epoch": 1.099116781157998,
+      "grad_norm": 2.9274160861968994,
+      "learning_rate": 4.6028515445788937e-05,
+      "loss": 0.7345,
+      "mean_token_accuracy": 0.755589383840561,
+      "num_tokens": 1547962.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.2556275844573974,
+      "epoch": 1.1089303238469088,
+      "grad_norm": 2.679661989212036,
+      "learning_rate": 4.5220275603718474e-05,
+      "loss": 0.74,
+      "mean_token_accuracy": 0.7552561581134796,
+      "num_tokens": 1561849.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.2320382475852967,
+      "epoch": 1.1187438665358194,
+      "grad_norm": 2.0650317668914795,
+      "learning_rate": 4.4413294346107394e-05,
+      "loss": 0.6574,
+      "mean_token_accuracy": 0.780710768699646,
+      "num_tokens": 1575503.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.2443540453910829,
+      "epoch": 1.1285574092247301,
+      "grad_norm": 2.355180263519287,
+      "learning_rate": 4.3607784165138474e-05,
+      "loss": 0.7153,
+      "mean_token_accuracy": 0.7624804854393006,
+      "num_tokens": 1589893.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.2355771660804749,
+      "epoch": 1.138370951913641,
+      "grad_norm": 3.929511547088623,
+      "learning_rate": 4.280395716563441e-05,
+      "loss": 0.6401,
+      "mean_token_accuracy": 0.788442063331604,
+      "num_tokens": 1603816.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.2718832731246947,
+      "epoch": 1.1481844946025515,
+      "grad_norm": 2.341512680053711,
+      "learning_rate": 4.2002025009206994e-05,
+      "loss": 0.7629,
+      "mean_token_accuracy": 0.7445987641811371,
+      "num_tokens": 1617087.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.2553988456726075,
+      "epoch": 1.1579980372914622,
+      "grad_norm": 2.8875086307525635,
+      "learning_rate": 4.120219885852292e-05,
+      "loss": 0.6945,
+      "mean_token_accuracy": 0.7652477264404297,
+      "num_tokens": 1631558.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.2692211627960206,
+      "epoch": 1.167811579980373,
+      "grad_norm": 3.1470041275024414,
+      "learning_rate": 4.040468932170097e-05,
+      "loss": 0.7447,
+      "mean_token_accuracy": 0.7493159770965576,
+      "num_tokens": 1645944.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.2586512207984923,
+      "epoch": 1.1776251226692835,
+      "grad_norm": 3.1192193031311035,
+      "learning_rate": 3.9609706396855296e-05,
+      "loss": 0.7629,
+      "mean_token_accuracy": 0.7435645341873169,
+      "num_tokens": 1659636.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.2512589931488036,
+      "epoch": 1.1874386653581943,
+      "grad_norm": 2.6082634925842285,
+      "learning_rate": 3.881745941679917e-05,
+      "loss": 0.703,
+      "mean_token_accuracy": 0.7597541630268096,
+      "num_tokens": 1673078.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.2518221020698548,
+      "epoch": 1.197252208047105,
+      "grad_norm": 3.082732915878296,
+      "learning_rate": 3.802815699392418e-05,
+      "loss": 0.7092,
+      "mean_token_accuracy": 0.7615098059177399,
+      "num_tokens": 1687154.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.2582939624786378,
+      "epoch": 1.2070657507360156,
+      "grad_norm": 3.465059280395508,
+      "learning_rate": 3.724200696526873e-05,
+      "loss": 0.7162,
+      "mean_token_accuracy": 0.7580594539642334,
+      "num_tokens": 1701533.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.238705861568451,
+      "epoch": 1.2168792934249264,
+      "grad_norm": 2.861467123031616,
+      "learning_rate": 3.6459216337791124e-05,
+      "loss": 0.7091,
+      "mean_token_accuracy": 0.7654488742351532,
+      "num_tokens": 1714722.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.240504229068756,
+      "epoch": 1.2266928361138372,
+      "grad_norm": 2.5317296981811523,
+      "learning_rate": 3.567999123386103e-05,
+      "loss": 0.7535,
+      "mean_token_accuracy": 0.7481034755706787,
+      "num_tokens": 1727989.0,
+      "step": 1250
+    },
+    {
+      "entropy": 1.2500628590583802,
+      "epoch": 1.2365063788027477,
+      "grad_norm": 2.7721686363220215,
+      "learning_rate": 3.490453683698376e-05,
+      "loss": 0.7102,
+      "mean_token_accuracy": 0.7544439673423767,
+      "num_tokens": 1742532.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.2480277180671693,
+      "epoch": 1.2463199214916585,
+      "grad_norm": 2.6612651348114014,
+      "learning_rate": 3.4133057337771995e-05,
+      "loss": 0.6933,
+      "mean_token_accuracy": 0.7625578582286835,
+      "num_tokens": 1755690.0,
+      "step": 1270
+    },
+    {
+      "entropy": 1.2488563537597657,
+      "epoch": 1.2561334641805693,
+      "grad_norm": 2.6166510581970215,
+      "learning_rate": 3.336575588017881e-05,
+      "loss": 0.7433,
+      "mean_token_accuracy": 0.7545452177524566,
+      "num_tokens": 1769918.0,
+      "step": 1280
+    },
+    {
+      "entropy": 1.2666580557823182,
+      "epoch": 1.2659470068694798,
+      "grad_norm": 2.894110918045044,
+      "learning_rate": 3.260283450800634e-05,
+      "loss": 0.7329,
+      "mean_token_accuracy": 0.753772646188736,
+      "num_tokens": 1783944.0,
+      "step": 1290
+    },
+    {
+      "entropy": 1.251064169406891,
+      "epoch": 1.2757605495583906,
+      "grad_norm": 3.757072925567627,
+      "learning_rate": 3.184449411170416e-05,
+      "loss": 0.7134,
+      "mean_token_accuracy": 0.767280787229538,
+      "num_tokens": 1797715.0,
+      "step": 1300
+    },
+    {
+      "entropy": 1.2521143436431885,
+      "epoch": 1.2855740922473013,
+      "grad_norm": 2.400064706802368,
+      "learning_rate": 3.109093437547128e-05,
+      "loss": 0.7255,
+      "mean_token_accuracy": 0.7518977701663971,
+      "num_tokens": 1811354.0,
+      "step": 1310
+    },
+    {
+      "entropy": 1.2557258129119873,
+      "epoch": 1.295387634936212,
+      "grad_norm": 2.8327138423919678,
+      "learning_rate": 3.0342353724675966e-05,
+      "loss": 0.7242,
+      "mean_token_accuracy": 0.7556073129177093,
+      "num_tokens": 1825286.0,
+      "step": 1320
+    },
+    {
+      "entropy": 1.2582063317298888,
+      "epoch": 1.3052011776251227,
+      "grad_norm": 2.933422327041626,
+      "learning_rate": 2.9598949273606702e-05,
+      "loss": 0.7573,
+      "mean_token_accuracy": 0.7498652696609497,
+      "num_tokens": 1839043.0,
+      "step": 1330
+    },
+    {
+      "entropy": 1.2396968483924866,
+      "epoch": 1.3150147203140334,
+      "grad_norm": 2.527557134628296,
+      "learning_rate": 2.8860916773568703e-05,
+      "loss": 0.7256,
+      "mean_token_accuracy": 0.7586526334285736,
+      "num_tokens": 1852342.0,
+      "step": 1340
+    },
+    {
+      "entropy": 1.2368186235427856,
+      "epoch": 1.324828263002944,
+      "grad_norm": 3.119965076446533,
+      "learning_rate": 2.8128450561339193e-05,
+      "loss": 0.686,
+      "mean_token_accuracy": 0.7728847086429596,
+      "num_tokens": 1866189.0,
+      "step": 1350
+    },
+    {
+      "entropy": 1.2524226903915405,
+      "epoch": 1.3346418056918548,
+      "grad_norm": 2.933042287826538,
+      "learning_rate": 2.740174350799507e-05,
+      "loss": 0.728,
+      "mean_token_accuracy": 0.760904735326767,
+      "num_tokens": 1880098.0,
+      "step": 1360
+    },
+    {
+      "entropy": 1.2240639209747315,
+      "epoch": 1.3444553483807655,
+      "grad_norm": 2.4190683364868164,
+      "learning_rate": 2.668098696812675e-05,
+      "loss": 0.6506,
+      "mean_token_accuracy": 0.7759924173355103,
+      "num_tokens": 1893007.0,
+      "step": 1370
+    },
+    {
+      "entropy": 1.241419780254364,
+      "epoch": 1.354268891069676,
+      "grad_norm": 2.9516451358795166,
+      "learning_rate": 2.596637072945099e-05,
+      "loss": 0.658,
+      "mean_token_accuracy": 0.7787473976612092,
+      "num_tokens": 1906405.0,
+      "step": 1380
+    },
+    {
+      "entropy": 1.2689218997955323,
+      "epoch": 1.3640824337585868,
+      "grad_norm": 3.411701202392578,
+      "learning_rate": 2.5258082962836614e-05,
+      "loss": 0.8255,
+      "mean_token_accuracy": 0.7264221429824829,
+      "num_tokens": 1920751.0,
+      "step": 1390
+    },
+    {
+      "entropy": 1.2496665716171265,
+      "epoch": 1.3738959764474976,
+      "grad_norm": 2.833319664001465,
+      "learning_rate": 2.4556310172755898e-05,
+      "loss": 0.7171,
+      "mean_token_accuracy": 0.7667763590812683,
+      "num_tokens": 1934411.0,
+      "step": 1400
+    },
+    {
+      "entropy": 1.2395281672477723,
+      "epoch": 1.3837095191364082,
+      "grad_norm": 2.8841261863708496,
+      "learning_rate": 2.3861237148174474e-05,
+      "loss": 0.6814,
+      "mean_token_accuracy": 0.7712886571884155,
+      "num_tokens": 1948013.0,
+      "step": 1410
+    },
+    {
+      "entropy": 1.2530240893363953,
+      "epoch": 1.393523061825319,
+      "grad_norm": 2.2632269859313965,
+      "learning_rate": 2.3173046913893502e-05,
+      "loss": 0.6896,
+      "mean_token_accuracy": 0.7679926335811615,
+      "num_tokens": 1962274.0,
+      "step": 1420
+    },
+    {
+      "entropy": 1.2642626643180848,
+      "epoch": 1.4033366045142297,
+      "grad_norm": 4.057086944580078,
+      "learning_rate": 2.249192068235581e-05,
+      "loss": 0.734,
+      "mean_token_accuracy": 0.7582364082336426,
+      "num_tokens": 1976604.0,
+      "step": 1430
+    },
+    {
+      "entropy": 1.2472557187080384,
+      "epoch": 1.4131501472031402,
+      "grad_norm": 2.4790313243865967,
+      "learning_rate": 2.181803780592959e-05,
+      "loss": 0.6884,
+      "mean_token_accuracy": 0.7712749302387237,
+      "num_tokens": 1991136.0,
+      "step": 1440
+    },
+    {
+      "entropy": 1.2573583006858826,
+      "epoch": 1.422963689892051,
+      "grad_norm": 3.6234960556030273,
+      "learning_rate": 2.1151575729681883e-05,
+      "loss": 0.761,
+      "mean_token_accuracy": 0.7374938070774079,
+      "num_tokens": 2005289.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.2349944472312928,
+      "epoch": 1.4327772325809618,
+      "grad_norm": 2.5186727046966553,
+      "learning_rate": 2.049270994465407e-05,
+      "loss": 0.6876,
+      "mean_token_accuracy": 0.7730244517326355,
+      "num_tokens": 2018916.0,
+      "step": 1460
+    },
+    {
+      "entropy": 1.2298003792762757,
+      "epoch": 1.4425907752698723,
+      "grad_norm": 2.3550288677215576,
+      "learning_rate": 1.9841613941652144e-05,
+      "loss": 0.637,
+      "mean_token_accuracy": 0.7788012862205506,
+      "num_tokens": 2032597.0,
+      "step": 1470
+    },
+    {
+      "entropy": 1.2459490180015564,
+      "epoch": 1.452404317958783,
+      "grad_norm": 2.7235300540924072,
+      "learning_rate": 1.9198459165563503e-05,
+      "loss": 0.6802,
+      "mean_token_accuracy": 0.7714063704013825,
+      "num_tokens": 2046720.0,
+      "step": 1480
+    },
+    {
+      "entropy": 1.2520262598991394,
+      "epoch": 1.4622178606476939,
+      "grad_norm": 2.912389039993286,
+      "learning_rate": 1.8563414970212555e-05,
+      "loss": 0.6615,
+      "mean_token_accuracy": 0.7768101215362548,
+      "num_tokens": 2060943.0,
+      "step": 1490
+    },
+    {
+      "entropy": 1.2493624329566955,
+      "epoch": 1.4720314033366044,
+      "grad_norm": 2.4805216789245605,
+      "learning_rate": 1.7936648573766957e-05,
+      "loss": 0.7245,
+      "mean_token_accuracy": 0.7542855679988861,
+      "num_tokens": 2074710.0,
+      "step": 1500
+    },
+    {
+      "entropy": 1.2552477240562439,
+      "epoch": 1.4818449460255152,
+      "grad_norm": 2.1514837741851807,
+      "learning_rate": 1.731832501470606e-05,
+      "loss": 0.6929,
+      "mean_token_accuracy": 0.7639276206493377,
+      "num_tokens": 2088707.0,
+      "step": 1510
+    },
+    {
+      "entropy": 1.2568463563919068,
+      "epoch": 1.491658488714426,
+      "grad_norm": 2.511904716491699,
+      "learning_rate": 1.670860710836355e-05,
+      "loss": 0.6563,
+      "mean_token_accuracy": 0.776709008216858,
+      "num_tokens": 2101428.0,
+      "step": 1520
+    },
+    {
+      "entropy": 1.2479868054389953,
+      "epoch": 1.5014720314033365,
+      "grad_norm": 2.0985195636749268,
+      "learning_rate": 1.6107655404055223e-05,
+      "loss": 0.6908,
+      "mean_token_accuracy": 0.7716210365295411,
+      "num_tokens": 2115066.0,
+      "step": 1530
+    },
+    {
+      "entropy": 1.2497215032577516,
+      "epoch": 1.5112855740922473,
+      "grad_norm": 2.7013895511627197,
+      "learning_rate": 1.5515628142803562e-05,
+      "loss": 0.6948,
+      "mean_token_accuracy": 0.7616774380207062,
+      "num_tokens": 2128953.0,
+      "step": 1540
+    },
+    {
+      "entropy": 1.2579123854637146,
+      "epoch": 1.521099116781158,
+      "grad_norm": 3.0048000812530518,
+      "learning_rate": 1.493268121567028e-05,
+      "loss": 0.716,
+      "mean_token_accuracy": 0.763817572593689,
+      "num_tokens": 2142756.0,
+      "step": 1550
+    },
+    {
+      "entropy": 1.2480177998542785,
+      "epoch": 1.5309126594700686,
+      "grad_norm": 3.407221555709839,
+      "learning_rate": 1.435896812270729e-05,
+      "loss": 0.7371,
+      "mean_token_accuracy": 0.7436653435230255,
+      "num_tokens": 2156108.0,
+      "step": 1560
+    },
+    {
+      "entropy": 1.2554169058799745,
+      "epoch": 1.5407262021589794,
+      "grad_norm": 3.1060776710510254,
+      "learning_rate": 1.3794639932537645e-05,
+      "loss": 0.757,
+      "mean_token_accuracy": 0.7477840483188629,
+      "num_tokens": 2169356.0,
+      "step": 1570
+    },
+    {
+      "entropy": 1.2660345435142517,
+      "epoch": 1.5505397448478901,
+      "grad_norm": 3.049757242202759,
+      "learning_rate": 1.323984524257641e-05,
+      "loss": 0.7345,
+      "mean_token_accuracy": 0.7469079375267029,
+      "num_tokens": 2183713.0,
+      "step": 1580
+    },
+    {
+      "entropy": 1.2590110421180725,
+      "epoch": 1.5603532875368007,
+      "grad_norm": 3.270705461502075,
+      "learning_rate": 1.2694730139902522e-05,
+      "loss": 0.7626,
+      "mean_token_accuracy": 0.7469726979732514,
+      "num_tokens": 2198089.0,
+      "step": 1590
+    },
+    {
+      "entropy": 1.2689306259155273,
+      "epoch": 1.5701668302257115,
+      "grad_norm": 2.790843963623047,
+      "learning_rate": 1.2159438162791348e-05,
+      "loss": 0.759,
+      "mean_token_accuracy": 0.7433544754981994,
+      "num_tokens": 2213070.0,
+      "step": 1600
+    },
+    {
+      "entropy": 1.2459934830665589,
+      "epoch": 1.5799803729146222,
+      "grad_norm": 2.270263671875,
+      "learning_rate": 1.1634110262918718e-05,
+      "loss": 0.694,
+      "mean_token_accuracy": 0.7705317616462708,
+      "num_tokens": 2227108.0,
+      "step": 1610
+    },
+    {
+      "entropy": 1.2674721240997315,
+      "epoch": 1.5897939156035328,
+      "grad_norm": 3.1888628005981445,
+      "learning_rate": 1.1118884768245863e-05,
+      "loss": 0.7636,
+      "mean_token_accuracy": 0.738410371541977,
+      "num_tokens": 2240704.0,
+      "step": 1620
+    },
+    {
+      "entropy": 1.2725833535194397,
+      "epoch": 1.5996074582924436,
+      "grad_norm": 2.8385000228881836,
+      "learning_rate": 1.0613897346595203e-05,
+      "loss": 0.77,
+      "mean_token_accuracy": 0.7319659352302551,
+      "num_tokens": 2254968.0,
+      "step": 1630
+    },
+    {
+      "entropy": 1.2558390736579894,
+      "epoch": 1.6094210009813543,
+      "grad_norm": 2.5580456256866455,
+      "learning_rate": 1.011928096992672e-05,
+      "loss": 0.6834,
+      "mean_token_accuracy": 0.7667473912239074,
+      "num_tokens": 2269175.0,
+      "step": 1640
+    },
+    {
+      "entropy": 1.2538867115974426,
+      "epoch": 1.6192345436702649,
+      "grad_norm": 3.2263309955596924,
+      "learning_rate": 9.635165879324083e-06,
+      "loss": 0.7524,
+      "mean_token_accuracy": 0.748347532749176,
+      "num_tokens": 2283760.0,
+      "step": 1650
+    },
+    {
+      "entropy": 1.2552440881729126,
+      "epoch": 1.6290480863591756,
+      "grad_norm": 2.529660224914551,
+      "learning_rate": 9.161679550699936e-06,
+      "loss": 0.6819,
+      "mean_token_accuracy": 0.7654806017875672,
+      "num_tokens": 2297542.0,
+      "step": 1660
+    },
+    {
+      "entropy": 1.2585554003715516,
+      "epoch": 1.6388616290480864,
+      "grad_norm": 3.8871724605560303,
+      "learning_rate": 8.698946661229262e-06,
+      "loss": 0.7457,
+      "mean_token_accuracy": 0.74978466629982,
+      "num_tokens": 2311381.0,
+      "step": 1670
+    },
+    {
+      "entropy": 1.2282718300819397,
+      "epoch": 1.648675171736997,
+      "grad_norm": 2.116325616836548,
+      "learning_rate": 8.24708905651972e-06,
+      "loss": 0.7246,
+      "mean_token_accuracy": 0.75484379529953,
+      "num_tokens": 2324526.0,
+      "step": 1680
+    },
+    {
+      "entropy": 1.2373629927635192,
+      "epoch": 1.6584887144259077,
+      "grad_norm": 2.604167938232422,
+      "learning_rate": 7.80622571852761e-06,
+      "loss": 0.6936,
+      "mean_token_accuracy": 0.7663450837135315,
+      "num_tokens": 2337382.0,
+      "step": 1690
+    },
+    {
+      "entropy": 1.247055208683014,
+      "epoch": 1.6683022571148185,
+      "grad_norm": 2.7145724296569824,
+      "learning_rate": 7.376472734227807e-06,
+      "loss": 0.7612,
+      "mean_token_accuracy": 0.7374716877937317,
+      "num_tokens": 2351843.0,
+      "step": 1700
+    },
+    {
+      "entropy": 1.229586088657379,
+      "epoch": 1.678115799803729,
+      "grad_norm": 2.8956756591796875,
+      "learning_rate": 6.957943265046052e-06,
+      "loss": 0.6341,
+      "mean_token_accuracy": 0.7813458263874054,
+      "num_tokens": 2365499.0,
+      "step": 1710
+    },
+    {
+      "entropy": 1.248737633228302,
+      "epoch": 1.6879293424926398,
+      "grad_norm": 2.3320205211639404,
+      "learning_rate": 6.5507475170616565e-06,
+      "loss": 0.6953,
+      "mean_token_accuracy": 0.758164393901825,
+      "num_tokens": 2379228.0,
+      "step": 1720
+    },
+    {
+      "entropy": 1.2544867873191834,
+      "epoch": 1.6977428851815506,
+      "grad_norm": 3.138793468475342,
+      "learning_rate": 6.154992711988183e-06,
+      "loss": 0.7625,
+      "mean_token_accuracy": 0.7475127398967742,
+      "num_tokens": 2392855.0,
+      "step": 1730
+    },
+    {
+      "entropy": 1.250640881061554,
+      "epoch": 1.7075564278704611,
+      "grad_norm": 3.420732021331787,
+      "learning_rate": 5.770783058940271e-06,
+      "loss": 0.7209,
+      "mean_token_accuracy": 0.75545534491539,
+      "num_tokens": 2406729.0,
+      "step": 1740
+    },
+    {
+      "entropy": 1.2905877470970153,
+      "epoch": 1.717369970559372,
+      "grad_norm": 2.654557704925537,
+      "learning_rate": 5.398219726993442e-06,
+      "loss": 0.7996,
+      "mean_token_accuracy": 0.724600100517273,
+      "num_tokens": 2420319.0,
+      "step": 1750
+    },
+    {
+      "entropy": 1.242275857925415,
+      "epoch": 1.7271835132482827,
+      "grad_norm": 2.5090105533599854,
+      "learning_rate": 5.03740081854458e-06,
+      "loss": 0.6777,
+      "mean_token_accuracy": 0.7697700500488281,
+      "num_tokens": 2434508.0,
+      "step": 1760
+    },
+    {
+      "entropy": 1.2769638299942017,
+      "epoch": 1.7369970559371932,
+      "grad_norm": 3.1650819778442383,
+      "learning_rate": 4.68842134347987e-06,
+      "loss": 0.8118,
+      "mean_token_accuracy": 0.729069709777832,
+      "num_tokens": 2447872.0,
+      "step": 1770
+    },
+    {
+      "entropy": 1.2345268607139588,
+      "epoch": 1.746810598626104,
+      "grad_norm": 2.4639127254486084,
+      "learning_rate": 4.351373194157016e-06,
+      "loss": 0.678,
+      "mean_token_accuracy": 0.7657662034034729,
+      "num_tokens": 2461151.0,
+      "step": 1780
+    },
+    {
+      "entropy": 1.2443835020065308,
+      "epoch": 1.7566241413150148,
+      "grad_norm": 2.4859766960144043,
+      "learning_rate": 4.026345121208403e-06,
+      "loss": 0.7286,
+      "mean_token_accuracy": 0.7519601106643676,
+      "num_tokens": 2474907.0,
+      "step": 1790
+    },
+    {
+      "entropy": 1.2543081402778626,
+      "epoch": 1.7664376840039253,
+      "grad_norm": 3.275564670562744,
+      "learning_rate": 3.7134227101715156e-06,
+      "loss": 0.7041,
+      "mean_token_accuracy": 0.7669776916503906,
+      "num_tokens": 2488408.0,
+      "step": 1800
+    },
+    {
+      "entropy": 1.2694288611412048,
+      "epoch": 1.776251226692836,
+      "grad_norm": 3.098207712173462,
+      "learning_rate": 3.4126883589527705e-06,
+      "loss": 0.7399,
+      "mean_token_accuracy": 0.746983003616333,
+      "num_tokens": 2503031.0,
+      "step": 1810
+    },
+    {
+      "entropy": 1.245356595516205,
+      "epoch": 1.7860647693817469,
+      "grad_norm": 2.92874813079834,
+      "learning_rate": 3.124221256130744e-06,
+      "loss": 0.7358,
+      "mean_token_accuracy": 0.760612016916275,
+      "num_tokens": 2516322.0,
+      "step": 1820
+    },
+    {
+      "entropy": 1.2525557637214662,
+      "epoch": 1.7958783120706574,
+      "grad_norm": 3.0641603469848633,
+      "learning_rate": 2.848097360104396e-06,
+      "loss": 0.7472,
+      "mean_token_accuracy": 0.7508267819881439,
+      "num_tokens": 2529795.0,
+      "step": 1830
+    },
+    {
+      "entropy": 1.2594183802604675,
+      "epoch": 1.8056918547595682,
+      "grad_norm": 2.1558589935302734,
+      "learning_rate": 2.584389379091945e-06,
+      "loss": 0.7451,
+      "mean_token_accuracy": 0.7472402095794678,
+      "num_tokens": 2542968.0,
+      "step": 1840
+    },
+    {
+      "entropy": 1.2379876255989075,
+      "epoch": 1.815505397448479,
+      "grad_norm": 2.5214271545410156,
+      "learning_rate": 2.333166751985516e-06,
+      "loss": 0.655,
+      "mean_token_accuracy": 0.7800517380237579,
+      "num_tokens": 2556526.0,
+      "step": 1850
+    },
+    {
+      "entropy": 1.2424226760864259,
+      "epoch": 1.8253189401373895,
+      "grad_norm": 2.2212626934051514,
+      "learning_rate": 2.0944956300666565e-06,
+      "loss": 0.6391,
+      "mean_token_accuracy": 0.7772759675979615,
+      "num_tokens": 2569913.0,
+      "step": 1860
+    },
+    {
+      "entropy": 1.2615416049957275,
+      "epoch": 1.8351324828263003,
+      "grad_norm": 2.6506869792938232,
+      "learning_rate": 1.8684388595876012e-06,
+      "loss": 0.7413,
+      "mean_token_accuracy": 0.7527687847614288,
+      "num_tokens": 2584227.0,
+      "step": 1870
+    },
+    {
+      "entropy": 1.246048879623413,
+      "epoch": 1.844946025515211,
+      "grad_norm": 2.299192428588867,
+      "learning_rate": 1.6550559652227126e-06,
+      "loss": 0.7319,
+      "mean_token_accuracy": 0.7512415587902069,
+      "num_tokens": 2598410.0,
+      "step": 1880
+    },
+    {
+      "entropy": 1.247852861881256,
+      "epoch": 1.8547595682041216,
+      "grad_norm": 2.740612745285034,
+      "learning_rate": 1.4544031343946385e-06,
+      "loss": 0.6166,
+      "mean_token_accuracy": 0.7872454166412354,
+      "num_tokens": 2613088.0,
+      "step": 1890
+    },
+    {
+      "entropy": 1.251421856880188,
+      "epoch": 1.8645731108930323,
+      "grad_norm": 2.1468136310577393,
+      "learning_rate": 1.2665332024791532e-06,
+      "loss": 0.6623,
+      "mean_token_accuracy": 0.7790079653263092,
+      "num_tokens": 2627112.0,
+      "step": 1900
+    },
+    {
+      "entropy": 1.242864727973938,
+      "epoch": 1.8743866535819431,
+      "grad_norm": 2.3495595455169678,
+      "learning_rate": 1.09149563889272e-06,
+      "loss": 0.7223,
+      "mean_token_accuracy": 0.7554054081439971,
+      "num_tokens": 2640791.0,
+      "step": 1910
+    },
+    {
+      "entropy": 1.2578393936157226,
+      "epoch": 1.8842001962708537,
+      "grad_norm": 3.0611588954925537,
+      "learning_rate": 9.293365340662996e-07,
+      "loss": 0.7159,
+      "mean_token_accuracy": 0.7581327319145202,
+      "num_tokens": 2654993.0,
+      "step": 1920
+    },
+    {
+      "entropy": 1.2369702816009522,
+      "epoch": 1.8940137389597644,
+      "grad_norm": 2.9516565799713135,
+      "learning_rate": 7.800985873089695e-07,
+      "loss": 0.6766,
+      "mean_token_accuracy": 0.7690627932548523,
+      "num_tokens": 2669247.0,
+      "step": 1930
+    },
+    {
+      "entropy": 1.2367492318153381,
+      "epoch": 1.9038272816486752,
+      "grad_norm": 4.053267002105713,
+      "learning_rate": 6.438210955644452e-07,
+      "loss": 0.6977,
+      "mean_token_accuracy": 0.7696449518203735,
+      "num_tokens": 2682844.0,
+      "step": 1940
+    },
+    {
+      "entropy": 1.2650766253471375,
+      "epoch": 1.9136408243375858,
+      "grad_norm": 3.4821527004241943,
+      "learning_rate": 5.205399430634917e-07,
+      "loss": 0.7512,
+      "mean_token_accuracy": 0.7400960445404052,
+      "num_tokens": 2696806.0,
+      "step": 1950
+    },
+    {
+      "entropy": 1.2482685685157775,
+      "epoch": 1.9234543670264965,
+      "grad_norm": 3.1683855056762695,
+      "learning_rate": 4.102875918750082e-07,
+      "loss": 0.7084,
+      "mean_token_accuracy": 0.7535434782505035,
+      "num_tokens": 2710979.0,
+      "step": 1960
+    },
+    {
+      "entropy": 1.2443427562713623,
+      "epoch": 1.9332679097154073,
+      "grad_norm": 3.2206838130950928,
+      "learning_rate": 3.1309307335820603e-07,
+      "loss": 0.6951,
+      "mean_token_accuracy": 0.7632800579071045,
+      "num_tokens": 2724880.0,
+      "step": 1970
+    },
+    {
+      "entropy": 1.2644654035568237,
+      "epoch": 1.9430814524043178,
+      "grad_norm": 3.127032518386841,
+      "learning_rate": 2.2898198051811148e-07,
+      "loss": 0.7599,
+      "mean_token_accuracy": 0.7434370875358581,
+      "num_tokens": 2738562.0,
+      "step": 1980
+    },
+    {
+      "entropy": 1.2518927335739136,
+      "epoch": 1.9528949950932286,
+      "grad_norm": 1.9619168043136597,
+      "learning_rate": 1.5797646126651245e-07,
+      "loss": 0.689,
+      "mean_token_accuracy": 0.7646110415458679,
+      "num_tokens": 2752355.0,
+      "step": 1990
+    },
+    {
+      "entropy": 1.2708698391914368,
+      "epoch": 1.9627085377821394,
+      "grad_norm": 2.7405574321746826,
+      "learning_rate": 1.0009521259003452e-07,
+      "loss": 0.7124,
+      "mean_token_accuracy": 0.7528804540634155,
+      "num_tokens": 2766143.0,
+      "step": 2000
+    },
+    {
+      "entropy": 1.2544511437416077,
+      "epoch": 1.97252208047105,
+      "grad_norm": 3.238013505935669,
+      "learning_rate": 5.53534756268681e-08,
+      "loss": 0.7301,
+      "mean_token_accuracy": 0.7523074209690094,
+      "num_tokens": 2779801.0,
+      "step": 2010
+    },
+    {
+      "entropy": 1.204650866985321,
+      "epoch": 1.9823356231599607,
+      "grad_norm": 3.4440979957580566,
+      "learning_rate": 2.376303165353977e-08,
+      "loss": 0.6254,
+      "mean_token_accuracy": 0.7939655542373657,
+      "num_tokens": 2793763.0,
+      "step": 2020
+    },
+    {
+      "entropy": 1.2504274725914002,
+      "epoch": 1.9921491658488715,
+      "grad_norm": 2.904874563217163,
+      "learning_rate": 5.33219898269377e-09,
+      "loss": 0.6873,
+      "mean_token_accuracy": 0.7712612926959992,
+      "num_tokens": 2807624.0,
+      "step": 2030
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2038,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1987310863414477e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/checkpoint-2038/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1afffdc36d44613211eaa40ebb50cb6234dafd1941c4b1ddaed9a47dc27660cc
+size 6417

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k",
+  "dataset_run_name": "cat_nums_30k_seed42_qwen25_7b_v1",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/cat_nums_30k_seed42_qwen25_7b_v1/filtered_10000.jsonl",
+  "data_file_sha256": "c7a1038f3653a8dffedbe1ad6e7bc4ebf0ef9ebe0a0463fb79ef2a95becd1c95",
+  "num_rows": 10000,
+  "v_teacher_path": null,
+  "v_teacher_sha256": null,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 2,
+    "lr": 0.0001,
+    "optim": "sgd",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": true,
+    "seed": 1,
+    "val_split": 0
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/train_manifest.json

@@ -0,0 +1,30 @@
+{
+  "run_name": "cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k",
+  "dataset_run_name": "cat_nums_30k_seed42_qwen25_7b_v1",
+  "filtered_basename": "filtered_10000.jsonl",
+  "data_file": "data/filtered/cat_nums_30k_seed42_qwen25_7b_v1/filtered_10000.jsonl",
+  "data_file_sha256": "c7a1038f3653a8dffedbe1ad6e7bc4ebf0ef9ebe0a0463fb79ef2a95becd1c95",
+  "num_rows": 10000,
+  "v_teacher_path": null,
+  "v_teacher_sha256": null,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct",
+  "lora": {
+    "r": 8,
+    "alpha": 32,
+    "dropout": 0.0,
+    "target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
+  },
+  "train": {
+    "epochs": 2,
+    "lr": 0.0001,
+    "optim": "sgd",
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.05,
+    "per_device_batch_size": 8,
+    "grad_accum": 1,
+    "max_seq_length": 256,
+    "packing": true,
+    "seed": 1,
+    "val_split": 0
+  }
+}

checkpoints/cat_qwen25_7b_r8_a32_sgd_rot_e2_lr1e-4_s1_10k/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1afffdc36d44613211eaa40ebb50cb6234dafd1941c4b1ddaed9a47dc27660cc
+size 6417