Sync.

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1231/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,62 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+model_name: backward_lora
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for backward_lora
+
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 1.0.0
+- Transformers: 5.4.0
+- Pytorch: 2.10.0
+- Datasets: 4.8.4
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d9655d13d272482af6ddd073ad46392e73ceead6c8fe81271ce3de19726bc2e
+size 69782384

chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint-1231/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-1231/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1231/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d9655d13d272482af6ddd073ad46392e73ceead6c8fe81271ce3de19726bc2e
+size 69782384

checkpoint-1231/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint-1231/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b01acfa3baf860f4cdad37f3b8088dc5dd910ad656ffdaa5536f6a62b8522d38
+size 139793343

checkpoint-1231/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4abe56394d78bb142f3236a730d9c534ff85674aca1c06e786c7ee5f0042f6c
+size 14565

checkpoint-1231/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93f73c7e83558049a56ede141f51e392800912a1c83d2a889834573ee21f56d5
+size 1453

checkpoint-1231/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoint-1231/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-1231/trainer_state.json

@@ -0,0 +1,1264 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1231,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2297935895621777,
+      "epoch": 0.008125126955108674,
+      "grad_norm": 0.4000815749168396,
+      "learning_rate": 4.8648648648648654e-05,
+      "loss": 2.834882926940918,
+      "mean_token_accuracy": 0.5321066098287701,
+      "num_tokens": 21417.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.5277322106063367,
+      "epoch": 0.01625025391021735,
+      "grad_norm": 0.3763589560985565,
+      "learning_rate": 0.0001027027027027027,
+      "loss": 2.3326297760009767,
+      "mean_token_accuracy": 0.555946185067296,
+      "num_tokens": 43184.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.9664923571050168,
+      "epoch": 0.02437538086532602,
+      "grad_norm": 0.32139986753463745,
+      "learning_rate": 0.00015675675675675676,
+      "loss": 1.709489631652832,
+      "mean_token_accuracy": 0.6315864238888025,
+      "num_tokens": 65130.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.16125695258379,
+      "epoch": 0.0325005078204347,
+      "grad_norm": 0.18259193003177643,
+      "learning_rate": 0.00019999861541352413,
+      "loss": 1.8902889251708985,
+      "mean_token_accuracy": 0.6346669655293227,
+      "num_tokens": 86484.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.947226920723915,
+      "epoch": 0.040625634775543366,
+      "grad_norm": 0.16834287345409393,
+      "learning_rate": 0.0001999501589126174,
+      "loss": 1.5713948249816894,
+      "mean_token_accuracy": 0.6738984193652868,
+      "num_tokens": 111843.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.9656577214598656,
+      "epoch": 0.04875076173065204,
+      "grad_norm": 0.19594942033290863,
+      "learning_rate": 0.00019983251142473933,
+      "loss": 1.8397003173828126,
+      "mean_token_accuracy": 0.6406010089442133,
+      "num_tokens": 139980.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.955870234966278,
+      "epoch": 0.05687588868576071,
+      "grad_norm": 0.2009470909833908,
+      "learning_rate": 0.00019964575439193965,
+      "loss": 1.5263800621032715,
+      "mean_token_accuracy": 0.6778148110955954,
+      "num_tokens": 164281.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.9609144352376462,
+      "epoch": 0.0650010156408694,
+      "grad_norm": 0.12439315766096115,
+      "learning_rate": 0.0001993900170976888,
+      "loss": 1.628624153137207,
+      "mean_token_accuracy": 0.6553534045815468,
+      "num_tokens": 187765.0,
+      "step": 80
+    },
+    {
+      "entropy": 2.079618126153946,
+      "epoch": 0.07312614259597806,
+      "grad_norm": 0.18688173592090607,
+      "learning_rate": 0.0001990654765773811,
+      "loss": 1.5780220031738281,
+      "mean_token_accuracy": 0.6635978065431118,
+      "num_tokens": 210273.0,
+      "step": 90
+    },
+    {
+      "entropy": 2.0404782213270662,
+      "epoch": 0.08125126955108673,
+      "grad_norm": 0.13348598778247833,
+      "learning_rate": 0.00019867235749578105,
+      "loss": 1.6611948013305664,
+      "mean_token_accuracy": 0.665754010155797,
+      "num_tokens": 233214.0,
+      "step": 100
+    },
+    {
+      "entropy": 2.005169425904751,
+      "epoch": 0.08937639650619542,
+      "grad_norm": 0.18108932673931122,
+      "learning_rate": 0.00019821093199149804,
+      "loss": 1.727360153198242,
+      "mean_token_accuracy": 0.6582555785775185,
+      "num_tokens": 257311.0,
+      "step": 110
+    },
+    {
+      "entropy": 2.105305866897106,
+      "epoch": 0.09750152346130408,
+      "grad_norm": 0.22547709941864014,
+      "learning_rate": 0.00019768151948859703,
+      "loss": 1.543342685699463,
+      "mean_token_accuracy": 0.6652009710669518,
+      "num_tokens": 279714.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.9379282005131244,
+      "epoch": 0.10562665041641275,
+      "grad_norm": 0.17329610884189606,
+      "learning_rate": 0.00019708448647547572,
+      "loss": 1.734849739074707,
+      "mean_token_accuracy": 0.6563399123027921,
+      "num_tokens": 304996.0,
+      "step": 130
+    },
+    {
+      "entropy": 2.0480446301400663,
+      "epoch": 0.11375177737152142,
+      "grad_norm": 0.1423395276069641,
+      "learning_rate": 0.00019642024625116117,
+      "loss": 1.6252754211425782,
+      "mean_token_accuracy": 0.6467269618064165,
+      "num_tokens": 329343.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.9707386873662471,
+      "epoch": 0.1218769043266301,
+      "grad_norm": 0.1983540803194046,
+      "learning_rate": 0.00019568925863920153,
+      "loss": 1.6639902114868164,
+      "mean_token_accuracy": 0.6920234341174364,
+      "num_tokens": 353472.0,
+      "step": 150
+    },
+    {
+      "entropy": 2.014208319038153,
+      "epoch": 0.1300020312817388,
+      "grad_norm": 0.21575114130973816,
+      "learning_rate": 0.00019489202966935083,
+      "loss": 1.6030187606811523,
+      "mean_token_accuracy": 0.6612130492925644,
+      "num_tokens": 375387.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.902009778469801,
+      "epoch": 0.13812715823684746,
+      "grad_norm": 0.1551465392112732,
+      "learning_rate": 0.00019402911122726757,
+      "loss": 1.5153232574462892,
+      "mean_token_accuracy": 0.6612629793584347,
+      "num_tokens": 401732.0,
+      "step": 170
+    },
+    {
+      "entropy": 2.0072270750999452,
+      "epoch": 0.14625228519195613,
+      "grad_norm": 0.10310117155313492,
+      "learning_rate": 0.00019310110067246905,
+      "loss": 1.7293834686279297,
+      "mean_token_accuracy": 0.6383478887379169,
+      "num_tokens": 424853.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.8907569527626038,
+      "epoch": 0.1543774121470648,
+      "grad_norm": 0.2138219177722931,
+      "learning_rate": 0.00019210864042480642,
+      "loss": 1.5713120460510255,
+      "mean_token_accuracy": 0.6743554215878248,
+      "num_tokens": 448291.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.9789937749505042,
+      "epoch": 0.16250253910217347,
+      "grad_norm": 0.15141330659389496,
+      "learning_rate": 0.00019105241751974622,
+      "loss": 1.572166633605957,
+      "mean_token_accuracy": 0.6682488189078868,
+      "num_tokens": 474028.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.845897100120783,
+      "epoch": 0.17062766605728213,
+      "grad_norm": 0.1917404979467392,
+      "learning_rate": 0.00018993316313276694,
+      "loss": 1.576710033416748,
+      "mean_token_accuracy": 0.6819654058665037,
+      "num_tokens": 498062.0,
+      "step": 210
+    },
+    {
+      "entropy": 2.0329321578145025,
+      "epoch": 0.17875279301239083,
+      "grad_norm": 0.16201205551624298,
+      "learning_rate": 0.000188751652073199,
+      "loss": 1.779829978942871,
+      "mean_token_accuracy": 0.6354137845337391,
+      "num_tokens": 521369.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.9103941090404988,
+      "epoch": 0.1868779199674995,
+      "grad_norm": 0.16283521056175232,
+      "learning_rate": 0.00018750870224785939,
+      "loss": 1.336418342590332,
+      "mean_token_accuracy": 0.678225276246667,
+      "num_tokens": 543485.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.9498405829071999,
+      "epoch": 0.19500304692260817,
+      "grad_norm": 0.22851474583148956,
+      "learning_rate": 0.0001862051740948515,
+      "loss": 1.4225791931152343,
+      "mean_token_accuracy": 0.6880000069737434,
+      "num_tokens": 567331.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.7782155938446522,
+      "epoch": 0.20312817387771684,
+      "grad_norm": 0.1257830709218979,
+      "learning_rate": 0.00018484196998792268,
+      "loss": 1.5439845085144044,
+      "mean_token_accuracy": 0.7007725708186626,
+      "num_tokens": 590687.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.9237571090459824,
+      "epoch": 0.2112533008328255,
+      "grad_norm": 0.15280641615390778,
+      "learning_rate": 0.00018342003361179176,
+      "loss": 1.5627306938171386,
+      "mean_token_accuracy": 0.6518422529101372,
+      "num_tokens": 611348.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.8481825087219477,
+      "epoch": 0.21937842778793418,
+      "grad_norm": 0.1840401291847229,
+      "learning_rate": 0.0001819403493088784,
+      "loss": 1.4731447219848632,
+      "mean_token_accuracy": 0.6832829281687737,
+      "num_tokens": 634210.0,
+      "step": 270
+    },
+    {
+      "entropy": 2.0215828701853753,
+      "epoch": 0.22750355474304285,
+      "grad_norm": 0.23153692483901978,
+      "learning_rate": 0.00018040394139788747,
+      "loss": 1.5676209449768066,
+      "mean_token_accuracy": 0.6582915022969246,
+      "num_tokens": 655604.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.9178163021802903,
+      "epoch": 0.23562868169815154,
+      "grad_norm": 0.20892998576164246,
+      "learning_rate": 0.00017881187346471925,
+      "loss": 1.3779240608215333,
+      "mean_token_accuracy": 0.6975597772747278,
+      "num_tokens": 678860.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.9228212662041186,
+      "epoch": 0.2437538086532602,
+      "grad_norm": 0.08906886726617813,
+      "learning_rate": 0.00017716524762619696,
+      "loss": 1.6122535705566405,
+      "mean_token_accuracy": 0.6579667629674077,
+      "num_tokens": 703872.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.921762604266405,
+      "epoch": 0.2518789356083689,
+      "grad_norm": 0.19159626960754395,
+      "learning_rate": 0.00017546520376712094,
+      "loss": 1.5556479454040528,
+      "mean_token_accuracy": 0.6880384348332882,
+      "num_tokens": 728412.0,
+      "step": 310
+    },
+    {
+      "entropy": 2.0291903913021088,
+      "epoch": 0.2600040625634776,
+      "grad_norm": 0.2337758094072342,
+      "learning_rate": 0.0001737129187511779,
+      "loss": 1.4515840530395507,
+      "mean_token_accuracy": 0.680047171190381,
+      "num_tokens": 749824.0,
+      "step": 320
+    },
+    {
+      "entropy": 2.117575029283762,
+      "epoch": 0.2681291895185862,
+      "grad_norm": 0.1862848997116089,
+      "learning_rate": 0.00017190960560625125,
+      "loss": 1.6359136581420899,
+      "mean_token_accuracy": 0.6340366339311003,
+      "num_tokens": 773090.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.9986298732459544,
+      "epoch": 0.2762543164736949,
+      "grad_norm": 0.15575817227363586,
+      "learning_rate": 0.0001700565126846965,
+      "loss": 1.5904709815979003,
+      "mean_token_accuracy": 0.6757547052577138,
+      "num_tokens": 793500.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.999005551636219,
+      "epoch": 0.28437944342880356,
+      "grad_norm": 0.11942591518163681,
+      "learning_rate": 0.0001681549227991634,
+      "loss": 1.4471738815307618,
+      "mean_token_accuracy": 0.6856243159621954,
+      "num_tokens": 815883.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.9639676824212073,
+      "epoch": 0.29250457038391225,
+      "grad_norm": 0.1216198205947876,
+      "learning_rate": 0.00016620615233456235,
+      "loss": 1.489202880859375,
+      "mean_token_accuracy": 0.6687835985794663,
+      "num_tokens": 839091.0,
+      "step": 360
+    },
+    {
+      "entropy": 2.0271141096949576,
+      "epoch": 0.3006296973390209,
+      "grad_norm": 0.1932452917098999,
+      "learning_rate": 0.00016421155033679083,
+      "loss": 1.8303487777709961,
+      "mean_token_accuracy": 0.6432886853814125,
+      "num_tokens": 861682.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.9302572283893824,
+      "epoch": 0.3087548242941296,
+      "grad_norm": 0.13882122933864594,
+      "learning_rate": 0.00016217249757884955,
+      "loss": 1.6748510360717774,
+      "mean_token_accuracy": 0.6589554883539677,
+      "num_tokens": 885779.0,
+      "step": 380
+    },
+    {
+      "entropy": 2.0911385156214237,
+      "epoch": 0.3168799512492383,
+      "grad_norm": 0.19290806353092194,
+      "learning_rate": 0.0001600904056049955,
+      "loss": 1.5958806991577148,
+      "mean_token_accuracy": 0.6881534941494465,
+      "num_tokens": 910323.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.98078987672925,
+      "epoch": 0.32500507820434693,
+      "grad_norm": 0.16565003991127014,
+      "learning_rate": 0.00015796671575359384,
+      "loss": 1.608301544189453,
+      "mean_token_accuracy": 0.656030460447073,
+      "num_tokens": 933556.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.8239209279417992,
+      "epoch": 0.33313020515945563,
+      "grad_norm": 0.15926608443260193,
+      "learning_rate": 0.00015580289815934401,
+      "loss": 1.6252323150634767,
+      "mean_token_accuracy": 0.6608501821756363,
+      "num_tokens": 958102.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.9631126798689364,
+      "epoch": 0.34125533211456427,
+      "grad_norm": 0.17559975385665894,
+      "learning_rate": 0.00015360045073557212,
+      "loss": 1.6342245101928712,
+      "mean_token_accuracy": 0.6673539975658059,
+      "num_tokens": 982664.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.9809245102107524,
+      "epoch": 0.34938045906967297,
+      "grad_norm": 0.1391943246126175,
+      "learning_rate": 0.00015136089813729274,
+      "loss": 1.3244733810424805,
+      "mean_token_accuracy": 0.6789292085915803,
+      "num_tokens": 1005145.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.9501206412911416,
+      "epoch": 0.35750558602478166,
+      "grad_norm": 0.16100212931632996,
+      "learning_rate": 0.00014908579070575936,
+      "loss": 1.552811622619629,
+      "mean_token_accuracy": 0.6740785460919142,
+      "num_tokens": 1029122.0,
+      "step": 440
+    },
+    {
+      "entropy": 2.0843734957277773,
+      "epoch": 0.3656307129798903,
+      "grad_norm": 0.18199782073497772,
+      "learning_rate": 0.00014677670339523284,
+      "loss": 1.5700489044189454,
+      "mean_token_accuracy": 0.6624482493847609,
+      "num_tokens": 1049948.0,
+      "step": 450
+    },
+    {
+      "entropy": 2.039452239125967,
+      "epoch": 0.373755839934999,
+      "grad_norm": 0.13164274394512177,
+      "learning_rate": 0.00014443523468271168,
+      "loss": 1.6703521728515625,
+      "mean_token_accuracy": 0.6379993572831154,
+      "num_tokens": 1072542.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.9393630679696798,
+      "epoch": 0.38188096689010764,
+      "grad_norm": 0.17090488970279694,
+      "learning_rate": 0.00014206300546137842,
+      "loss": 1.5892380714416503,
+      "mean_token_accuracy": 0.6695126354694366,
+      "num_tokens": 1097904.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.9853965878486632,
+      "epoch": 0.39000609384521634,
+      "grad_norm": 0.18743044137954712,
+      "learning_rate": 0.00013966165791852862,
+      "loss": 1.5395258903503417,
+      "mean_token_accuracy": 0.6718302246183157,
+      "num_tokens": 1119677.0,
+      "step": 480
+    },
+    {
+      "entropy": 2.010790354758501,
+      "epoch": 0.398131220800325,
+      "grad_norm": 0.2401738464832306,
+      "learning_rate": 0.00013723285439875836,
+      "loss": 1.3885634422302247,
+      "mean_token_accuracy": 0.704565803706646,
+      "num_tokens": 1141914.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.997419325262308,
+      "epoch": 0.4062563477554337,
+      "grad_norm": 0.18942418694496155,
+      "learning_rate": 0.00013477827625319824,
+      "loss": 1.7568876266479492,
+      "mean_token_accuracy": 0.6620310273021459,
+      "num_tokens": 1163422.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.9955256462097168,
+      "epoch": 0.4143814747105424,
+      "grad_norm": 0.14334970712661743,
+      "learning_rate": 0.00013229962267558983,
+      "loss": 1.4751434326171875,
+      "mean_token_accuracy": 0.6814542073756457,
+      "num_tokens": 1188603.0,
+      "step": 510
+    },
+    {
+      "entropy": 2.024552647024393,
+      "epoch": 0.422506601665651,
+      "grad_norm": 0.17734989523887634,
+      "learning_rate": 0.00012979860952601037,
+      "loss": 1.7202960968017578,
+      "mean_token_accuracy": 0.6529360726475716,
+      "num_tokens": 1214616.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.9225171849131584,
+      "epoch": 0.4306317286207597,
+      "grad_norm": 0.15514154732227325,
+      "learning_rate": 0.00012727696814306033,
+      "loss": 1.5036141395568847,
+      "mean_token_accuracy": 0.6889493480324745,
+      "num_tokens": 1237875.0,
+      "step": 530
+    },
+    {
+      "entropy": 2.0581463038921357,
+      "epoch": 0.43875685557586835,
+      "grad_norm": 0.2063213586807251,
+      "learning_rate": 0.0001247364441453357,
+      "loss": 1.7413753509521483,
+      "mean_token_accuracy": 0.6764257464557886,
+      "num_tokens": 1258144.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.9071560092270374,
+      "epoch": 0.44688198253097705,
+      "grad_norm": 0.1607491821050644,
+      "learning_rate": 0.00012217879622301514,
+      "loss": 1.5522031784057617,
+      "mean_token_accuracy": 0.670783769339323,
+      "num_tokens": 1286898.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.8957423724234104,
+      "epoch": 0.4550071094860857,
+      "grad_norm": 0.16279400885105133,
+      "learning_rate": 0.00011960579492039783,
+      "loss": 1.6234397888183594,
+      "mean_token_accuracy": 0.6461524413898587,
+      "num_tokens": 1309139.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.8500158667564393,
+      "epoch": 0.4631322364411944,
+      "grad_norm": 0.12710601091384888,
+      "learning_rate": 0.00011701922141023567,
+      "loss": 1.599781608581543,
+      "mean_token_accuracy": 0.6745339043438434,
+      "num_tokens": 1331405.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.9107272654771805,
+      "epoch": 0.4712573633963031,
+      "grad_norm": 0.1596456915140152,
+      "learning_rate": 0.0001144208662607078,
+      "loss": 1.6400453567504882,
+      "mean_token_accuracy": 0.658748508989811,
+      "num_tokens": 1354598.0,
+      "step": 580
+    },
+    {
+      "entropy": 2.042666858434677,
+      "epoch": 0.4793824903514117,
+      "grad_norm": 0.1512349545955658,
+      "learning_rate": 0.00011181252819589081,
+      "loss": 1.4765432357788086,
+      "mean_token_accuracy": 0.6941530019044876,
+      "num_tokens": 1376377.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.994793000817299,
+      "epoch": 0.4875076173065204,
+      "grad_norm": 0.2148679494857788,
+      "learning_rate": 0.0001091960128505833,
+      "loss": 1.5141250610351562,
+      "mean_token_accuracy": 0.6714166019111871,
+      "num_tokens": 1396427.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.838432003185153,
+      "epoch": 0.49563274426162907,
+      "grad_norm": 0.15919055044651031,
+      "learning_rate": 0.00010657313152034634,
+      "loss": 1.4393814086914063,
+      "mean_token_accuracy": 0.7031023040413856,
+      "num_tokens": 1417585.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.904674556106329,
+      "epoch": 0.5037578712167378,
+      "grad_norm": 0.15948715806007385,
+      "learning_rate": 0.00010394569990762529,
+      "loss": 1.6298444747924805,
+      "mean_token_accuracy": 0.6670018272474408,
+      "num_tokens": 1438360.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.87869431078434,
+      "epoch": 0.5118829981718465,
+      "grad_norm": 0.15786612033843994,
+      "learning_rate": 0.00010131553686482076,
+      "loss": 1.6015625,
+      "mean_token_accuracy": 0.6777582582086324,
+      "num_tokens": 1462646.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.9668042108416557,
+      "epoch": 0.5200081251269552,
+      "grad_norm": 0.12120363116264343,
+      "learning_rate": 9.868446313517926e-05,
+      "loss": 1.3548295021057128,
+      "mean_token_accuracy": 0.7010443404316902,
+      "num_tokens": 1489685.0,
+      "step": 640
+    },
+    {
+      "entropy": 2.1403770953416825,
+      "epoch": 0.5281332520820637,
+      "grad_norm": 0.16519835591316223,
+      "learning_rate": 9.605430009237474e-05,
+      "loss": 1.4757318496704102,
+      "mean_token_accuracy": 0.6792042136192322,
+      "num_tokens": 1512744.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.9175119310617448,
+      "epoch": 0.5362583790371724,
+      "grad_norm": 0.20694105327129364,
+      "learning_rate": 9.342686847965367e-05,
+      "loss": 1.5559422492980957,
+      "mean_token_accuracy": 0.6729984357953072,
+      "num_tokens": 1536621.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.8454748161137104,
+      "epoch": 0.5443835059922811,
+      "grad_norm": 0.1792803555727005,
+      "learning_rate": 9.080398714941671e-05,
+      "loss": 1.518070888519287,
+      "mean_token_accuracy": 0.6720059467479587,
+      "num_tokens": 1561403.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.8773800782859325,
+      "epoch": 0.5525086329473898,
+      "grad_norm": 0.1524674892425537,
+      "learning_rate": 8.818747180410921e-05,
+      "loss": 1.4121664047241211,
+      "mean_token_accuracy": 0.6844700604677201,
+      "num_tokens": 1584267.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.8458331644535064,
+      "epoch": 0.5606337599024985,
+      "grad_norm": 0.13008670508861542,
+      "learning_rate": 8.557913373929221e-05,
+      "loss": 1.3976998329162598,
+      "mean_token_accuracy": 0.7024147201329469,
+      "num_tokens": 1607485.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.9154148779809475,
+      "epoch": 0.5687588868576071,
+      "grad_norm": 0.158284530043602,
+      "learning_rate": 8.298077858976435e-05,
+      "loss": 1.4725753784179687,
+      "mean_token_accuracy": 0.6702791546471417,
+      "num_tokens": 1632634.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.9224348738789558,
+      "epoch": 0.5768840138127158,
+      "grad_norm": 0.16229023039340973,
+      "learning_rate": 8.03942050796022e-05,
+      "loss": 1.7397420883178711,
+      "mean_token_accuracy": 0.6504182794131339,
+      "num_tokens": 1655589.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.899767429381609,
+      "epoch": 0.5850091407678245,
+      "grad_norm": 0.1681690663099289,
+      "learning_rate": 7.782120377698488e-05,
+      "loss": 1.5257241249084472,
+      "mean_token_accuracy": 0.6727124411612749,
+      "num_tokens": 1679908.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.9996836170554162,
+      "epoch": 0.5931342677229332,
+      "grad_norm": 0.1216152012348175,
+      "learning_rate": 7.526355585466431e-05,
+      "loss": 1.5072300910949707,
+      "mean_token_accuracy": 0.6908476516604424,
+      "num_tokens": 1704031.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.926835112273693,
+      "epoch": 0.6012593946780418,
+      "grad_norm": 0.14855222404003143,
+      "learning_rate": 7.27230318569397e-05,
+      "loss": 1.566415500640869,
+      "mean_token_accuracy": 0.6910070944577456,
+      "num_tokens": 1726263.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.8111965112388133,
+      "epoch": 0.6093845216331505,
+      "grad_norm": 0.12660442292690277,
+      "learning_rate": 7.020139047398966e-05,
+      "loss": 1.6191877365112304,
+      "mean_token_accuracy": 0.6778128875419498,
+      "num_tokens": 1754645.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.9391085349023343,
+      "epoch": 0.6175096485882592,
+      "grad_norm": 0.1532779335975647,
+      "learning_rate": 6.77003773244102e-05,
+      "loss": 1.4570680618286134,
+      "mean_token_accuracy": 0.6896491345018149,
+      "num_tokens": 1777982.0,
+      "step": 760
+    },
+    {
+      "entropy": 2.12602856233716,
+      "epoch": 0.6256347755433679,
+      "grad_norm": 0.10179242491722107,
+      "learning_rate": 6.522172374680177e-05,
+      "loss": 1.7333276748657227,
+      "mean_token_accuracy": 0.6630699183791876,
+      "num_tokens": 1799019.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.812769826501608,
+      "epoch": 0.6337599024984766,
+      "grad_norm": 0.15165607631206512,
+      "learning_rate": 6.276714560124167e-05,
+      "loss": 1.5185394287109375,
+      "mean_token_accuracy": 0.6665599981322885,
+      "num_tokens": 1824205.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.9916068516671657,
+      "epoch": 0.6418850294535852,
+      "grad_norm": 0.0905507355928421,
+      "learning_rate": 6.03383420814714e-05,
+      "loss": 1.7346548080444335,
+      "mean_token_accuracy": 0.659119957126677,
+      "num_tokens": 1845109.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.9239101201295852,
+      "epoch": 0.6500101564086939,
+      "grad_norm": 0.10977555066347122,
+      "learning_rate": 5.7936994538621605e-05,
+      "loss": 1.4736296653747558,
+      "mean_token_accuracy": 0.6861130066215992,
+      "num_tokens": 1871993.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.841003315895796,
+      "epoch": 0.6581352833638026,
+      "grad_norm": 0.143220454454422,
+      "learning_rate": 5.556476531728836e-05,
+      "loss": 1.5567984580993652,
+      "mean_token_accuracy": 0.6601953901350498,
+      "num_tokens": 1893142.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.8885623142123222,
+      "epoch": 0.6662604103189113,
+      "grad_norm": 0.14200888574123383,
+      "learning_rate": 5.322329660476715e-05,
+      "loss": 1.58746395111084,
+      "mean_token_accuracy": 0.6524630783125758,
+      "num_tokens": 1915797.0,
+      "step": 820
+    },
+    {
+      "entropy": 2.0285811945796013,
+      "epoch": 0.67438553727402,
+      "grad_norm": 0.1395159512758255,
+      "learning_rate": 5.0914209294240644e-05,
+      "loss": 1.7256057739257813,
+      "mean_token_accuracy": 0.6439347952604294,
+      "num_tokens": 1938143.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.9870487309992313,
+      "epoch": 0.6825106642291285,
+      "grad_norm": 0.17539235949516296,
+      "learning_rate": 4.863910186270726e-05,
+      "loss": 1.6983217239379882,
+      "mean_token_accuracy": 0.6498236363753677,
+      "num_tokens": 1961108.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.928568222373724,
+      "epoch": 0.6906357911842372,
+      "grad_norm": 0.07064857333898544,
+      "learning_rate": 4.639954926442792e-05,
+      "loss": 1.6033218383789063,
+      "mean_token_accuracy": 0.6499791458249092,
+      "num_tokens": 1984934.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.848902615904808,
+      "epoch": 0.6987609181393459,
+      "grad_norm": 0.14302754402160645,
+      "learning_rate": 4.4197101840655995e-05,
+      "loss": 1.5059507369995118,
+      "mean_token_accuracy": 0.6897634303197264,
+      "num_tokens": 2010312.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.9174821764230727,
+      "epoch": 0.7068860450944546,
+      "grad_norm": 0.1510535478591919,
+      "learning_rate": 4.203328424640619e-05,
+      "loss": 1.4589906692504884,
+      "mean_token_accuracy": 0.6884843675419688,
+      "num_tokens": 2035628.0,
+      "step": 870
+    },
+    {
+      "entropy": 2.164568604528904,
+      "epoch": 0.7150111720495633,
+      "grad_norm": 0.17442022264003754,
+      "learning_rate": 3.9909594395004543e-05,
+      "loss": 1.622574806213379,
+      "mean_token_accuracy": 0.6489699203521013,
+      "num_tokens": 2052903.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.8666753612458706,
+      "epoch": 0.7231362990046719,
+      "grad_norm": 0.15167613327503204,
+      "learning_rate": 3.7827502421150496e-05,
+      "loss": 1.4149527549743652,
+      "mean_token_accuracy": 0.7061239235103131,
+      "num_tokens": 2076063.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.8622376903891564,
+      "epoch": 0.7312614259597806,
+      "grad_norm": 0.16495293378829956,
+      "learning_rate": 3.5788449663209164e-05,
+      "loss": 1.9993160247802735,
+      "mean_token_accuracy": 0.6758415076881648,
+      "num_tokens": 2104647.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.9537413019686938,
+      "epoch": 0.7393865529148893,
+      "grad_norm": 0.10427695512771606,
+      "learning_rate": 3.379384766543767e-05,
+      "loss": 1.4439330101013184,
+      "mean_token_accuracy": 0.6766054792329669,
+      "num_tokens": 2127012.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.9438895985484124,
+      "epoch": 0.747511679869998,
+      "grad_norm": 0.12728701531887054,
+      "learning_rate": 3.1845077200836636e-05,
+      "loss": 1.4841145515441894,
+      "mean_token_accuracy": 0.6948042303323746,
+      "num_tokens": 2150261.0,
+      "step": 920
+    },
+    {
+      "entropy": 2.0161457642912866,
+      "epoch": 0.7556368068251066,
+      "grad_norm": 0.13816353678703308,
+      "learning_rate": 2.9943487315303488e-05,
+      "loss": 1.6217718124389648,
+      "mean_token_accuracy": 0.6536502651870251,
+      "num_tokens": 2174027.0,
+      "step": 930
+    },
+    {
+      "entropy": 2.0158501587808133,
+      "epoch": 0.7637619337802153,
+      "grad_norm": 0.1435156613588333,
+      "learning_rate": 2.8090394393748776e-05,
+      "loss": 1.592111301422119,
+      "mean_token_accuracy": 0.6928597155958414,
+      "num_tokens": 2194322.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.897237876802683,
+      "epoch": 0.771887060735324,
+      "grad_norm": 0.16316495835781097,
+      "learning_rate": 2.628708124882212e-05,
+      "loss": 1.361118984222412,
+      "mean_token_accuracy": 0.6913821775466203,
+      "num_tokens": 2218351.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.9368196219205855,
+      "epoch": 0.7800121876904327,
+      "grad_norm": 0.1807290017604828,
+      "learning_rate": 2.4534796232879088e-05,
+      "loss": 1.6152563095092773,
+      "mean_token_accuracy": 0.6723910128697753,
+      "num_tokens": 2239161.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.8723509922623633,
+      "epoch": 0.7881373146455414,
+      "grad_norm": 0.12329761683940887,
+      "learning_rate": 2.2834752373803093e-05,
+      "loss": 1.8951400756835937,
+      "mean_token_accuracy": 0.6896139021962882,
+      "num_tokens": 2262279.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.9010609984397888,
+      "epoch": 0.79626244160065,
+      "grad_norm": 0.1437692791223526,
+      "learning_rate": 2.1188126535280773e-05,
+      "loss": 1.643233299255371,
+      "mean_token_accuracy": 0.6725624911487103,
+      "num_tokens": 2285696.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.909880855679512,
+      "epoch": 0.8043875685557587,
+      "grad_norm": 0.20211046934127808,
+      "learning_rate": 1.9596058602112532e-05,
+      "loss": 1.5162543296813964,
+      "mean_token_accuracy": 0.6684278029948473,
+      "num_tokens": 2308763.0,
+      "step": 990
+    },
+    {
+      "entropy": 2.029320741444826,
+      "epoch": 0.8125126955108674,
+      "grad_norm": 0.1881091296672821,
+      "learning_rate": 1.805965069112161e-05,
+      "loss": 1.5837879180908203,
+      "mean_token_accuracy": 0.6746429082006216,
+      "num_tokens": 2332473.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.9360487267374993,
+      "epoch": 0.820637822465976,
+      "grad_norm": 0.15552105009555817,
+      "learning_rate": 1.657996638820826e-05,
+      "loss": 1.5739222526550294,
+      "mean_token_accuracy": 0.6809787165373564,
+      "num_tokens": 2356129.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.9467576876282693,
+      "epoch": 0.8287629494210847,
+      "grad_norm": 0.1283911019563675,
+      "learning_rate": 1.515803001207733e-05,
+      "loss": 1.583531951904297,
+      "mean_token_accuracy": 0.6996182476170361,
+      "num_tokens": 2377892.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.7585147149860858,
+      "epoch": 0.8368880763761933,
+      "grad_norm": 0.13573195040225983,
+      "learning_rate": 1.3794825905148556e-05,
+      "loss": 1.4390297889709474,
+      "mean_token_accuracy": 0.6716821633279324,
+      "num_tokens": 2404528.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.879253412038088,
+      "epoch": 0.845013203331302,
+      "grad_norm": 0.21262964606285095,
+      "learning_rate": 1.2491297752140641e-05,
+      "loss": 1.5953598022460938,
+      "mean_token_accuracy": 0.6597621485590934,
+      "num_tokens": 2430050.0,
+      "step": 1040
+    },
+    {
+      "entropy": 2.016388051211834,
+      "epoch": 0.8531383302864107,
+      "grad_norm": 0.1353832334280014,
+      "learning_rate": 1.1248347926801029e-05,
+      "loss": 1.5296365737915039,
+      "mean_token_accuracy": 0.6748634882271289,
+      "num_tokens": 2449747.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.8552228838205338,
+      "epoch": 0.8612634572415194,
+      "grad_norm": 0.14027485251426697,
+      "learning_rate": 1.0066836867233087e-05,
+      "loss": 1.3939000129699708,
+      "mean_token_accuracy": 0.7101718856021761,
+      "num_tokens": 2474602.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.881848829239607,
+      "epoch": 0.8693885841966281,
+      "grad_norm": 0.0767660140991211,
+      "learning_rate": 8.94758248025378e-06,
+      "loss": 1.6371177673339843,
+      "mean_token_accuracy": 0.6367173055186868,
+      "num_tokens": 2498614.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.8897530138492584,
+      "epoch": 0.8775137111517367,
+      "grad_norm": 0.10006698220968246,
+      "learning_rate": 7.891359575193613e-06,
+      "loss": 1.6738821029663087,
+      "mean_token_accuracy": 0.6588716686703264,
+      "num_tokens": 2521304.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.8935686632990838,
+      "epoch": 0.8856388381068454,
+      "grad_norm": 0.12744677066802979,
+      "learning_rate": 6.89889932753095e-06,
+      "loss": 1.605390739440918,
+      "mean_token_accuracy": 0.6406372897326946,
+      "num_tokens": 2545673.0,
+      "step": 1090
+    },
+    {
+      "entropy": 2.038338375091553,
+      "epoch": 0.8937639650619541,
+      "grad_norm": 0.11845318973064423,
+      "learning_rate": 5.9708887727324525e-06,
+      "loss": 1.6947301864624023,
+      "mean_token_accuracy": 0.6590529561042786,
+      "num_tokens": 2565571.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.8626789033412934,
+      "epoch": 0.9018890920170628,
+      "grad_norm": 0.16855306923389435,
+      "learning_rate": 5.107970330649203e-06,
+      "loss": 1.6673316955566406,
+      "mean_token_accuracy": 0.6753990761935711,
+      "num_tokens": 2590825.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.9398136168718338,
+      "epoch": 0.9100142189721714,
+      "grad_norm": 0.08136139065027237,
+      "learning_rate": 4.310741360798498e-06,
+      "loss": 1.4246086120605468,
+      "mean_token_accuracy": 0.6877250816673041,
+      "num_tokens": 2615077.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.9428842857480049,
+      "epoch": 0.9181393459272801,
+      "grad_norm": 0.11654014140367508,
+      "learning_rate": 3.5797537488388323e-06,
+      "loss": 1.4067914962768555,
+      "mean_token_accuracy": 0.6889954376965761,
+      "num_tokens": 2637050.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.9370744809508325,
+      "epoch": 0.9262644728823888,
+      "grad_norm": 0.19872760772705078,
+      "learning_rate": 2.9155135245242937e-06,
+      "loss": 1.3668843269348145,
+      "mean_token_accuracy": 0.6870502477511764,
+      "num_tokens": 2658421.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.827120342105627,
+      "epoch": 0.9343895998374975,
+      "grad_norm": 0.13387741148471832,
+      "learning_rate": 2.3184805114029874e-06,
+      "loss": 1.5319839477539063,
+      "mean_token_accuracy": 0.6793972857296466,
+      "num_tokens": 2685099.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.7636012688279152,
+      "epoch": 0.9425147267926062,
+      "grad_norm": 0.14174917340278625,
+      "learning_rate": 1.7890680085019595e-06,
+      "loss": 1.4695980072021484,
+      "mean_token_accuracy": 0.6799993138760329,
+      "num_tokens": 2715099.0,
+      "step": 1160
+    },
+    {
+      "entropy": 2.030419469252229,
+      "epoch": 0.9506398537477148,
+      "grad_norm": 0.1873985081911087,
+      "learning_rate": 1.327642504218951e-06,
+      "loss": 1.7033708572387696,
+      "mean_token_accuracy": 0.6299138834699989,
+      "num_tokens": 2737953.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.9925663582980633,
+      "epoch": 0.9587649807028235,
+      "grad_norm": 0.13355718553066254,
+      "learning_rate": 9.345234226189159e-07,
+      "loss": 1.7240467071533203,
+      "mean_token_accuracy": 0.6594984937459231,
+      "num_tokens": 2762404.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.9695138450711966,
+      "epoch": 0.9668901076579322,
+      "grad_norm": 0.12667681276798248,
+      "learning_rate": 6.099829023112235e-07,
+      "loss": 1.4509984970092773,
+      "mean_token_accuracy": 0.6755119284614921,
+      "num_tokens": 2782406.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.9426129013299942,
+      "epoch": 0.9750152346130408,
+      "grad_norm": 0.16676834225654602,
+      "learning_rate": 3.5424560806036623e-07,
+      "loss": 1.499323844909668,
+      "mean_token_accuracy": 0.6907339114695787,
+      "num_tokens": 2804526.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.9200426444411278,
+      "epoch": 0.9831403615681495,
+      "grad_norm": 0.21395370364189148,
+      "learning_rate": 1.6748857526066586e-07,
+      "loss": 1.4064488410949707,
+      "mean_token_accuracy": 0.6922622859477997,
+      "num_tokens": 2826453.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.9110548786818982,
+      "epoch": 0.9912654885232581,
+      "grad_norm": 0.06316321343183517,
+      "learning_rate": 4.9841087382618276e-08,
+      "loss": 1.6310953140258788,
+      "mean_token_accuracy": 0.7007135484367609,
+      "num_tokens": 2854164.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.880154137313366,
+      "epoch": 0.9993906154783668,
+      "grad_norm": 0.13141165673732758,
+      "learning_rate": 1.3845864758610383e-09,
+      "loss": 1.4970791816711426,
+      "mean_token_accuracy": 0.6883187366649508,
+      "num_tokens": 2878060.0,
+      "step": 1230
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1231,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.465582342060851e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1231/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e720c84011f9b0ed4df8c50c9a1645ddea7ccdee1accb75ebc6bbedf8c39ad5
+size 5677

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e720c84011f9b0ed4df8c50c9a1645ddea7ccdee1accb75ebc6bbedf8c39ad5
+size 5677