Add outputs/E0-baseline

.gitattributes

@@ -41,3 +41,10 @@ logs/figures/E0_entropy_live.png filter=lfs diff=lfs merge=lfs -text
 logs/figures/E0_live_dashboard.png filter=lfs diff=lfs merge=lfs -text
 outputs/pairs_4b/multipos_train.jsonl filter=lfs diff=lfs merge=lfs -text
 outputs/pool_4b/pool_train.jsonl filter=lfs diff=lfs merge=lfs -text
+outputs/E0-baseline/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+outputs/E0-baseline/checkpoint-150/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+outputs/E0-baseline/checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+outputs/E0-baseline/checkpoint-250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+outputs/E0-baseline/checkpoint-300/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+outputs/E0-baseline/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+outputs/E0-baseline/final/tokenizer.json filter=lfs diff=lfs merge=lfs -text

outputs/E0-baseline/README.md

@@ -0,0 +1,67 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: transformers
+model_name: E0-baseline
+tags:
+- generated_from_trainer
+- grpo
+- trl
+licence: license
+---
+
+# Model Card for E0-baseline
+
+This model is a fine-tuned version of [Qwen/Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/pranav2278/div-grpo/runs/hhvcvkst) 
+
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 1.10.0
+- Transformers: 5.15.0
+- Pytorch: 2.13.0
+- Datasets: 5.0.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{shao2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

outputs/E0-baseline/checkpoint-100/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

outputs/E0-baseline/checkpoint-100/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

outputs/E0-baseline/checkpoint-100/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58b3ac37aaa4179bc96d7e703dc29bc19755704e1761cdb06e990b9f8c77be82
+size 264308896

outputs/E0-baseline/checkpoint-100/chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

outputs/E0-baseline/checkpoint-100/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

outputs/E0-baseline/checkpoint-100/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

outputs/E0-baseline/checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6
+size 7697

outputs/E0-baseline/checkpoint-150/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

outputs/E0-baseline/checkpoint-150/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

outputs/E0-baseline/checkpoint-150/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbafc11133a226488bab7d56c436ec4755811aa888b20c9438b4949404543856
+size 264308896

outputs/E0-baseline/checkpoint-150/chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

outputs/E0-baseline/checkpoint-150/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

outputs/E0-baseline/checkpoint-150/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

outputs/E0-baseline/checkpoint-150/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6
+size 7697

outputs/E0-baseline/checkpoint-200/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

outputs/E0-baseline/checkpoint-200/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

outputs/E0-baseline/checkpoint-200/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:794e31718cff267ee61edd0a5fa827f87f1677b8f7de4a21a2058d2f3a5a21fd
+size 264308896

outputs/E0-baseline/checkpoint-200/chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

outputs/E0-baseline/checkpoint-200/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

outputs/E0-baseline/checkpoint-200/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

outputs/E0-baseline/checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6
+size 7697

outputs/E0-baseline/checkpoint-250/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

outputs/E0-baseline/checkpoint-250/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

outputs/E0-baseline/checkpoint-250/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61ec658a42b607a64873f110f786c01afb192f0abf7fbc70a1f4ee67e79f279a
+size 264308896

outputs/E0-baseline/checkpoint-250/chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

outputs/E0-baseline/checkpoint-250/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

outputs/E0-baseline/checkpoint-250/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

outputs/E0-baseline/checkpoint-250/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6
+size 7697

outputs/E0-baseline/checkpoint-300/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

outputs/E0-baseline/checkpoint-300/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

outputs/E0-baseline/checkpoint-300/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f546b7b6fd17327ebb9554c32f1730033ed34378b532ee5fe945489c3a60734
+size 264308896

outputs/E0-baseline/checkpoint-300/chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

outputs/E0-baseline/checkpoint-300/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

outputs/E0-baseline/checkpoint-300/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

outputs/E0-baseline/checkpoint-300/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6
+size 7697

outputs/E0-baseline/checkpoint-50/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

outputs/E0-baseline/checkpoint-50/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

outputs/E0-baseline/checkpoint-50/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75620b4583564edd639fd6f92613b6bf52d8fca52850fd95cf3f82686b022bdd
+size 264308896

outputs/E0-baseline/checkpoint-50/chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

outputs/E0-baseline/checkpoint-50/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

outputs/E0-baseline/checkpoint-50/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

outputs/E0-baseline/checkpoint-50/trainer_state.json

@@ -0,0 +1,1684 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.1,
+  "eval_steps": 500,
+  "global_step": 50,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 593.0,
+      "completions/max_terminated_length": 593.0,
+      "completions/mean_length": 529.0,
+      "completions/mean_terminated_length": 529.0,
+      "completions/min_length": 482.0,
+      "completions/min_terminated_length": 482.0,
+      "entropy": 1.2025159299373627,
+      "epoch": 0.002,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22668755054473877,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.2398601919412613,
+      "num_tokens": 10400.0,
+      "reward": 6.375,
+      "reward_std": 0.5,
+      "rewards/quality_reward/mean": 6.375,
+      "rewards/quality_reward/std": 0.5,
+      "sampling/importance_sampling_ratio/max": 1.6477458477020264,
+      "sampling/importance_sampling_ratio/mean": 0.5064857602119446,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.40894174575805664,
+      "sampling/sampling_logp_difference/mean": 0.026567919179797173,
+      "step": 1,
+      "step_time": 12.760562099982053
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 644.0,
+      "completions/max_terminated_length": 644.0,
+      "completions/mean_length": 562.25,
+      "completions/mean_terminated_length": 562.25,
+      "completions/min_length": 497.0,
+      "completions/min_terminated_length": 497.0,
+      "entropy": 1.1930748969316483,
+      "epoch": 0.004,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12967805564403534,
+      "kl": 0.0,
+      "learning_rate": 3e-06,
+      "loss": -0.08571265637874603,
+      "num_tokens": 20924.0,
+      "reward": 6.1875,
+      "reward_std": 1.1086779832839966,
+      "rewards/quality_reward/mean": 6.1875,
+      "rewards/quality_reward/std": 1.1086779832839966,
+      "sampling/importance_sampling_ratio/max": 1.0037232637405396,
+      "sampling/importance_sampling_ratio/mean": 0.41275694966316223,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.45699238777160645,
+      "sampling/sampling_logp_difference/mean": 0.025086138397455215,
+      "step": 2,
+      "step_time": 11.406366533134133
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 542.0,
+      "completions/max_terminated_length": 542.0,
+      "completions/mean_length": 483.625,
+      "completions/mean_terminated_length": 483.625,
+      "completions/min_length": 407.0,
+      "completions/min_terminated_length": 407.0,
+      "entropy": 1.1096689626574516,
+      "epoch": 0.006,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22966289520263672,
+      "kl": 0.0008355328354809899,
+      "learning_rate": 6e-06,
+      "loss": 0.020913563668727875,
+      "num_tokens": 30222.0,
+      "reward": 6.875,
+      "reward_std": 0.3415650427341461,
+      "rewards/quality_reward/mean": 6.875,
+      "rewards/quality_reward/std": 0.3415650427341461,
+      "sampling/importance_sampling_ratio/max": 1.927120566368103,
+      "sampling/importance_sampling_ratio/mean": 0.6114993095397949,
+      "sampling/importance_sampling_ratio/min": 0.11067161709070206,
+      "sampling/sampling_logp_difference/max": 0.4620504379272461,
+      "sampling/sampling_logp_difference/mean": 0.024864500388503075,
+      "step": 3,
+      "step_time": 26.480680393986404
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 608.0,
+      "completions/max_terminated_length": 608.0,
+      "completions/mean_length": 524.75,
+      "completions/mean_terminated_length": 524.75,
+      "completions/min_length": 473.0,
+      "completions/min_terminated_length": 473.0,
+      "entropy": 1.211122527718544,
+      "epoch": 0.008,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.30378466844558716,
+      "kl": 0.0008403196770814247,
+      "learning_rate": 9e-06,
+      "loss": -0.12959149479866028,
+      "num_tokens": 40410.0,
+      "reward": 6.25,
+      "reward_std": 0.8563488721847534,
+      "rewards/quality_reward/mean": 6.25,
+      "rewards/quality_reward/std": 0.8563488721847534,
+      "sampling/importance_sampling_ratio/max": 2.406888961791992,
+      "sampling/importance_sampling_ratio/mean": 0.5457419753074646,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.3680229187011719,
+      "sampling/sampling_logp_difference/mean": 0.02656388282775879,
+      "step": 4,
+      "step_time": 22.95301443943754
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 546.0,
+      "completions/max_terminated_length": 546.0,
+      "completions/mean_length": 487.5625,
+      "completions/mean_terminated_length": 487.5625,
+      "completions/min_length": 441.0,
+      "completions/min_terminated_length": 441.0,
+      "entropy": 1.247180238366127,
+      "epoch": 0.01,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5199307799339294,
+      "kl": 0.0008723233368073124,
+      "learning_rate": 1.2e-05,
+      "loss": 0.11524428427219391,
+      "num_tokens": 49915.0,
+      "reward": 6.4375,
+      "reward_std": 0.6291528940200806,
+      "rewards/quality_reward/mean": 6.4375,
+      "rewards/quality_reward/std": 0.6291528940200806,
+      "sampling/importance_sampling_ratio/max": 2.1544158458709717,
+      "sampling/importance_sampling_ratio/mean": 0.963020920753479,
+      "sampling/importance_sampling_ratio/min": 0.04948288947343826,
+      "sampling/sampling_logp_difference/max": 0.4774587154388428,
+      "sampling/sampling_logp_difference/mean": 0.02625642716884613,
+      "step": 5,
+      "step_time": 22.37928077671677
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 648.0,
+      "completions/max_terminated_length": 648.0,
+      "completions/mean_length": 533.0,
+      "completions/mean_terminated_length": 533.0,
+      "completions/min_length": 394.0,
+      "completions/min_terminated_length": 394.0,
+      "entropy": 1.1693861335515976,
+      "epoch": 0.012,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.27643197774887085,
+      "kl": 0.0009261858467652928,
+      "learning_rate": 1.5e-05,
+      "loss": 0.15093231201171875,
+      "num_tokens": 60219.0,
+      "reward": 6.1875,
+      "reward_std": 0.981070876121521,
+      "rewards/quality_reward/mean": 6.1875,
+      "rewards/quality_reward/std": 0.981070876121521,
+      "sampling/importance_sampling_ratio/max": 2.125091791152954,
+      "sampling/importance_sampling_ratio/mean": 0.7733402252197266,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.7049552202224731,
+      "sampling/sampling_logp_difference/mean": 0.025986071676015854,
+      "step": 6,
+      "step_time": 21.00841654697433
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 574.0,
+      "completions/max_terminated_length": 574.0,
+      "completions/mean_length": 505.6875,
+      "completions/mean_terminated_length": 505.6875,
+      "completions/min_length": 425.0,
+      "completions/min_terminated_length": 425.0,
+      "entropy": 1.2473183795809746,
+      "epoch": 0.014,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2166663259267807,
+      "kl": 0.0009701631606731098,
+      "learning_rate": 1.8e-05,
+      "loss": -0.08582288026809692,
+      "num_tokens": 69902.0,
+      "reward": 7.1875,
+      "reward_std": 0.75,
+      "rewards/quality_reward/mean": 7.1875,
+      "rewards/quality_reward/std": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.8287386894226074,
+      "sampling/importance_sampling_ratio/mean": 0.5286832451820374,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.38030898571014404,
+      "sampling/sampling_logp_difference/mean": 0.0264718160033226,
+      "step": 7,
+      "step_time": 46.596127359196544
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 537.0,
+      "completions/max_terminated_length": 537.0,
+      "completions/mean_length": 482.4375,
+      "completions/mean_terminated_length": 482.4375,
+      "completions/min_length": 392.0,
+      "completions/min_terminated_length": 392.0,
+      "entropy": 1.1040107272565365,
+      "epoch": 0.016,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18874908983707428,
+      "kl": 0.0010721117541834246,
+      "learning_rate": 2.1e-05,
+      "loss": -0.1946130096912384,
+      "num_tokens": 79501.0,
+      "reward": 6.75,
+      "reward_std": 0.5773502588272095,
+      "rewards/quality_reward/mean": 6.75,
+      "rewards/quality_reward/std": 0.5773502588272095,
+      "sampling/importance_sampling_ratio/max": 2.3561792373657227,
+      "sampling/importance_sampling_ratio/mean": 0.6918502449989319,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.26114892959594727,
+      "sampling/sampling_logp_difference/mean": 0.02554757334291935,
+      "step": 8,
+      "step_time": 25.433595282491297
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 554.0,
+      "completions/max_terminated_length": 554.0,
+      "completions/mean_length": 490.1875,
+      "completions/mean_terminated_length": 490.1875,
+      "completions/min_length": 463.0,
+      "completions/min_terminated_length": 463.0,
+      "entropy": 1.1377170644700527,
+      "epoch": 0.018,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.204215869307518,
+      "kl": 0.002002388624532614,
+      "learning_rate": 2.4e-05,
+      "loss": -0.08130021393299103,
+      "num_tokens": 88976.0,
+      "reward": 6.5,
+      "reward_std": 0.5163977742195129,
+      "rewards/quality_reward/mean": 6.5,
+      "rewards/quality_reward/std": 0.5163977742195129,
+      "sampling/importance_sampling_ratio/max": 2.0982444286346436,
+      "sampling/importance_sampling_ratio/mean": 0.5873216986656189,
+      "sampling/importance_sampling_ratio/min": 0.04890051856637001,
+      "sampling/sampling_logp_difference/max": 0.8512144088745117,
+      "sampling/sampling_logp_difference/mean": 0.02638406865298748,
+      "step": 9,
+      "step_time": 18.427699581719935
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 588.0,
+      "completions/max_terminated_length": 588.0,
+      "completions/mean_length": 500.1875,
+      "completions/mean_terminated_length": 500.1875,
+      "completions/min_length": 400.0,
+      "completions/min_terminated_length": 400.0,
+      "entropy": 1.1230391450226307,
+      "epoch": 0.02,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1985069215297699,
+      "kl": 0.0026735300780273974,
+      "learning_rate": 2.7000000000000002e-05,
+      "loss": -0.12387816607952118,
+      "num_tokens": 98603.0,
+      "reward": 6.8125,
+      "reward_std": 0.40311288833618164,
+      "rewards/quality_reward/mean": 6.8125,
+      "rewards/quality_reward/std": 0.40311288833618164,
+      "sampling/importance_sampling_ratio/max": 1.983199954032898,
+      "sampling/importance_sampling_ratio/mean": 0.5639468431472778,
+      "sampling/importance_sampling_ratio/min": 0.012905921787023544,
+      "sampling/sampling_logp_difference/max": 0.3653905391693115,
+      "sampling/sampling_logp_difference/mean": 0.025383003056049347,
+      "step": 10,
+      "step_time": 14.99981931829825
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 526.0,
+      "completions/max_terminated_length": 526.0,
+      "completions/mean_length": 459.6875,
+      "completions/mean_terminated_length": 459.6875,
+      "completions/min_length": 379.0,
+      "completions/min_terminated_length": 379.0,
+      "entropy": 1.213625729084015,
+      "epoch": 0.022,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3695620596408844,
+      "kl": 0.00424640768324025,
+      "learning_rate": 3e-05,
+      "loss": -0.06913074851036072,
+      "num_tokens": 107734.0,
+      "reward": 6.8125,
+      "reward_std": 0.40311288833618164,
+      "rewards/quality_reward/mean": 6.8125,
+      "rewards/quality_reward/std": 0.40311288833618164,
+      "sampling/importance_sampling_ratio/max": 2.000136375427246,
+      "sampling/importance_sampling_ratio/mean": 0.8144867420196533,
+      "sampling/importance_sampling_ratio/min": 0.06302778422832489,
+      "sampling/sampling_logp_difference/max": 0.3647327423095703,
+      "sampling/sampling_logp_difference/mean": 0.026674197986721992,
+      "step": 11,
+      "step_time": 30.46549107739702
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 555.0,
+      "completions/max_terminated_length": 555.0,
+      "completions/mean_length": 482.0625,
+      "completions/mean_terminated_length": 482.0625,
+      "completions/min_length": 428.0,
+      "completions/min_terminated_length": 428.0,
+      "entropy": 1.191277615725994,
+      "epoch": 0.024,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.35457733273506165,
+      "kl": 0.007200263120466843,
+      "learning_rate": 3e-05,
+      "loss": 0.22097699344158173,
+      "num_tokens": 117199.0,
+      "reward": 5.9375,
+      "reward_std": 0.9979145526885986,
+      "rewards/quality_reward/mean": 5.9375,
+      "rewards/quality_reward/std": 0.9979145526885986,
+      "sampling/importance_sampling_ratio/max": 1.5660414695739746,
+      "sampling/importance_sampling_ratio/mean": 0.649204432964325,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.3921785354614258,
+      "sampling/sampling_logp_difference/mean": 0.02726689912378788,
+      "step": 12,
+      "step_time": 15.719243939965963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 577.0,
+      "completions/max_terminated_length": 577.0,
+      "completions/mean_length": 464.3125,
+      "completions/mean_terminated_length": 464.3125,
+      "completions/min_length": 391.0,
+      "completions/min_terminated_length": 391.0,
+      "entropy": 1.24251464381814,
+      "epoch": 0.026,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.30047014355659485,
+      "kl": 0.0058551667898427695,
+      "learning_rate": 3e-05,
+      "loss": -0.07746122777462006,
+      "num_tokens": 126556.0,
+      "reward": 6.6875,
+      "reward_std": 0.4787135720252991,
+      "rewards/quality_reward/mean": 6.6875,
+      "rewards/quality_reward/std": 0.4787135720252991,
+      "sampling/importance_sampling_ratio/max": 2.6028401851654053,
+      "sampling/importance_sampling_ratio/mean": 0.7561360597610474,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.4946432113647461,
+      "sampling/sampling_logp_difference/mean": 0.02639186754822731,
+      "step": 13,
+      "step_time": 18.08984712138772
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 511.0,
+      "completions/max_terminated_length": 511.0,
+      "completions/mean_length": 444.8125,
+      "completions/mean_terminated_length": 444.8125,
+      "completions/min_length": 389.0,
+      "completions/min_terminated_length": 389.0,
+      "entropy": 1.1501125395298004,
+      "epoch": 0.028,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.24105942249298096,
+      "kl": 0.012796793889719993,
+      "learning_rate": 3e-05,
+      "loss": 0.10855278372764587,
+      "num_tokens": 135417.0,
+      "reward": 3.1875,
+      "reward_std": 3.310966730117798,
+      "rewards/quality_reward/mean": 3.1875,
+      "rewards/quality_reward/std": 3.310966730117798,
+      "sampling/importance_sampling_ratio/max": 2.541518211364746,
+      "sampling/importance_sampling_ratio/mean": 0.5903321504592896,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.8390979766845703,
+      "sampling/sampling_logp_difference/mean": 0.02838001400232315,
+      "step": 14,
+      "step_time": 20.055794408079237
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 497.0,
+      "completions/max_terminated_length": 497.0,
+      "completions/mean_length": 442.25,
+      "completions/mean_terminated_length": 442.25,
+      "completions/min_length": 386.0,
+      "completions/min_terminated_length": 386.0,
+      "entropy": 1.1262825280427933,
+      "epoch": 0.03,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19040776789188385,
+      "kl": 0.010701361010433175,
+      "learning_rate": 3e-05,
+      "loss": -0.007966680452227592,
+      "num_tokens": 144205.0,
+      "reward": 5.875,
+      "reward_std": 0.8850612044334412,
+      "rewards/quality_reward/mean": 5.875,
+      "rewards/quality_reward/std": 0.8850612044334412,
+      "sampling/importance_sampling_ratio/max": 1.0754293203353882,
+      "sampling/importance_sampling_ratio/mean": 0.41446638107299805,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.3982200622558594,
+      "sampling/sampling_logp_difference/mean": 0.027210108935832977,
+      "step": 15,
+      "step_time": 14.176074750721455
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 531.0,
+      "completions/max_terminated_length": 531.0,
+      "completions/mean_length": 478.125,
+      "completions/mean_terminated_length": 478.125,
+      "completions/min_length": 433.0,
+      "completions/min_terminated_length": 433.0,
+      "entropy": 1.2985924184322357,
+      "epoch": 0.032,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.23625652492046356,
+      "kl": 0.0213887135614641,
+      "learning_rate": 3e-05,
+      "loss": -0.21546132862567902,
+      "num_tokens": 153543.0,
+      "reward": 6.5,
+      "reward_std": 0.730296790599823,
+      "rewards/quality_reward/mean": 6.5,
+      "rewards/quality_reward/std": 0.730296790599823,
+      "sampling/importance_sampling_ratio/max": 2.0074336528778076,
+      "sampling/importance_sampling_ratio/mean": 0.49076417088508606,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.5095968246459961,
+      "sampling/sampling_logp_difference/mean": 0.028833162039518356,
+      "step": 16,
+      "step_time": 15.848205763846636
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 526.0,
+      "completions/max_terminated_length": 526.0,
+      "completions/mean_length": 469.25,
+      "completions/mean_terminated_length": 469.25,
+      "completions/min_length": 423.0,
+      "completions/min_terminated_length": 423.0,
+      "entropy": 1.3148910626769066,
+      "epoch": 0.034,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17765119671821594,
+      "kl": 0.02128879475640133,
+      "learning_rate": 3e-05,
+      "loss": -0.0828079879283905,
+      "num_tokens": 163043.0,
+      "reward": 6.0625,
+      "reward_std": 1.0626225471496582,
+      "rewards/quality_reward/mean": 6.0625,
+      "rewards/quality_reward/std": 1.0626225471496582,
+      "sampling/importance_sampling_ratio/max": 1.5988941192626953,
+      "sampling/importance_sampling_ratio/mean": 0.5021570324897766,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.591062068939209,
+      "sampling/sampling_logp_difference/mean": 0.030804751440882683,
+      "step": 17,
+      "step_time": 28.313011147081852
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 501.0,
+      "completions/max_terminated_length": 501.0,
+      "completions/mean_length": 447.0,
+      "completions/mean_terminated_length": 447.0,
+      "completions/min_length": 388.0,
+      "completions/min_terminated_length": 388.0,
+      "entropy": 1.390664003789425,
+      "epoch": 0.036,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.37244123220443726,
+      "kl": 0.019650122558232397,
+      "learning_rate": 3e-05,
+      "loss": -0.01184748113155365,
+      "num_tokens": 172379.0,
+      "reward": 6.0,
+      "reward_std": 0.8944272398948669,
+      "rewards/quality_reward/mean": 6.0,
+      "rewards/quality_reward/std": 0.8944272398948669,
+      "sampling/importance_sampling_ratio/max": 2.675238847732544,
+      "sampling/importance_sampling_ratio/mean": 0.9581233263015747,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.5762512683868408,
+      "sampling/sampling_logp_difference/mean": 0.03126702085137367,
+      "step": 18,
+      "step_time": 30.991567107848823
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 498.0,
+      "completions/max_terminated_length": 498.0,
+      "completions/mean_length": 447.75,
+      "completions/mean_terminated_length": 447.75,
+      "completions/min_length": 383.0,
+      "completions/min_terminated_length": 383.0,
+      "entropy": 1.3287223279476166,
+      "epoch": 0.038,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.23734751343727112,
+      "kl": 0.022738213243428618,
+      "learning_rate": 3e-05,
+      "loss": 0.040024783462285995,
+      "num_tokens": 181239.0,
+      "reward": 6.4375,
+      "reward_std": 1.209338665008545,
+      "rewards/quality_reward/mean": 6.4375,
+      "rewards/quality_reward/std": 1.209338665008545,
+      "sampling/importance_sampling_ratio/max": 2.095722198486328,
+      "sampling/importance_sampling_ratio/mean": 0.6408629417419434,
+      "sampling/importance_sampling_ratio/min": 0.1203346848487854,
+      "sampling/sampling_logp_difference/max": 0.4722297191619873,
+      "sampling/sampling_logp_difference/mean": 0.030378391966223717,
+      "step": 19,
+      "step_time": 18.615950418636203
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 659.0,
+      "completions/max_terminated_length": 659.0,
+      "completions/mean_length": 452.3125,
+      "completions/mean_terminated_length": 452.3125,
+      "completions/min_length": 376.0,
+      "completions/min_terminated_length": 376.0,
+      "entropy": 1.1001618690788746,
+      "epoch": 0.04,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.34637194871902466,
+      "kl": 0.015380491153337061,
+      "learning_rate": 3e-05,
+      "loss": 0.1691148728132248,
+      "num_tokens": 190068.0,
+      "reward": 6.125,
+      "reward_std": 1.3102163076400757,
+      "rewards/quality_reward/mean": 6.125,
+      "rewards/quality_reward/std": 1.3102163076400757,
+      "sampling/importance_sampling_ratio/max": 2.6566128730773926,
+      "sampling/importance_sampling_ratio/mean": 0.7767290472984314,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.6531033515930176,
+      "sampling/sampling_logp_difference/mean": 0.030463142320513725,
+      "step": 20,
+      "step_time": 15.741292077116668
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 589.0,
+      "completions/max_terminated_length": 589.0,
+      "completions/mean_length": 457.5625,
+      "completions/mean_terminated_length": 457.5625,
+      "completions/min_length": 390.0,
+      "completions/min_terminated_length": 390.0,
+      "entropy": 1.3708399310708046,
+      "epoch": 0.042,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2514922618865967,
+      "kl": 0.018277494702488184,
+      "learning_rate": 3e-05,
+      "loss": -0.13425321877002716,
+      "num_tokens": 198941.0,
+      "reward": 6.5625,
+      "reward_std": 0.9639329314231873,
+      "rewards/quality_reward/mean": 6.5625,
+      "rewards/quality_reward/std": 0.9639329314231873,
+      "sampling/importance_sampling_ratio/max": 1.5612297058105469,
+      "sampling/importance_sampling_ratio/mean": 0.601022481918335,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.4818992614746094,
+      "sampling/sampling_logp_difference/mean": 0.0315740592777729,
+      "step": 21,
+      "step_time": 17.24192419089377
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 508.0,
+      "completions/max_terminated_length": 508.0,
+      "completions/mean_length": 421.4375,
+      "completions/mean_terminated_length": 421.4375,
+      "completions/min_length": 360.0,
+      "completions/min_terminated_length": 360.0,
+      "entropy": 1.136269599199295,
+      "epoch": 0.044,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2268020063638687,
+      "kl": 0.017973962530959398,
+      "learning_rate": 3e-05,
+      "loss": 0.010622119531035423,
+      "num_tokens": 207300.0,
+      "reward": 6.5,
+      "reward_std": 0.8164966106414795,
+      "rewards/quality_reward/mean": 6.5,
+      "rewards/quality_reward/std": 0.8164966106414795,
+      "sampling/importance_sampling_ratio/max": 1.7986358404159546,
+      "sampling/importance_sampling_ratio/mean": 0.46136727929115295,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.5335149765014648,
+      "sampling/sampling_logp_difference/mean": 0.02804401144385338,
+      "step": 22,
+      "step_time": 19.37282825494185
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 503.0,
+      "completions/max_terminated_length": 503.0,
+      "completions/mean_length": 452.6875,
+      "completions/mean_terminated_length": 452.6875,
+      "completions/min_length": 381.0,
+      "completions/min_terminated_length": 381.0,
+      "entropy": 1.3299130946397781,
+      "epoch": 0.046,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.33994877338409424,
+      "kl": 0.021804221265483648,
+      "learning_rate": 3e-05,
+      "loss": -0.24404363334178925,
+      "num_tokens": 216343.0,
+      "reward": 6.375,
+      "reward_std": 0.8062257766723633,
+      "rewards/quality_reward/mean": 6.375,
+      "rewards/quality_reward/std": 0.8062257766723633,
+      "sampling/importance_sampling_ratio/max": 2.452249526977539,
+      "sampling/importance_sampling_ratio/mean": 0.747193455696106,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.6328549385070801,
+      "sampling/sampling_logp_difference/mean": 0.02918298915028572,
+      "step": 23,
+      "step_time": 15.356728344224393
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 524.0,
+      "completions/max_terminated_length": 524.0,
+      "completions/mean_length": 462.4375,
+      "completions/mean_terminated_length": 462.4375,
+      "completions/min_length": 410.0,
+      "completions/min_terminated_length": 410.0,
+      "entropy": 1.379701942205429,
+      "epoch": 0.048,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3076202869415283,
+      "kl": 0.020299295720178634,
+      "learning_rate": 3e-05,
+      "loss": -0.09123071283102036,
+      "num_tokens": 225550.0,
+      "reward": 6.9375,
+      "reward_std": 0.4425306022167206,
+      "rewards/quality_reward/mean": 6.9375,
+      "rewards/quality_reward/std": 0.4425306022167206,
+      "sampling/importance_sampling_ratio/max": 1.5845099687576294,
+      "sampling/importance_sampling_ratio/mean": 0.6625345945358276,
+      "sampling/importance_sampling_ratio/min": 0.04495502635836601,
+      "sampling/sampling_logp_difference/max": 0.36547183990478516,
+      "sampling/sampling_logp_difference/mean": 0.028946854174137115,
+      "step": 24,
+      "step_time": 14.95462113339454
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 618.0,
+      "completions/max_terminated_length": 618.0,
+      "completions/mean_length": 482.4375,
+      "completions/mean_terminated_length": 482.4375,
+      "completions/min_length": 382.0,
+      "completions/min_terminated_length": 382.0,
+      "entropy": 1.2471638694405556,
+      "epoch": 0.05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22180576622486115,
+      "kl": 0.018309170845896006,
+      "learning_rate": 3e-05,
+      "loss": -0.1412944793701172,
+      "num_tokens": 235005.0,
+      "reward": 6.6875,
+      "reward_std": 0.6020797491073608,
+      "rewards/quality_reward/mean": 6.6875,
+      "rewards/quality_reward/std": 0.6020797491073608,
+      "sampling/importance_sampling_ratio/max": 2.6407876014709473,
+      "sampling/importance_sampling_ratio/mean": 0.7712795734405518,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.4363563060760498,
+      "sampling/sampling_logp_difference/mean": 0.02898716926574707,
+      "step": 25,
+      "step_time": 16.46686205593869
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 555.0,
+      "completions/max_terminated_length": 555.0,
+      "completions/mean_length": 455.75,
+      "completions/mean_terminated_length": 455.75,
+      "completions/min_length": 364.0,
+      "completions/min_terminated_length": 364.0,
+      "entropy": 1.28530602902174,
+      "epoch": 0.052,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.37535253167152405,
+      "kl": 0.020504546992015094,
+      "learning_rate": 3e-05,
+      "loss": 0.10839397460222244,
+      "num_tokens": 243953.0,
+      "reward": 6.25,
+      "reward_std": 1.0645813941955566,
+      "rewards/quality_reward/mean": 6.25,
+      "rewards/quality_reward/std": 1.0645813941955566,
+      "sampling/importance_sampling_ratio/max": 2.0567266941070557,
+      "sampling/importance_sampling_ratio/mean": 0.5800145864486694,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.4523625373840332,
+      "sampling/sampling_logp_difference/mean": 0.027610119432210922,
+      "step": 26,
+      "step_time": 22.121026155073196
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 514.0,
+      "completions/max_terminated_length": 514.0,
+      "completions/mean_length": 462.75,
+      "completions/mean_terminated_length": 462.75,
+      "completions/min_length": 400.0,
+      "completions/min_terminated_length": 400.0,
+      "entropy": 1.2253629937767982,
+      "epoch": 0.054,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2340388149023056,
+      "kl": 0.020236384589225054,
+      "learning_rate": 3e-05,
+      "loss": 0.04823978245258331,
+      "num_tokens": 253237.0,
+      "reward": 6.375,
+      "reward_std": 0.8062257766723633,
+      "rewards/quality_reward/mean": 6.375,
+      "rewards/quality_reward/std": 0.8062257766723633,
+      "sampling/importance_sampling_ratio/max": 1.1560932397842407,
+      "sampling/importance_sampling_ratio/mean": 0.40753045678138733,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.35022830963134766,
+      "sampling/sampling_logp_difference/mean": 0.028367744758725166,
+      "step": 27,
+      "step_time": 15.10967448912561
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 538.0,
+      "completions/max_terminated_length": 538.0,
+      "completions/mean_length": 482.0625,
+      "completions/mean_terminated_length": 482.0625,
+      "completions/min_length": 438.0,
+      "completions/min_terminated_length": 438.0,
+      "entropy": 1.3779077678918839,
+      "epoch": 0.056,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5531017184257507,
+      "kl": 0.01706669357372448,
+      "learning_rate": 3e-05,
+      "loss": 0.0933581069111824,
+      "num_tokens": 262454.0,
+      "reward": 6.375,
+      "reward_std": 0.7187952995300293,
+      "rewards/quality_reward/mean": 6.375,
+      "rewards/quality_reward/std": 0.7187952995300293,
+      "sampling/importance_sampling_ratio/max": 2.5711586475372314,
+      "sampling/importance_sampling_ratio/mean": 0.7730721831321716,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.343827486038208,
+      "sampling/sampling_logp_difference/mean": 0.028883326798677444,
+      "step": 28,
+      "step_time": 38.59647103771567
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 563.0,
+      "completions/max_terminated_length": 563.0,
+      "completions/mean_length": 477.0,
+      "completions/mean_terminated_length": 477.0,
+      "completions/min_length": 410.0,
+      "completions/min_terminated_length": 410.0,
+      "entropy": 1.3791070505976677,
+      "epoch": 0.058,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.33771538734436035,
+      "kl": 0.02141271048458293,
+      "learning_rate": 3e-05,
+      "loss": 0.010216176509857178,
+      "num_tokens": 271918.0,
+      "reward": 5.75,
+      "reward_std": 1.2909945249557495,
+      "rewards/quality_reward/mean": 5.75,
+      "rewards/quality_reward/std": 1.2909945249557495,
+      "sampling/importance_sampling_ratio/max": 2.4999797344207764,
+      "sampling/importance_sampling_ratio/mean": 1.0250636339187622,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.3258817195892334,
+      "sampling/sampling_logp_difference/mean": 0.029029540717601776,
+      "step": 29,
+      "step_time": 23.610272648278624
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 575.0,
+      "completions/max_terminated_length": 575.0,
+      "completions/mean_length": 471.1875,
+      "completions/mean_terminated_length": 471.1875,
+      "completions/min_length": 381.0,
+      "completions/min_terminated_length": 381.0,
+      "entropy": 1.4652926102280617,
+      "epoch": 0.06,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.21955685317516327,
+      "kl": 0.019562674744520336,
+      "learning_rate": 3e-05,
+      "loss": -0.014814459718763828,
+      "num_tokens": 281305.0,
+      "reward": 6.625,
+      "reward_std": 0.8850612044334412,
+      "rewards/quality_reward/mean": 6.625,
+      "rewards/quality_reward/std": 0.8850612044334412,
+      "sampling/importance_sampling_ratio/max": 2.802098274230957,
+      "sampling/importance_sampling_ratio/mean": 0.866391658782959,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.6354451179504395,
+      "sampling/sampling_logp_difference/mean": 0.03177585452795029,
+      "step": 30,
+      "step_time": 16.961607525125146
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 602.0,
+      "completions/max_terminated_length": 602.0,
+      "completions/mean_length": 490.5,
+      "completions/mean_terminated_length": 490.5,
+      "completions/min_length": 411.0,
+      "completions/min_terminated_length": 411.0,
+      "entropy": 1.1957123205065727,
+      "epoch": 0.062,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12176825106143951,
+      "kl": 0.026934220048133284,
+      "learning_rate": 3e-05,
+      "loss": -0.08307373523712158,
+      "num_tokens": 291409.0,
+      "reward": 6.8125,
+      "reward_std": 0.8341662883758545,
+      "rewards/quality_reward/mean": 6.8125,
+      "rewards/quality_reward/std": 0.8341662883758545,
+      "sampling/importance_sampling_ratio/max": 1.610858678817749,
+      "sampling/importance_sampling_ratio/mean": 0.6937527656555176,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.4253416061401367,
+      "sampling/sampling_logp_difference/mean": 0.02871524728834629,
+      "step": 31,
+      "step_time": 15.012207658961415
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 508.0,
+      "completions/max_terminated_length": 508.0,
+      "completions/mean_length": 462.0,
+      "completions/mean_terminated_length": 462.0,
+      "completions/min_length": 381.0,
+      "completions/min_terminated_length": 381.0,
+      "entropy": 1.3018206283450127,
+      "epoch": 0.064,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4979296624660492,
+      "kl": 0.03539742121938616,
+      "learning_rate": 3e-05,
+      "loss": 0.0017175457905977964,
+      "num_tokens": 300649.0,
+      "reward": 6.875,
+      "reward_std": 0.5,
+      "rewards/quality_reward/mean": 6.875,
+      "rewards/quality_reward/std": 0.5,
+      "sampling/importance_sampling_ratio/max": 1.8527640104293823,
+      "sampling/importance_sampling_ratio/mean": 0.7824680209159851,
+      "sampling/importance_sampling_ratio/min": 0.07447884231805801,
+      "sampling/sampling_logp_difference/max": 0.5221483707427979,
+      "sampling/sampling_logp_difference/mean": 0.029107660055160522,
+      "step": 32,
+      "step_time": 22.78309725271538
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 584.0,
+      "completions/max_terminated_length": 584.0,
+      "completions/mean_length": 474.0,
+      "completions/mean_terminated_length": 474.0,
+      "completions/min_length": 382.0,
+      "completions/min_terminated_length": 382.0,
+      "entropy": 1.321175642311573,
+      "epoch": 0.066,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22016967833042145,
+      "kl": 0.029592803912237287,
+      "learning_rate": 3e-05,
+      "loss": 0.05625719577074051,
+      "num_tokens": 309889.0,
+      "reward": 6.5625,
+      "reward_std": 0.6291528940200806,
+      "rewards/quality_reward/mean": 6.5625,
+      "rewards/quality_reward/std": 0.6291528940200806,
+      "sampling/importance_sampling_ratio/max": 2.9875097274780273,
+      "sampling/importance_sampling_ratio/mean": 0.7832435369491577,
+      "sampling/importance_sampling_ratio/min": 0.18006731569766998,
+      "sampling/sampling_logp_difference/max": 0.38585615158081055,
+      "sampling/sampling_logp_difference/mean": 0.027828343212604523,
+      "step": 33,
+      "step_time": 44.51360382232815
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 510.0,
+      "completions/max_terminated_length": 510.0,
+      "completions/mean_length": 459.9375,
+      "completions/mean_terminated_length": 459.9375,
+      "completions/min_length": 410.0,
+      "completions/min_terminated_length": 410.0,
+      "entropy": 1.2222414836287498,
+      "epoch": 0.068,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2873040437698364,
+      "kl": 0.02840927499346435,
+      "learning_rate": 3e-05,
+      "loss": -0.037432070821523666,
+      "num_tokens": 318904.0,
+      "reward": 6.1875,
+      "reward_std": 1.1086779832839966,
+      "rewards/quality_reward/mean": 6.1875,
+      "rewards/quality_reward/std": 1.1086779832839966,
+      "sampling/importance_sampling_ratio/max": 2.6647069454193115,
+      "sampling/importance_sampling_ratio/mean": 0.5744763016700745,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.41987133026123047,
+      "sampling/sampling_logp_difference/mean": 0.029470665380358696,
+      "step": 34,
+      "step_time": 133.03877451224253
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 522.0,
+      "completions/max_terminated_length": 522.0,
+      "completions/mean_length": 462.1875,
+      "completions/mean_terminated_length": 462.1875,
+      "completions/min_length": 413.0,
+      "completions/min_terminated_length": 413.0,
+      "entropy": 1.1665974482893944,
+      "epoch": 0.07,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2490653246641159,
+      "kl": 0.025841041060630232,
+      "learning_rate": 3e-05,
+      "loss": -0.20422951877117157,
+      "num_tokens": 328011.0,
+      "reward": 6.9375,
+      "reward_std": 1.1814539432525635,
+      "rewards/quality_reward/mean": 6.9375,
+      "rewards/quality_reward/std": 1.1814539432525635,
+      "sampling/importance_sampling_ratio/max": 2.510730266571045,
+      "sampling/importance_sampling_ratio/mean": 0.7954420447349548,
+      "sampling/importance_sampling_ratio/min": 0.020566223189234734,
+      "sampling/sampling_logp_difference/max": 0.36430132389068604,
+      "sampling/sampling_logp_difference/mean": 0.026844775304198265,
+      "step": 35,
+      "step_time": 21.5843787365593
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 549.0,
+      "completions/max_terminated_length": 549.0,
+      "completions/mean_length": 492.0,
+      "completions/mean_terminated_length": 492.0,
+      "completions/min_length": 429.0,
+      "completions/min_terminated_length": 429.0,
+      "entropy": 1.3467887043952942,
+      "epoch": 0.072,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15268851816654205,
+      "kl": 0.023660800594370812,
+      "learning_rate": 3e-05,
+      "loss": -0.11188814043998718,
+      "num_tokens": 337731.0,
+      "reward": 6.5625,
+      "reward_std": 0.7274384498596191,
+      "rewards/quality_reward/mean": 6.5625,
+      "rewards/quality_reward/std": 0.7274384498596191,
+      "sampling/importance_sampling_ratio/max": 1.6270908117294312,
+      "sampling/importance_sampling_ratio/mean": 0.614537239074707,
+      "sampling/importance_sampling_ratio/min": 0.10853960365056992,
+      "sampling/sampling_logp_difference/max": 0.4023854732513428,
+      "sampling/sampling_logp_difference/mean": 0.028961554169654846,
+      "step": 36,
+      "step_time": 18.843947287183255
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 513.0,
+      "completions/max_terminated_length": 513.0,
+      "completions/mean_length": 460.75,
+      "completions/mean_terminated_length": 460.75,
+      "completions/min_length": 399.0,
+      "completions/min_terminated_length": 399.0,
+      "entropy": 1.2476315572857857,
+      "epoch": 0.074,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.42548227310180664,
+      "kl": 0.022181478852871805,
+      "learning_rate": 3e-05,
+      "loss": 0.37223517894744873,
+      "num_tokens": 346815.0,
+      "reward": 6.5625,
+      "reward_std": 0.5123475790023804,
+      "rewards/quality_reward/mean": 6.5625,
+      "rewards/quality_reward/std": 0.5123475790023804,
+      "sampling/importance_sampling_ratio/max": 2.6624510288238525,
+      "sampling/importance_sampling_ratio/mean": 0.7942180633544922,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.4037543535232544,
+      "sampling/sampling_logp_difference/mean": 0.028780322521924973,
+      "step": 37,
+      "step_time": 42.04662919091061
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 611.0,
+      "completions/max_terminated_length": 611.0,
+      "completions/mean_length": 452.0,
+      "completions/mean_terminated_length": 452.0,
+      "completions/min_length": 363.0,
+      "completions/min_terminated_length": 363.0,
+      "entropy": 1.1745503433048725,
+      "epoch": 0.076,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16980381309986115,
+      "kl": 0.017483733594417572,
+      "learning_rate": 3e-05,
+      "loss": -0.09029137343168259,
+      "num_tokens": 355711.0,
+      "reward": 6.9375,
+      "reward_std": 0.9287087917327881,
+      "rewards/quality_reward/mean": 6.9375,
+      "rewards/quality_reward/std": 0.9287087917327881,
+      "sampling/importance_sampling_ratio/max": 2.3201518058776855,
+      "sampling/importance_sampling_ratio/mean": 0.721072793006897,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.30433225631713867,
+      "sampling/sampling_logp_difference/mean": 0.026646045967936516,
+      "step": 38,
+      "step_time": 38.63075139513239
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 521.0,
+      "completions/max_terminated_length": 521.0,
+      "completions/mean_length": 466.8125,
+      "completions/mean_terminated_length": 466.8125,
+      "completions/min_length": 429.0,
+      "completions/min_terminated_length": 429.0,
+      "entropy": 1.2664988860487938,
+      "epoch": 0.078,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.21305795013904572,
+      "kl": 0.021121050289366394,
+      "learning_rate": 3e-05,
+      "loss": -0.03154226019978523,
+      "num_tokens": 364860.0,
+      "reward": 6.5625,
+      "reward_std": 0.8139410614967346,
+      "rewards/quality_reward/mean": 6.5625,
+      "rewards/quality_reward/std": 0.8139410614967346,
+      "sampling/importance_sampling_ratio/max": 2.0297553539276123,
+      "sampling/importance_sampling_ratio/mean": 0.674609363079071,
+      "sampling/importance_sampling_ratio/min": 0.11245065927505493,
+      "sampling/sampling_logp_difference/max": 0.37443917989730835,
+      "sampling/sampling_logp_difference/mean": 0.028257746249437332,
+      "step": 39,
+      "step_time": 30.028073193039745
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 557.0,
+      "completions/max_terminated_length": 557.0,
+      "completions/mean_length": 485.1875,
+      "completions/mean_terminated_length": 485.1875,
+      "completions/min_length": 430.0,
+      "completions/min_terminated_length": 430.0,
+      "entropy": 1.3458357080817223,
+      "epoch": 0.08,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12235487997531891,
+      "kl": 0.018535695446189493,
+      "learning_rate": 3e-05,
+      "loss": -0.035816628485918045,
+      "num_tokens": 374607.0,
+      "reward": 7.0,
+      "reward_std": 0.3651483952999115,
+      "rewards/quality_reward/mean": 7.0,
+      "rewards/quality_reward/std": 0.3651483952999115,
+      "sampling/importance_sampling_ratio/max": 2.242042303085327,
+      "sampling/importance_sampling_ratio/mean": 0.5344723463058472,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.36985254287719727,
+      "sampling/sampling_logp_difference/mean": 0.029600802809000015,
+      "step": 40,
+      "step_time": 15.446288945619017
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 594.0,
+      "completions/max_terminated_length": 594.0,
+      "completions/mean_length": 498.875,
+      "completions/mean_terminated_length": 498.875,
+      "completions/min_length": 429.0,
+      "completions/min_terminated_length": 429.0,
+      "entropy": 1.3402792811393738,
+      "epoch": 0.082,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15642686188220978,
+      "kl": 0.013967045990284532,
+      "learning_rate": 3e-05,
+      "loss": 0.0011727213859558105,
+      "num_tokens": 384317.0,
+      "reward": 6.375,
+      "reward_std": 0.8850612044334412,
+      "rewards/quality_reward/mean": 6.375,
+      "rewards/quality_reward/std": 0.8850612044334412,
+      "sampling/importance_sampling_ratio/max": 1.5691092014312744,
+      "sampling/importance_sampling_ratio/mean": 0.3696203827857971,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.6487679481506348,
+      "sampling/sampling_logp_difference/mean": 0.03018251620233059,
+      "step": 41,
+      "step_time": 18.15720977121964
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 568.0,
+      "completions/max_terminated_length": 568.0,
+      "completions/mean_length": 482.0625,
+      "completions/mean_terminated_length": 482.0625,
+      "completions/min_length": 440.0,
+      "completions/min_terminated_length": 440.0,
+      "entropy": 1.3317564576864243,
+      "epoch": 0.084,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3236794173717499,
+      "kl": 0.01707366103073582,
+      "learning_rate": 3e-05,
+      "loss": 0.10363321751356125,
+      "num_tokens": 393934.0,
+      "reward": 6.6875,
+      "reward_std": 0.7932003140449524,
+      "rewards/quality_reward/mean": 6.6875,
+      "rewards/quality_reward/std": 0.7932003140449524,
+      "sampling/importance_sampling_ratio/max": 2.0568668842315674,
+      "sampling/importance_sampling_ratio/mean": 0.6415404677391052,
+      "sampling/importance_sampling_ratio/min": 0.07682990282773972,
+      "sampling/sampling_logp_difference/max": 0.7769032716751099,
+      "sampling/sampling_logp_difference/mean": 0.02936164103448391,
+      "step": 42,
+      "step_time": 15.066733688581735
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 606.0,
+      "completions/max_terminated_length": 606.0,
+      "completions/mean_length": 510.9375,
+      "completions/mean_terminated_length": 510.9375,
+      "completions/min_length": 466.0,
+      "completions/min_terminated_length": 466.0,
+      "entropy": 1.2099780030548573,
+      "epoch": 0.086,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1771102100610733,
+      "kl": 0.01784718461567536,
+      "learning_rate": 3e-05,
+      "loss": -0.027566466480493546,
+      "num_tokens": 403893.0,
+      "reward": 6.4375,
+      "reward_std": 1.209338665008545,
+      "rewards/quality_reward/mean": 6.4375,
+      "rewards/quality_reward/std": 1.209338665008545,
+      "sampling/importance_sampling_ratio/max": 2.3430161476135254,
+      "sampling/importance_sampling_ratio/mean": 0.7769261598587036,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.36153650283813477,
+      "sampling/sampling_logp_difference/mean": 0.028799494728446007,
+      "step": 43,
+      "step_time": 16.90863296529278
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 551.0,
+      "completions/max_terminated_length": 551.0,
+      "completions/mean_length": 494.1875,
+      "completions/mean_terminated_length": 494.1875,
+      "completions/min_length": 432.0,
+      "completions/min_terminated_length": 432.0,
+      "entropy": 1.2924985438585281,
+      "epoch": 0.088,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3432070314884186,
+      "kl": 0.014529847539961338,
+      "learning_rate": 3e-05,
+      "loss": -0.04157561436295509,
+      "num_tokens": 413312.0,
+      "reward": 6.875,
+      "reward_std": 0.3415650427341461,
+      "rewards/quality_reward/mean": 6.875,
+      "rewards/quality_reward/std": 0.3415650427341461,
+      "sampling/importance_sampling_ratio/max": 2.726816177368164,
+      "sampling/importance_sampling_ratio/mean": 0.8989821672439575,
+      "sampling/importance_sampling_ratio/min": 0.07410597801208496,
+      "sampling/sampling_logp_difference/max": 0.31444358825683594,
+      "sampling/sampling_logp_difference/mean": 0.028122060000896454,
+      "step": 44,
+      "step_time": 16.7880685236305
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 637.0,
+      "completions/max_terminated_length": 637.0,
+      "completions/mean_length": 543.5,
+      "completions/mean_terminated_length": 543.5,
+      "completions/min_length": 484.0,
+      "completions/min_terminated_length": 484.0,
+      "entropy": 1.206408604979515,
+      "epoch": 0.09,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.36233776807785034,
+      "kl": 0.015796773659531027,
+      "learning_rate": 3e-05,
+      "loss": 0.029176680371165276,
+      "num_tokens": 423624.0,
+      "reward": 6.375,
+      "reward_std": 0.6191391944885254,
+      "rewards/quality_reward/mean": 6.375,
+      "rewards/quality_reward/std": 0.6191391944885254,
+      "sampling/importance_sampling_ratio/max": 1.677195429801941,
+      "sampling/importance_sampling_ratio/mean": 0.6537867188453674,
+      "sampling/importance_sampling_ratio/min": 0.09822077304124832,
+      "sampling/sampling_logp_difference/max": 0.31381869316101074,
+      "sampling/sampling_logp_difference/mean": 0.02717999368906021,
+      "step": 45,
+      "step_time": 23.35960763087496
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 643.0,
+      "completions/max_terminated_length": 643.0,
+      "completions/mean_length": 534.5625,
+      "completions/mean_terminated_length": 534.5625,
+      "completions/min_length": 453.0,
+      "completions/min_terminated_length": 453.0,
+      "entropy": 1.2577891275286674,
+      "epoch": 0.092,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20219561457633972,
+      "kl": 0.014481160265859216,
+      "learning_rate": 3e-05,
+      "loss": 0.18850615620613098,
+      "num_tokens": 433817.0,
+      "reward": 6.75,
+      "reward_std": 0.44721361994743347,
+      "rewards/quality_reward/mean": 6.75,
+      "rewards/quality_reward/std": 0.44721361994743347,
+      "sampling/importance_sampling_ratio/max": 2.203894853591919,
+      "sampling/importance_sampling_ratio/mean": 0.697495698928833,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.35237741470336914,
+      "sampling/sampling_logp_difference/mean": 0.02723248302936554,
+      "step": 46,
+      "step_time": 19.56032704282552
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 642.0,
+      "completions/max_terminated_length": 642.0,
+      "completions/mean_length": 538.25,
+      "completions/mean_terminated_length": 538.25,
+      "completions/min_length": 469.0,
+      "completions/min_terminated_length": 469.0,
+      "entropy": 1.3271892964839935,
+      "epoch": 0.094,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22262753546237946,
+      "kl": 0.012554377142805606,
+      "learning_rate": 3e-05,
+      "loss": 0.06984467804431915,
+      "num_tokens": 444045.0,
+      "reward": 6.0625,
+      "reward_std": 1.0626225471496582,
+      "rewards/quality_reward/mean": 6.0625,
+      "rewards/quality_reward/std": 1.0626225471496582,
+      "sampling/importance_sampling_ratio/max": 2.2261502742767334,
+      "sampling/importance_sampling_ratio/mean": 0.6712950468063354,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.4496123790740967,
+      "sampling/sampling_logp_difference/mean": 0.028838323429226875,
+      "step": 47,
+      "step_time": 21.226045075803995
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 628.0,
+      "completions/max_terminated_length": 628.0,
+      "completions/mean_length": 514.25,
+      "completions/mean_terminated_length": 514.25,
+      "completions/min_length": 424.0,
+      "completions/min_terminated_length": 424.0,
+      "entropy": 1.1054812744259834,
+      "epoch": 0.096,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3244606554508209,
+      "kl": 0.0157591889728792,
+      "learning_rate": 3e-05,
+      "loss": 0.09024985134601593,
+      "num_tokens": 453945.0,
+      "reward": 6.375,
+      "reward_std": 0.8062257766723633,
+      "rewards/quality_reward/mean": 6.375,
+      "rewards/quality_reward/std": 0.8062257766723633,
+      "sampling/importance_sampling_ratio/max": 2.6745388507843018,
+      "sampling/importance_sampling_ratio/mean": 0.7607913017272949,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.3650559186935425,
+      "sampling/sampling_logp_difference/mean": 0.026116060093045235,
+      "step": 48,
+      "step_time": 17.565261672716588
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 585.0,
+      "completions/max_terminated_length": 585.0,
+      "completions/mean_length": 502.4375,
+      "completions/mean_terminated_length": 502.4375,
+      "completions/min_length": 439.0,
+      "completions/min_terminated_length": 439.0,
+      "entropy": 1.2281213253736496,
+      "epoch": 0.098,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4033137857913971,
+      "kl": 0.015613102470524609,
+      "learning_rate": 3e-05,
+      "loss": -0.2898017466068268,
+      "num_tokens": 463576.0,
+      "reward": 6.4375,
+      "reward_std": 1.2632629871368408,
+      "rewards/quality_reward/mean": 6.4375,
+      "rewards/quality_reward/std": 1.2632629871368408,
+      "sampling/importance_sampling_ratio/max": 2.6974196434020996,
+      "sampling/importance_sampling_ratio/mean": 0.7124314308166504,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 0.3675417900085449,
+      "sampling/sampling_logp_difference/mean": 0.028341906145215034,
+      "step": 49,
+      "step_time": 29.838082558009773
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 573.0,
+      "completions/max_terminated_length": 573.0,
+      "completions/mean_length": 530.1875,
+      "completions/mean_terminated_length": 530.1875,
+      "completions/min_length": 458.0,
+      "completions/min_terminated_length": 458.0,
+      "entropy": 1.1955150067806244,
+      "epoch": 0.1,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.23144562542438507,
+      "kl": 0.01374751579714939,
+      "learning_rate": 3e-05,
+      "loss": -0.19065965712070465,
+      "num_tokens": 473915.0,
+      "reward": 6.3125,
+      "reward_std": 1.1954776048660278,
+      "rewards/quality_reward/mean": 6.3125,
+      "rewards/quality_reward/std": 1.1954776048660278,
+      "sampling/importance_sampling_ratio/max": 2.54063081741333,
+      "sampling/importance_sampling_ratio/mean": 0.8162041902542114,
+      "sampling/importance_sampling_ratio/min": 0.1628064066171646,
+      "sampling/sampling_logp_difference/max": 0.2777421474456787,
+      "sampling/sampling_logp_difference/mean": 0.02853373810648918,
+      "step": 50,
+      "step_time": 16.047010839451104
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 300,
+  "num_input_tokens_seen": 473915,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

outputs/E0-baseline/checkpoint-50/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6
+size 7697