GRPO trained API testing agent (200 steps, reward 7.0)

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-150/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,71 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+model_name: grpo_api_tester
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- grpo
+- lora
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for grpo_api_tester
+
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/mayankpratapsingh022-no/api-testing-grpo-v3/runs/64ixydir) 
+
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 1.0.0
+- Transformers: 5.5.0
+- Pytorch: 2.5.1+cu121
+- Datasets: 4.8.4
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{shao2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b901e70822382d594fea5f25d6c35255f8f8a8e6672725dd1a5ccc1f03e80ea4
+size 12859984

chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint-100/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-100/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-100/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3af3135e0e3d72aa87b4f3f80a05b691cc345124c98e180a8eb7ed8b763b5c0
+size 12859984

checkpoint-100/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint-100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0ba93647d9d5d0c63ab0551a702750bc5e319d960b47bb97269ae9484d0e7dc
+size 25784378

checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:107ffc3c6a81e0357a53f0323b12cf4b4be6fb32b58864913dd16ecf758541ff
+size 14244

checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33cd2334e8e11b8041f92de5ce16bfbf92f1da6c9b2885131f752b65b608ca48
+size 1064

checkpoint-100/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoint-100/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-100/trainer_state.json

@@ -0,0 +1,574 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.35,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 2360.0,
+      "completions/mean_length": 2797.55,
+      "completions/mean_terminated_length": 2071.1333984375,
+      "completions/min_length": 1845.0,
+      "completions/min_terminated_length": 1845.0,
+      "entropy": 0.09876374900341034,
+      "epoch": 0.05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.164414182305336,
+      "learning_rate": 1.9600000000000002e-05,
+      "loss": 0.24185845851898194,
+      "num_tokens": 78039.0,
+      "reward": 3.5581498622894285,
+      "reward_std": 5.68612232208252,
+      "rewards/combined_reward_fn/mean": 3.5581498622894285,
+      "rewards/combined_reward_fn/std": 5.686122417449951,
+      "step": 5,
+      "step_time": 108.90133817999158
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3651.2,
+      "completions/max_terminated_length": 2558.4,
+      "completions/mean_length": 2587.95,
+      "completions/mean_terminated_length": 2181.400048828125,
+      "completions/min_length": 1912.2,
+      "completions/min_terminated_length": 1912.2,
+      "entropy": 0.090367890894413,
+      "epoch": 0.1,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16067856550216675,
+      "learning_rate": 1.91e-05,
+      "loss": 0.12473864555358886,
+      "num_tokens": 151926.0,
+      "reward": 5.4179000616073605,
+      "reward_std": 5.021739101409912,
+      "rewards/combined_reward_fn/mean": 5.4179000616073605,
+      "rewards/combined_reward_fn/std": 5.021739077568054,
+      "step": 10,
+      "step_time": 101.38653199401452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.55,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 1866.2,
+      "completions/mean_length": 3242.2,
+      "completions/mean_terminated_length": 1720.3333740234375,
+      "completions/min_length": 2368.6,
+      "completions/min_terminated_length": 1549.4,
+      "entropy": 0.10808060467243194,
+      "epoch": 0.15,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11999707669019699,
+      "learning_rate": 1.86e-05,
+      "loss": 0.1977977991104126,
+      "num_tokens": 238722.0,
+      "reward": 2.5534999847412108,
+      "reward_std": 5.597218799591064,
+      "rewards/combined_reward_fn/mean": 2.5534999847412108,
+      "rewards/combined_reward_fn/std": 5.5972188949584964,
+      "step": 15,
+      "step_time": 123.89894773198758
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3873.0,
+      "completions/max_terminated_length": 2985.6,
+      "completions/mean_length": 2729.5,
+      "completions/mean_terminated_length": 2360.933349609375,
+      "completions/min_length": 1923.0,
+      "completions/min_terminated_length": 1923.0,
+      "entropy": 0.09633768200874329,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1394500881433487,
+      "learning_rate": 1.8100000000000003e-05,
+      "loss": 0.07866127490997314,
+      "num_tokens": 315336.0,
+      "reward": 4.6565999507904055,
+      "reward_std": 4.923504161834717,
+      "rewards/combined_reward_fn/mean": 4.6565999507904055,
+      "rewards/combined_reward_fn/std": 4.923504161834717,
+      "step": 20,
+      "step_time": 111.82979753639083
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 3064.6,
+      "completions/mean_length": 3032.15,
+      "completions/mean_terminated_length": 2677.5333984375,
+      "completions/min_length": 2128.6,
+      "completions/min_terminated_length": 2128.6,
+      "entropy": 0.09437247812747955,
+      "epoch": 0.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12652961909770966,
+      "learning_rate": 1.76e-05,
+      "loss": 0.15245428085327148,
+      "num_tokens": 397983.0,
+      "reward": 4.033899927139283,
+      "reward_std": 5.289630603790283,
+      "rewards/combined_reward_fn/mean": 4.033899927139283,
+      "rewards/combined_reward_fn/std": 5.289630889892578,
+      "step": 25,
+      "step_time": 118.41992252940545
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 3939.8,
+      "completions/max_terminated_length": 2742.6,
+      "completions/mean_length": 2717.1,
+      "completions/mean_terminated_length": 2232.1333984375,
+      "completions/min_length": 1779.6,
+      "completions/min_terminated_length": 1779.6,
+      "entropy": 0.09805799871683121,
+      "epoch": 0.3,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12929749488830566,
+      "learning_rate": 1.7100000000000002e-05,
+      "loss": 0.12725646495819093,
+      "num_tokens": 474357.0,
+      "reward": 3.0387999057769775,
+      "reward_std": 5.2611589431762695,
+      "rewards/combined_reward_fn/mean": 3.0387999057769775,
+      "rewards/combined_reward_fn/std": 5.261159038543701,
+      "step": 30,
+      "step_time": 116.51373047439847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3458.0,
+      "completions/max_terminated_length": 2525.4,
+      "completions/mean_length": 2463.15,
+      "completions/mean_terminated_length": 2185.7333984375,
+      "completions/min_length": 1770.4,
+      "completions/min_terminated_length": 1770.4,
+      "entropy": 0.08664824664592743,
+      "epoch": 0.35,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.270455926656723,
+      "learning_rate": 1.66e-05,
+      "loss": 0.15937269926071168,
+      "num_tokens": 545644.0,
+      "reward": 5.9645998001098635,
+      "reward_std": 5.026243317127228,
+      "rewards/combined_reward_fn/mean": 5.9645998001098635,
+      "rewards/combined_reward_fn/std": 5.026243567466736,
+      "step": 35,
+      "step_time": 101.08890176118584
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3698.6,
+      "completions/max_terminated_length": 2838.0,
+      "completions/mean_length": 2698.25,
+      "completions/mean_terminated_length": 2359.450048828125,
+      "completions/min_length": 1920.4,
+      "completions/min_terminated_length": 1920.4,
+      "entropy": 0.0883295014500618,
+      "epoch": 0.4,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14239603281021118,
+      "learning_rate": 1.6100000000000002e-05,
+      "loss": 0.08291782140731811,
+      "num_tokens": 621613.0,
+      "reward": 5.177750015258789,
+      "reward_std": 5.595348834991455,
+      "rewards/combined_reward_fn/mean": 5.177750015258789,
+      "rewards/combined_reward_fn/std": 5.595348834991455,
+      "step": 40,
+      "step_time": 109.44098777900217
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3677.6,
+      "completions/max_terminated_length": 3216.2,
+      "completions/mean_length": 2742.0,
+      "completions/mean_terminated_length": 2395.95,
+      "completions/min_length": 1894.0,
+      "completions/min_terminated_length": 1894.0,
+      "entropy": 0.0947173461318016,
+      "epoch": 0.45,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17538224160671234,
+      "learning_rate": 1.5600000000000003e-05,
+      "loss": 0.11553004980087281,
+      "num_tokens": 698573.0,
+      "reward": 6.005599927902222,
+      "reward_std": 3.041484570503235,
+      "rewards/combined_reward_fn/mean": 6.005599927902222,
+      "rewards/combined_reward_fn/std": 3.0414846658706667,
+      "step": 45,
+      "step_time": 109.76969621782192
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3819.2,
+      "completions/max_terminated_length": 3098.2,
+      "completions/mean_length": 2648.6,
+      "completions/mean_terminated_length": 2391.9833984375,
+      "completions/min_length": 1841.2,
+      "completions/min_terminated_length": 1841.2,
+      "entropy": 0.09659376293420792,
+      "epoch": 0.5,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16343547403812408,
+      "learning_rate": 1.5100000000000001e-05,
+      "loss": 0.027741509675979614,
+      "num_tokens": 773653.0,
+      "reward": 4.999349927902221,
+      "reward_std": 5.17122323513031,
+      "rewards/combined_reward_fn/mean": 4.999349927902221,
+      "rewards/combined_reward_fn/std": 5.1712233543396,
+      "step": 50,
+      "step_time": 112.32290486479178
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3323.4,
+      "completions/max_terminated_length": 2723.8,
+      "completions/mean_length": 2411.05,
+      "completions/mean_terminated_length": 2215.033349609375,
+      "completions/min_length": 1761.4,
+      "completions/min_terminated_length": 1761.4,
+      "entropy": 0.08948126435279846,
+      "epoch": 0.55,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13685713708400726,
+      "learning_rate": 1.46e-05,
+      "loss": 0.13086402416229248,
+      "num_tokens": 844002.0,
+      "reward": 5.210179996490479,
+      "reward_std": 4.280504465103149,
+      "rewards/combined_reward_fn/mean": 5.210179996490479,
+      "rewards/combined_reward_fn/std": 4.280504584312439,
+      "step": 55,
+      "step_time": 97.91599068460054
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3355.6,
+      "completions/max_terminated_length": 2794.4,
+      "completions/mean_length": 2484.55,
+      "completions/mean_terminated_length": 2306.183349609375,
+      "completions/min_length": 1866.0,
+      "completions/min_terminated_length": 1866.0,
+      "entropy": 0.10166355371475219,
+      "epoch": 0.6,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12462152540683746,
+      "learning_rate": 1.41e-05,
+      "loss": 0.09216442108154296,
+      "num_tokens": 915773.0,
+      "reward": 7.240950012207032,
+      "reward_std": 3.289798986911774,
+      "rewards/combined_reward_fn/mean": 7.240950012207032,
+      "rewards/combined_reward_fn/std": 3.289798891544342,
+      "step": 60,
+      "step_time": 96.35008933099452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3067.6,
+      "completions/max_terminated_length": 2370.2,
+      "completions/mean_length": 2421.7,
+      "completions/mean_terminated_length": 2132.4666748046875,
+      "completions/min_length": 1923.0,
+      "completions/min_terminated_length": 1923.0,
+      "entropy": 0.09233423769474029,
+      "epoch": 0.65,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.148515522480011,
+      "learning_rate": 1.3600000000000002e-05,
+      "loss": 0.11075043678283691,
+      "num_tokens": 986315.0,
+      "reward": 4.487199974060059,
+      "reward_std": 6.28532190322876,
+      "rewards/combined_reward_fn/mean": 4.487199974060059,
+      "rewards/combined_reward_fn/std": 6.28532190322876,
+      "step": 65,
+      "step_time": 91.01395496120676
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 3027.0,
+      "completions/max_terminated_length": 2829.6,
+      "completions/mean_length": 2464.3,
+      "completions/mean_terminated_length": 2382.11669921875,
+      "completions/min_length": 2018.4,
+      "completions/min_terminated_length": 2018.4,
+      "entropy": 0.09054866135120392,
+      "epoch": 0.7,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1560310572385788,
+      "learning_rate": 1.3100000000000002e-05,
+      "loss": -0.004711458086967468,
+      "num_tokens": 1057649.0,
+      "reward": 6.76639986038208,
+      "reward_std": 4.127006316184998,
+      "rewards/combined_reward_fn/mean": 6.76639986038208,
+      "rewards/combined_reward_fn/std": 4.1270065069198605,
+      "step": 70,
+      "step_time": 91.32873015338555
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3489.8,
+      "completions/max_terminated_length": 3149.2,
+      "completions/mean_length": 2482.4,
+      "completions/mean_terminated_length": 2306.71669921875,
+      "completions/min_length": 1768.8,
+      "completions/min_terminated_length": 1768.8,
+      "entropy": 0.0954986572265625,
+      "epoch": 0.75,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18958808481693268,
+      "learning_rate": 1.2600000000000001e-05,
+      "loss": 0.05534006357192993,
+      "num_tokens": 1129313.0,
+      "reward": 3.817350149154663,
+      "reward_std": 5.574188280105591,
+      "rewards/combined_reward_fn/mean": 3.817350149154663,
+      "rewards/combined_reward_fn/std": 5.574188184738159,
+      "step": 75,
+      "step_time": 103.64483559140936
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3285.8,
+      "completions/max_terminated_length": 3066.6,
+      "completions/mean_length": 2593.25,
+      "completions/mean_terminated_length": 2402.533349609375,
+      "completions/min_length": 1864.6,
+      "completions/min_terminated_length": 1864.6,
+      "entropy": 0.08767899870872498,
+      "epoch": 0.8,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17502494156360626,
+      "learning_rate": 1.2100000000000001e-05,
+      "loss": 0.08078988194465637,
+      "num_tokens": 1203294.0,
+      "reward": 6.482049989700317,
+      "reward_std": 3.041076159477234,
+      "rewards/combined_reward_fn/mean": 6.482049989700317,
+      "rewards/combined_reward_fn/std": 3.041076183319092,
+      "step": 80,
+      "step_time": 97.3799867952126
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3385.6,
+      "completions/max_terminated_length": 2757.4,
+      "completions/mean_length": 2499.25,
+      "completions/mean_terminated_length": 2302.983349609375,
+      "completions/min_length": 1863.4,
+      "completions/min_terminated_length": 1863.4,
+      "entropy": 0.09770430624485016,
+      "epoch": 0.85,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12575364112854004,
+      "learning_rate": 1.16e-05,
+      "loss": 0.08057029843330384,
+      "num_tokens": 1275307.0,
+      "reward": 5.090700006484985,
+      "reward_std": 5.10671033859253,
+      "rewards/combined_reward_fn/mean": 5.090700006484985,
+      "rewards/combined_reward_fn/std": 5.106710624694824,
+      "step": 85,
+      "step_time": 101.53690337940353
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 3158.4,
+      "completions/max_terminated_length": 2922.6,
+      "completions/mean_length": 2394.3,
+      "completions/mean_terminated_length": 2307.26669921875,
+      "completions/min_length": 1780.2,
+      "completions/min_terminated_length": 1780.2,
+      "entropy": 0.09393323212862015,
+      "epoch": 0.9,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12616610527038574,
+      "learning_rate": 1.1100000000000002e-05,
+      "loss": 0.06031765937805176,
+      "num_tokens": 1345249.0,
+      "reward": 4.1813901424407955,
+      "reward_std": 4.4276350975036625,
+      "rewards/combined_reward_fn/mean": 4.1813901424407955,
+      "rewards/combined_reward_fn/std": 4.427635371685028,
+      "step": 90,
+      "step_time": 100.51337632600917
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 2942.8,
+      "completions/max_terminated_length": 2810.8,
+      "completions/mean_length": 2343.85,
+      "completions/mean_terminated_length": 2270.8,
+      "completions/min_length": 1906.6,
+      "completions/min_terminated_length": 1906.6,
+      "entropy": 0.09947865307331086,
+      "epoch": 0.95,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22282515466213226,
+      "learning_rate": 1.0600000000000002e-05,
+      "loss": -0.007735538482666016,
+      "num_tokens": 1414158.0,
+      "reward": 7.395149993896484,
+      "reward_std": 3.5335000276565554,
+      "rewards/combined_reward_fn/mean": 7.395149993896484,
+      "rewards/combined_reward_fn/std": 3.533500051498413,
+      "step": 95,
+      "step_time": 92.49767726239516
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3397.0,
+      "completions/max_terminated_length": 2508.2,
+      "completions/mean_length": 2472.05,
+      "completions/mean_terminated_length": 2197.616748046875,
+      "completions/min_length": 1896.2,
+      "completions/min_terminated_length": 1896.2,
+      "entropy": 0.1014406830072403,
+      "epoch": 1.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14389891922473907,
+      "learning_rate": 1.0100000000000002e-05,
+      "loss": 0.10350224971771241,
+      "num_tokens": 1485615.0,
+      "reward": 4.888350105285644,
+      "reward_std": 5.132816410064697,
+      "rewards/combined_reward_fn/mean": 4.888350105285644,
+      "rewards/combined_reward_fn/std": 5.1328165292739865,
+      "step": 100,
+      "step_time": 105.17912857598859
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 200,
+  "num_input_tokens_seen": 1485615,
+  "num_train_epochs": 2,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:291194e3ba32f6e08ebdda0990ba6335439bdc4b3136db9a90a3cca222fe0461
+size 6712

checkpoint-150/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-150/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-150/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:837276d69e214a2b20d569b84f4639d3a409f3cb3130239b0a1efd9bb07f776b
+size 12859984

checkpoint-150/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint-150/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eff56ffb00be4555df9f4e033c01e576df91a0697f59a275a4b1b967d80de6a8
+size 25784378

checkpoint-150/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cb0e8796aee0ce473bb8d8c7d1bfbce3591fd7d4b7ef31a37cbfab8375d0bf4
+size 14244

checkpoint-150/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8525ddc478a14472e3ce372156a9476206f0f61bfa815893d66d05b3fa9e3a1d
+size 1064

checkpoint-150/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoint-150/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-150/trainer_state.json

@@ -0,0 +1,844 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.5,
+  "eval_steps": 500,
+  "global_step": 150,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.35,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 2360.0,
+      "completions/mean_length": 2797.55,
+      "completions/mean_terminated_length": 2071.1333984375,
+      "completions/min_length": 1845.0,
+      "completions/min_terminated_length": 1845.0,
+      "entropy": 0.09876374900341034,
+      "epoch": 0.05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.164414182305336,
+      "learning_rate": 1.9600000000000002e-05,
+      "loss": 0.24185845851898194,
+      "num_tokens": 78039.0,
+      "reward": 3.5581498622894285,
+      "reward_std": 5.68612232208252,
+      "rewards/combined_reward_fn/mean": 3.5581498622894285,
+      "rewards/combined_reward_fn/std": 5.686122417449951,
+      "step": 5,
+      "step_time": 108.90133817999158
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3651.2,
+      "completions/max_terminated_length": 2558.4,
+      "completions/mean_length": 2587.95,
+      "completions/mean_terminated_length": 2181.400048828125,
+      "completions/min_length": 1912.2,
+      "completions/min_terminated_length": 1912.2,
+      "entropy": 0.090367890894413,
+      "epoch": 0.1,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16067856550216675,
+      "learning_rate": 1.91e-05,
+      "loss": 0.12473864555358886,
+      "num_tokens": 151926.0,
+      "reward": 5.4179000616073605,
+      "reward_std": 5.021739101409912,
+      "rewards/combined_reward_fn/mean": 5.4179000616073605,
+      "rewards/combined_reward_fn/std": 5.021739077568054,
+      "step": 10,
+      "step_time": 101.38653199401452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.55,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 1866.2,
+      "completions/mean_length": 3242.2,
+      "completions/mean_terminated_length": 1720.3333740234375,
+      "completions/min_length": 2368.6,
+      "completions/min_terminated_length": 1549.4,
+      "entropy": 0.10808060467243194,
+      "epoch": 0.15,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11999707669019699,
+      "learning_rate": 1.86e-05,
+      "loss": 0.1977977991104126,
+      "num_tokens": 238722.0,
+      "reward": 2.5534999847412108,
+      "reward_std": 5.597218799591064,
+      "rewards/combined_reward_fn/mean": 2.5534999847412108,
+      "rewards/combined_reward_fn/std": 5.5972188949584964,
+      "step": 15,
+      "step_time": 123.89894773198758
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3873.0,
+      "completions/max_terminated_length": 2985.6,
+      "completions/mean_length": 2729.5,
+      "completions/mean_terminated_length": 2360.933349609375,
+      "completions/min_length": 1923.0,
+      "completions/min_terminated_length": 1923.0,
+      "entropy": 0.09633768200874329,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1394500881433487,
+      "learning_rate": 1.8100000000000003e-05,
+      "loss": 0.07866127490997314,
+      "num_tokens": 315336.0,
+      "reward": 4.6565999507904055,
+      "reward_std": 4.923504161834717,
+      "rewards/combined_reward_fn/mean": 4.6565999507904055,
+      "rewards/combined_reward_fn/std": 4.923504161834717,
+      "step": 20,
+      "step_time": 111.82979753639083
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 3064.6,
+      "completions/mean_length": 3032.15,
+      "completions/mean_terminated_length": 2677.5333984375,
+      "completions/min_length": 2128.6,
+      "completions/min_terminated_length": 2128.6,
+      "entropy": 0.09437247812747955,
+      "epoch": 0.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12652961909770966,
+      "learning_rate": 1.76e-05,
+      "loss": 0.15245428085327148,
+      "num_tokens": 397983.0,
+      "reward": 4.033899927139283,
+      "reward_std": 5.289630603790283,
+      "rewards/combined_reward_fn/mean": 4.033899927139283,
+      "rewards/combined_reward_fn/std": 5.289630889892578,
+      "step": 25,
+      "step_time": 118.41992252940545
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 3939.8,
+      "completions/max_terminated_length": 2742.6,
+      "completions/mean_length": 2717.1,
+      "completions/mean_terminated_length": 2232.1333984375,
+      "completions/min_length": 1779.6,
+      "completions/min_terminated_length": 1779.6,
+      "entropy": 0.09805799871683121,
+      "epoch": 0.3,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12929749488830566,
+      "learning_rate": 1.7100000000000002e-05,
+      "loss": 0.12725646495819093,
+      "num_tokens": 474357.0,
+      "reward": 3.0387999057769775,
+      "reward_std": 5.2611589431762695,
+      "rewards/combined_reward_fn/mean": 3.0387999057769775,
+      "rewards/combined_reward_fn/std": 5.261159038543701,
+      "step": 30,
+      "step_time": 116.51373047439847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3458.0,
+      "completions/max_terminated_length": 2525.4,
+      "completions/mean_length": 2463.15,
+      "completions/mean_terminated_length": 2185.7333984375,
+      "completions/min_length": 1770.4,
+      "completions/min_terminated_length": 1770.4,
+      "entropy": 0.08664824664592743,
+      "epoch": 0.35,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.270455926656723,
+      "learning_rate": 1.66e-05,
+      "loss": 0.15937269926071168,
+      "num_tokens": 545644.0,
+      "reward": 5.9645998001098635,
+      "reward_std": 5.026243317127228,
+      "rewards/combined_reward_fn/mean": 5.9645998001098635,
+      "rewards/combined_reward_fn/std": 5.026243567466736,
+      "step": 35,
+      "step_time": 101.08890176118584
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3698.6,
+      "completions/max_terminated_length": 2838.0,
+      "completions/mean_length": 2698.25,
+      "completions/mean_terminated_length": 2359.450048828125,
+      "completions/min_length": 1920.4,
+      "completions/min_terminated_length": 1920.4,
+      "entropy": 0.0883295014500618,
+      "epoch": 0.4,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14239603281021118,
+      "learning_rate": 1.6100000000000002e-05,
+      "loss": 0.08291782140731811,
+      "num_tokens": 621613.0,
+      "reward": 5.177750015258789,
+      "reward_std": 5.595348834991455,
+      "rewards/combined_reward_fn/mean": 5.177750015258789,
+      "rewards/combined_reward_fn/std": 5.595348834991455,
+      "step": 40,
+      "step_time": 109.44098777900217
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3677.6,
+      "completions/max_terminated_length": 3216.2,
+      "completions/mean_length": 2742.0,
+      "completions/mean_terminated_length": 2395.95,
+      "completions/min_length": 1894.0,
+      "completions/min_terminated_length": 1894.0,
+      "entropy": 0.0947173461318016,
+      "epoch": 0.45,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17538224160671234,
+      "learning_rate": 1.5600000000000003e-05,
+      "loss": 0.11553004980087281,
+      "num_tokens": 698573.0,
+      "reward": 6.005599927902222,
+      "reward_std": 3.041484570503235,
+      "rewards/combined_reward_fn/mean": 6.005599927902222,
+      "rewards/combined_reward_fn/std": 3.0414846658706667,
+      "step": 45,
+      "step_time": 109.76969621782192
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3819.2,
+      "completions/max_terminated_length": 3098.2,
+      "completions/mean_length": 2648.6,
+      "completions/mean_terminated_length": 2391.9833984375,
+      "completions/min_length": 1841.2,
+      "completions/min_terminated_length": 1841.2,
+      "entropy": 0.09659376293420792,
+      "epoch": 0.5,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16343547403812408,
+      "learning_rate": 1.5100000000000001e-05,
+      "loss": 0.027741509675979614,
+      "num_tokens": 773653.0,
+      "reward": 4.999349927902221,
+      "reward_std": 5.17122323513031,
+      "rewards/combined_reward_fn/mean": 4.999349927902221,
+      "rewards/combined_reward_fn/std": 5.1712233543396,
+      "step": 50,
+      "step_time": 112.32290486479178
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3323.4,
+      "completions/max_terminated_length": 2723.8,
+      "completions/mean_length": 2411.05,
+      "completions/mean_terminated_length": 2215.033349609375,
+      "completions/min_length": 1761.4,
+      "completions/min_terminated_length": 1761.4,
+      "entropy": 0.08948126435279846,
+      "epoch": 0.55,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13685713708400726,
+      "learning_rate": 1.46e-05,
+      "loss": 0.13086402416229248,
+      "num_tokens": 844002.0,
+      "reward": 5.210179996490479,
+      "reward_std": 4.280504465103149,
+      "rewards/combined_reward_fn/mean": 5.210179996490479,
+      "rewards/combined_reward_fn/std": 4.280504584312439,
+      "step": 55,
+      "step_time": 97.91599068460054
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3355.6,
+      "completions/max_terminated_length": 2794.4,
+      "completions/mean_length": 2484.55,
+      "completions/mean_terminated_length": 2306.183349609375,
+      "completions/min_length": 1866.0,
+      "completions/min_terminated_length": 1866.0,
+      "entropy": 0.10166355371475219,
+      "epoch": 0.6,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12462152540683746,
+      "learning_rate": 1.41e-05,
+      "loss": 0.09216442108154296,
+      "num_tokens": 915773.0,
+      "reward": 7.240950012207032,
+      "reward_std": 3.289798986911774,
+      "rewards/combined_reward_fn/mean": 7.240950012207032,
+      "rewards/combined_reward_fn/std": 3.289798891544342,
+      "step": 60,
+      "step_time": 96.35008933099452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3067.6,
+      "completions/max_terminated_length": 2370.2,
+      "completions/mean_length": 2421.7,
+      "completions/mean_terminated_length": 2132.4666748046875,
+      "completions/min_length": 1923.0,
+      "completions/min_terminated_length": 1923.0,
+      "entropy": 0.09233423769474029,
+      "epoch": 0.65,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.148515522480011,
+      "learning_rate": 1.3600000000000002e-05,
+      "loss": 0.11075043678283691,
+      "num_tokens": 986315.0,
+      "reward": 4.487199974060059,
+      "reward_std": 6.28532190322876,
+      "rewards/combined_reward_fn/mean": 4.487199974060059,
+      "rewards/combined_reward_fn/std": 6.28532190322876,
+      "step": 65,
+      "step_time": 91.01395496120676
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 3027.0,
+      "completions/max_terminated_length": 2829.6,
+      "completions/mean_length": 2464.3,
+      "completions/mean_terminated_length": 2382.11669921875,
+      "completions/min_length": 2018.4,
+      "completions/min_terminated_length": 2018.4,
+      "entropy": 0.09054866135120392,
+      "epoch": 0.7,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1560310572385788,
+      "learning_rate": 1.3100000000000002e-05,
+      "loss": -0.004711458086967468,
+      "num_tokens": 1057649.0,
+      "reward": 6.76639986038208,
+      "reward_std": 4.127006316184998,
+      "rewards/combined_reward_fn/mean": 6.76639986038208,
+      "rewards/combined_reward_fn/std": 4.1270065069198605,
+      "step": 70,
+      "step_time": 91.32873015338555
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3489.8,
+      "completions/max_terminated_length": 3149.2,
+      "completions/mean_length": 2482.4,
+      "completions/mean_terminated_length": 2306.71669921875,
+      "completions/min_length": 1768.8,
+      "completions/min_terminated_length": 1768.8,
+      "entropy": 0.0954986572265625,
+      "epoch": 0.75,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18958808481693268,
+      "learning_rate": 1.2600000000000001e-05,
+      "loss": 0.05534006357192993,
+      "num_tokens": 1129313.0,
+      "reward": 3.817350149154663,
+      "reward_std": 5.574188280105591,
+      "rewards/combined_reward_fn/mean": 3.817350149154663,
+      "rewards/combined_reward_fn/std": 5.574188184738159,
+      "step": 75,
+      "step_time": 103.64483559140936
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3285.8,
+      "completions/max_terminated_length": 3066.6,
+      "completions/mean_length": 2593.25,
+      "completions/mean_terminated_length": 2402.533349609375,
+      "completions/min_length": 1864.6,
+      "completions/min_terminated_length": 1864.6,
+      "entropy": 0.08767899870872498,
+      "epoch": 0.8,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17502494156360626,
+      "learning_rate": 1.2100000000000001e-05,
+      "loss": 0.08078988194465637,
+      "num_tokens": 1203294.0,
+      "reward": 6.482049989700317,
+      "reward_std": 3.041076159477234,
+      "rewards/combined_reward_fn/mean": 6.482049989700317,
+      "rewards/combined_reward_fn/std": 3.041076183319092,
+      "step": 80,
+      "step_time": 97.3799867952126
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3385.6,
+      "completions/max_terminated_length": 2757.4,
+      "completions/mean_length": 2499.25,
+      "completions/mean_terminated_length": 2302.983349609375,
+      "completions/min_length": 1863.4,
+      "completions/min_terminated_length": 1863.4,
+      "entropy": 0.09770430624485016,
+      "epoch": 0.85,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12575364112854004,
+      "learning_rate": 1.16e-05,
+      "loss": 0.08057029843330384,
+      "num_tokens": 1275307.0,
+      "reward": 5.090700006484985,
+      "reward_std": 5.10671033859253,
+      "rewards/combined_reward_fn/mean": 5.090700006484985,
+      "rewards/combined_reward_fn/std": 5.106710624694824,
+      "step": 85,
+      "step_time": 101.53690337940353
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 3158.4,
+      "completions/max_terminated_length": 2922.6,
+      "completions/mean_length": 2394.3,
+      "completions/mean_terminated_length": 2307.26669921875,
+      "completions/min_length": 1780.2,
+      "completions/min_terminated_length": 1780.2,
+      "entropy": 0.09393323212862015,
+      "epoch": 0.9,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12616610527038574,
+      "learning_rate": 1.1100000000000002e-05,
+      "loss": 0.06031765937805176,
+      "num_tokens": 1345249.0,
+      "reward": 4.1813901424407955,
+      "reward_std": 4.4276350975036625,
+      "rewards/combined_reward_fn/mean": 4.1813901424407955,
+      "rewards/combined_reward_fn/std": 4.427635371685028,
+      "step": 90,
+      "step_time": 100.51337632600917
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 2942.8,
+      "completions/max_terminated_length": 2810.8,
+      "completions/mean_length": 2343.85,
+      "completions/mean_terminated_length": 2270.8,
+      "completions/min_length": 1906.6,
+      "completions/min_terminated_length": 1906.6,
+      "entropy": 0.09947865307331086,
+      "epoch": 0.95,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22282515466213226,
+      "learning_rate": 1.0600000000000002e-05,
+      "loss": -0.007735538482666016,
+      "num_tokens": 1414158.0,
+      "reward": 7.395149993896484,
+      "reward_std": 3.5335000276565554,
+      "rewards/combined_reward_fn/mean": 7.395149993896484,
+      "rewards/combined_reward_fn/std": 3.533500051498413,
+      "step": 95,
+      "step_time": 92.49767726239516
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3397.0,
+      "completions/max_terminated_length": 2508.2,
+      "completions/mean_length": 2472.05,
+      "completions/mean_terminated_length": 2197.616748046875,
+      "completions/min_length": 1896.2,
+      "completions/min_terminated_length": 1896.2,
+      "entropy": 0.1014406830072403,
+      "epoch": 1.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14389891922473907,
+      "learning_rate": 1.0100000000000002e-05,
+      "loss": 0.10350224971771241,
+      "num_tokens": 1485615.0,
+      "reward": 4.888350105285644,
+      "reward_std": 5.132816410064697,
+      "rewards/combined_reward_fn/mean": 4.888350105285644,
+      "rewards/combined_reward_fn/std": 5.1328165292739865,
+      "step": 100,
+      "step_time": 105.17912857598859
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3233.4,
+      "completions/max_terminated_length": 2584.4,
+      "completions/mean_length": 2312.45,
+      "completions/mean_terminated_length": 2121.500048828125,
+      "completions/min_length": 1802.6,
+      "completions/min_terminated_length": 1802.6,
+      "entropy": 0.09058995842933655,
+      "epoch": 1.05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13503997027873993,
+      "learning_rate": 9.600000000000001e-06,
+      "loss": 0.06539378762245178,
+      "num_tokens": 1553912.0,
+      "reward": 5.310800075531006,
+      "reward_std": 5.286368298530578,
+      "rewards/combined_reward_fn/mean": 5.310800075531006,
+      "rewards/combined_reward_fn/std": 5.2863685131073,
+      "step": 105,
+      "step_time": 101.96216435760725
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3560.0,
+      "completions/max_terminated_length": 2975.6,
+      "completions/mean_length": 2525.1,
+      "completions/mean_terminated_length": 2350.86669921875,
+      "completions/min_length": 1907.2,
+      "completions/min_terminated_length": 1907.2,
+      "entropy": 0.09794304817914963,
+      "epoch": 1.1,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11566527932882309,
+      "learning_rate": 9.100000000000001e-06,
+      "loss": 0.10097475051879883,
+      "num_tokens": 1626398.0,
+      "reward": 5.373299884796142,
+      "reward_std": 3.8057665109634398,
+      "rewards/combined_reward_fn/mean": 5.373299884796142,
+      "rewards/combined_reward_fn/std": 3.8057666063308715,
+      "step": 110,
+      "step_time": 110.98522113121581
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3259.2,
+      "completions/max_terminated_length": 2561.8,
+      "completions/mean_length": 2418.6,
+      "completions/mean_terminated_length": 2230.16669921875,
+      "completions/min_length": 1993.6,
+      "completions/min_terminated_length": 1993.6,
+      "entropy": 0.08254525065422058,
+      "epoch": 1.15,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17645150423049927,
+      "learning_rate": 8.6e-06,
+      "loss": 0.010759654641151428,
+      "num_tokens": 1696810.0,
+      "reward": 3.8589000701904297,
+      "reward_std": 4.830600321292877,
+      "rewards/combined_reward_fn/mean": 3.8589000701904297,
+      "rewards/combined_reward_fn/std": 4.8306004226207735,
+      "step": 115,
+      "step_time": 101.04576327840331
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3757.4,
+      "completions/max_terminated_length": 2404.2,
+      "completions/mean_length": 2464.25,
+      "completions/mean_terminated_length": 2059.8833984375,
+      "completions/min_length": 1768.4,
+      "completions/min_terminated_length": 1768.4,
+      "entropy": 0.08660776913166046,
+      "epoch": 1.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15443815290927887,
+      "learning_rate": 8.1e-06,
+      "loss": 0.22375082969665527,
+      "num_tokens": 1768159.0,
+      "reward": 5.913300037384033,
+      "reward_std": 4.695694637298584,
+      "rewards/combined_reward_fn/mean": 5.913300037384033,
+      "rewards/combined_reward_fn/std": 4.6956947326660154,
+      "step": 120,
+      "step_time": 115.54127626260743
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 2899.8,
+      "completions/max_terminated_length": 2640.4,
+      "completions/mean_length": 2322.3,
+      "completions/mean_terminated_length": 2230.45,
+      "completions/min_length": 1914.6,
+      "completions/min_terminated_length": 1914.6,
+      "entropy": 0.08824309259653092,
+      "epoch": 1.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16980406641960144,
+      "learning_rate": 7.600000000000001e-06,
+      "loss": 0.024670909345149993,
+      "num_tokens": 1836645.0,
+      "reward": 5.819709968566895,
+      "reward_std": 4.154629921913147,
+      "rewards/combined_reward_fn/mean": 5.819709968566895,
+      "rewards/combined_reward_fn/std": 4.154630184173584,
+      "step": 125,
+      "step_time": 90.41040673980024
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3268.4,
+      "completions/max_terminated_length": 2870.2,
+      "completions/mean_length": 2633.25,
+      "completions/mean_terminated_length": 2480.300048828125,
+      "completions/min_length": 2124.8,
+      "completions/min_terminated_length": 2124.8,
+      "entropy": 0.09058777540922165,
+      "epoch": 1.3,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11714135110378265,
+      "learning_rate": 7.100000000000001e-06,
+      "loss": 0.06325926780700683,
+      "num_tokens": 1911426.0,
+      "reward": 6.010169887542725,
+      "reward_std": 4.046402001380921,
+      "rewards/combined_reward_fn/mean": 6.010169887542725,
+      "rewards/combined_reward_fn/std": 4.046402168273926,
+      "step": 130,
+      "step_time": 101.50792271901155
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2831.6,
+      "completions/max_terminated_length": 2831.6,
+      "completions/mean_length": 2293.7,
+      "completions/mean_terminated_length": 2293.7,
+      "completions/min_length": 1818.2,
+      "completions/min_terminated_length": 1818.2,
+      "entropy": 0.10288319885730743,
+      "epoch": 1.35,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1425861418247223,
+      "learning_rate": 6.600000000000001e-06,
+      "loss": -0.03755061030387878,
+      "num_tokens": 1979416.0,
+      "reward": 6.556550073623657,
+      "reward_std": 3.4997296571731566,
+      "rewards/combined_reward_fn/mean": 6.556550073623657,
+      "rewards/combined_reward_fn/std": 3.4997297048568727,
+      "step": 135,
+      "step_time": 88.84195423700149
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3319.4,
+      "completions/max_terminated_length": 2498.6,
+      "completions/mean_length": 2664.25,
+      "completions/mean_terminated_length": 2199.083349609375,
+      "completions/min_length": 1739.6,
+      "completions/min_terminated_length": 1739.6,
+      "entropy": 0.08756150305271149,
+      "epoch": 1.4,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12729056179523468,
+      "learning_rate": 6.1e-06,
+      "loss": 0.05446885228157043,
+      "num_tokens": 2054829.0,
+      "reward": 3.3084999561309814,
+      "reward_std": 6.0273261070251465,
+      "rewards/combined_reward_fn/mean": 3.3084999561309814,
+      "rewards/combined_reward_fn/std": 6.0273261070251465,
+      "step": 140,
+      "step_time": 107.17153390740278
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3832.4,
+      "completions/max_terminated_length": 2908.4,
+      "completions/mean_length": 2760.1,
+      "completions/mean_terminated_length": 2483.6833740234374,
+      "completions/min_length": 2114.4,
+      "completions/min_terminated_length": 2114.4,
+      "entropy": 0.09180340468883515,
+      "epoch": 1.45,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12470444291830063,
+      "learning_rate": 5.600000000000001e-06,
+      "loss": 0.12354047298431396,
+      "num_tokens": 2132031.0,
+      "reward": 4.915040111541748,
+      "reward_std": 5.242267656326294,
+      "rewards/combined_reward_fn/mean": 4.915040111541748,
+      "rewards/combined_reward_fn/std": 5.242267799377442,
+      "step": 145,
+      "step_time": 118.50906647598603
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3527.6,
+      "completions/max_terminated_length": 2709.4,
+      "completions/mean_length": 2601.1,
+      "completions/mean_terminated_length": 2348.533349609375,
+      "completions/min_length": 1946.6,
+      "completions/min_terminated_length": 1946.6,
+      "entropy": 0.08960584700107574,
+      "epoch": 1.5,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16406011581420898,
+      "learning_rate": 5.1e-06,
+      "loss": 0.12920334339141845,
+      "num_tokens": 2206113.0,
+      "reward": 3.5731899976730346,
+      "reward_std": 5.10755033493042,
+      "rewards/combined_reward_fn/mean": 3.5731899976730346,
+      "rewards/combined_reward_fn/std": 5.107550430297851,
+      "step": 150,
+      "step_time": 111.50722688417882
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 200,
+  "num_input_tokens_seen": 2206113,
+  "num_train_epochs": 2,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-150/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:291194e3ba32f6e08ebdda0990ba6335439bdc4b3136db9a90a3cca222fe0461
+size 6712

checkpoint-200/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-200/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-200/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b901e70822382d594fea5f25d6c35255f8f8a8e6672725dd1a5ccc1f03e80ea4
+size 12859984

checkpoint-200/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint-200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77659a390afc239239b31f5f3ed0b7ee2482f5a925b888ec3e1aee02e50e56a8
+size 25784378

checkpoint-200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a40412e54825d9a05f0b9f7b221070e94f0657afc1e290e2b403630e094a442a
+size 14244

checkpoint-200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb148ad6cd6502048dc3eee274a6708e5b4f6b72c0d56ebc272935446c220acd
+size 1064

checkpoint-200/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoint-200/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-200/trainer_state.json

@@ -0,0 +1,1114 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.35,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 2360.0,
+      "completions/mean_length": 2797.55,
+      "completions/mean_terminated_length": 2071.1333984375,
+      "completions/min_length": 1845.0,
+      "completions/min_terminated_length": 1845.0,
+      "entropy": 0.09876374900341034,
+      "epoch": 0.05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.164414182305336,
+      "learning_rate": 1.9600000000000002e-05,
+      "loss": 0.24185845851898194,
+      "num_tokens": 78039.0,
+      "reward": 3.5581498622894285,
+      "reward_std": 5.68612232208252,
+      "rewards/combined_reward_fn/mean": 3.5581498622894285,
+      "rewards/combined_reward_fn/std": 5.686122417449951,
+      "step": 5,
+      "step_time": 108.90133817999158
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3651.2,
+      "completions/max_terminated_length": 2558.4,
+      "completions/mean_length": 2587.95,
+      "completions/mean_terminated_length": 2181.400048828125,
+      "completions/min_length": 1912.2,
+      "completions/min_terminated_length": 1912.2,
+      "entropy": 0.090367890894413,
+      "epoch": 0.1,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16067856550216675,
+      "learning_rate": 1.91e-05,
+      "loss": 0.12473864555358886,
+      "num_tokens": 151926.0,
+      "reward": 5.4179000616073605,
+      "reward_std": 5.021739101409912,
+      "rewards/combined_reward_fn/mean": 5.4179000616073605,
+      "rewards/combined_reward_fn/std": 5.021739077568054,
+      "step": 10,
+      "step_time": 101.38653199401452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.55,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 1866.2,
+      "completions/mean_length": 3242.2,
+      "completions/mean_terminated_length": 1720.3333740234375,
+      "completions/min_length": 2368.6,
+      "completions/min_terminated_length": 1549.4,
+      "entropy": 0.10808060467243194,
+      "epoch": 0.15,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11999707669019699,
+      "learning_rate": 1.86e-05,
+      "loss": 0.1977977991104126,
+      "num_tokens": 238722.0,
+      "reward": 2.5534999847412108,
+      "reward_std": 5.597218799591064,
+      "rewards/combined_reward_fn/mean": 2.5534999847412108,
+      "rewards/combined_reward_fn/std": 5.5972188949584964,
+      "step": 15,
+      "step_time": 123.89894773198758
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3873.0,
+      "completions/max_terminated_length": 2985.6,
+      "completions/mean_length": 2729.5,
+      "completions/mean_terminated_length": 2360.933349609375,
+      "completions/min_length": 1923.0,
+      "completions/min_terminated_length": 1923.0,
+      "entropy": 0.09633768200874329,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1394500881433487,
+      "learning_rate": 1.8100000000000003e-05,
+      "loss": 0.07866127490997314,
+      "num_tokens": 315336.0,
+      "reward": 4.6565999507904055,
+      "reward_std": 4.923504161834717,
+      "rewards/combined_reward_fn/mean": 4.6565999507904055,
+      "rewards/combined_reward_fn/std": 4.923504161834717,
+      "step": 20,
+      "step_time": 111.82979753639083
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 4096.0,
+      "completions/max_terminated_length": 3064.6,
+      "completions/mean_length": 3032.15,
+      "completions/mean_terminated_length": 2677.5333984375,
+      "completions/min_length": 2128.6,
+      "completions/min_terminated_length": 2128.6,
+      "entropy": 0.09437247812747955,
+      "epoch": 0.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12652961909770966,
+      "learning_rate": 1.76e-05,
+      "loss": 0.15245428085327148,
+      "num_tokens": 397983.0,
+      "reward": 4.033899927139283,
+      "reward_std": 5.289630603790283,
+      "rewards/combined_reward_fn/mean": 4.033899927139283,
+      "rewards/combined_reward_fn/std": 5.289630889892578,
+      "step": 25,
+      "step_time": 118.41992252940545
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 3939.8,
+      "completions/max_terminated_length": 2742.6,
+      "completions/mean_length": 2717.1,
+      "completions/mean_terminated_length": 2232.1333984375,
+      "completions/min_length": 1779.6,
+      "completions/min_terminated_length": 1779.6,
+      "entropy": 0.09805799871683121,
+      "epoch": 0.3,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12929749488830566,
+      "learning_rate": 1.7100000000000002e-05,
+      "loss": 0.12725646495819093,
+      "num_tokens": 474357.0,
+      "reward": 3.0387999057769775,
+      "reward_std": 5.2611589431762695,
+      "rewards/combined_reward_fn/mean": 3.0387999057769775,
+      "rewards/combined_reward_fn/std": 5.261159038543701,
+      "step": 30,
+      "step_time": 116.51373047439847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3458.0,
+      "completions/max_terminated_length": 2525.4,
+      "completions/mean_length": 2463.15,
+      "completions/mean_terminated_length": 2185.7333984375,
+      "completions/min_length": 1770.4,
+      "completions/min_terminated_length": 1770.4,
+      "entropy": 0.08664824664592743,
+      "epoch": 0.35,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.270455926656723,
+      "learning_rate": 1.66e-05,
+      "loss": 0.15937269926071168,
+      "num_tokens": 545644.0,
+      "reward": 5.9645998001098635,
+      "reward_std": 5.026243317127228,
+      "rewards/combined_reward_fn/mean": 5.9645998001098635,
+      "rewards/combined_reward_fn/std": 5.026243567466736,
+      "step": 35,
+      "step_time": 101.08890176118584
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3698.6,
+      "completions/max_terminated_length": 2838.0,
+      "completions/mean_length": 2698.25,
+      "completions/mean_terminated_length": 2359.450048828125,
+      "completions/min_length": 1920.4,
+      "completions/min_terminated_length": 1920.4,
+      "entropy": 0.0883295014500618,
+      "epoch": 0.4,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14239603281021118,
+      "learning_rate": 1.6100000000000002e-05,
+      "loss": 0.08291782140731811,
+      "num_tokens": 621613.0,
+      "reward": 5.177750015258789,
+      "reward_std": 5.595348834991455,
+      "rewards/combined_reward_fn/mean": 5.177750015258789,
+      "rewards/combined_reward_fn/std": 5.595348834991455,
+      "step": 40,
+      "step_time": 109.44098777900217
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3677.6,
+      "completions/max_terminated_length": 3216.2,
+      "completions/mean_length": 2742.0,
+      "completions/mean_terminated_length": 2395.95,
+      "completions/min_length": 1894.0,
+      "completions/min_terminated_length": 1894.0,
+      "entropy": 0.0947173461318016,
+      "epoch": 0.45,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17538224160671234,
+      "learning_rate": 1.5600000000000003e-05,
+      "loss": 0.11553004980087281,
+      "num_tokens": 698573.0,
+      "reward": 6.005599927902222,
+      "reward_std": 3.041484570503235,
+      "rewards/combined_reward_fn/mean": 6.005599927902222,
+      "rewards/combined_reward_fn/std": 3.0414846658706667,
+      "step": 45,
+      "step_time": 109.76969621782192
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3819.2,
+      "completions/max_terminated_length": 3098.2,
+      "completions/mean_length": 2648.6,
+      "completions/mean_terminated_length": 2391.9833984375,
+      "completions/min_length": 1841.2,
+      "completions/min_terminated_length": 1841.2,
+      "entropy": 0.09659376293420792,
+      "epoch": 0.5,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16343547403812408,
+      "learning_rate": 1.5100000000000001e-05,
+      "loss": 0.027741509675979614,
+      "num_tokens": 773653.0,
+      "reward": 4.999349927902221,
+      "reward_std": 5.17122323513031,
+      "rewards/combined_reward_fn/mean": 4.999349927902221,
+      "rewards/combined_reward_fn/std": 5.1712233543396,
+      "step": 50,
+      "step_time": 112.32290486479178
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3323.4,
+      "completions/max_terminated_length": 2723.8,
+      "completions/mean_length": 2411.05,
+      "completions/mean_terminated_length": 2215.033349609375,
+      "completions/min_length": 1761.4,
+      "completions/min_terminated_length": 1761.4,
+      "entropy": 0.08948126435279846,
+      "epoch": 0.55,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13685713708400726,
+      "learning_rate": 1.46e-05,
+      "loss": 0.13086402416229248,
+      "num_tokens": 844002.0,
+      "reward": 5.210179996490479,
+      "reward_std": 4.280504465103149,
+      "rewards/combined_reward_fn/mean": 5.210179996490479,
+      "rewards/combined_reward_fn/std": 4.280504584312439,
+      "step": 55,
+      "step_time": 97.91599068460054
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3355.6,
+      "completions/max_terminated_length": 2794.4,
+      "completions/mean_length": 2484.55,
+      "completions/mean_terminated_length": 2306.183349609375,
+      "completions/min_length": 1866.0,
+      "completions/min_terminated_length": 1866.0,
+      "entropy": 0.10166355371475219,
+      "epoch": 0.6,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12462152540683746,
+      "learning_rate": 1.41e-05,
+      "loss": 0.09216442108154296,
+      "num_tokens": 915773.0,
+      "reward": 7.240950012207032,
+      "reward_std": 3.289798986911774,
+      "rewards/combined_reward_fn/mean": 7.240950012207032,
+      "rewards/combined_reward_fn/std": 3.289798891544342,
+      "step": 60,
+      "step_time": 96.35008933099452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3067.6,
+      "completions/max_terminated_length": 2370.2,
+      "completions/mean_length": 2421.7,
+      "completions/mean_terminated_length": 2132.4666748046875,
+      "completions/min_length": 1923.0,
+      "completions/min_terminated_length": 1923.0,
+      "entropy": 0.09233423769474029,
+      "epoch": 0.65,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.148515522480011,
+      "learning_rate": 1.3600000000000002e-05,
+      "loss": 0.11075043678283691,
+      "num_tokens": 986315.0,
+      "reward": 4.487199974060059,
+      "reward_std": 6.28532190322876,
+      "rewards/combined_reward_fn/mean": 4.487199974060059,
+      "rewards/combined_reward_fn/std": 6.28532190322876,
+      "step": 65,
+      "step_time": 91.01395496120676
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 3027.0,
+      "completions/max_terminated_length": 2829.6,
+      "completions/mean_length": 2464.3,
+      "completions/mean_terminated_length": 2382.11669921875,
+      "completions/min_length": 2018.4,
+      "completions/min_terminated_length": 2018.4,
+      "entropy": 0.09054866135120392,
+      "epoch": 0.7,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1560310572385788,
+      "learning_rate": 1.3100000000000002e-05,
+      "loss": -0.004711458086967468,
+      "num_tokens": 1057649.0,
+      "reward": 6.76639986038208,
+      "reward_std": 4.127006316184998,
+      "rewards/combined_reward_fn/mean": 6.76639986038208,
+      "rewards/combined_reward_fn/std": 4.1270065069198605,
+      "step": 70,
+      "step_time": 91.32873015338555
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3489.8,
+      "completions/max_terminated_length": 3149.2,
+      "completions/mean_length": 2482.4,
+      "completions/mean_terminated_length": 2306.71669921875,
+      "completions/min_length": 1768.8,
+      "completions/min_terminated_length": 1768.8,
+      "entropy": 0.0954986572265625,
+      "epoch": 0.75,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18958808481693268,
+      "learning_rate": 1.2600000000000001e-05,
+      "loss": 0.05534006357192993,
+      "num_tokens": 1129313.0,
+      "reward": 3.817350149154663,
+      "reward_std": 5.574188280105591,
+      "rewards/combined_reward_fn/mean": 3.817350149154663,
+      "rewards/combined_reward_fn/std": 5.574188184738159,
+      "step": 75,
+      "step_time": 103.64483559140936
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3285.8,
+      "completions/max_terminated_length": 3066.6,
+      "completions/mean_length": 2593.25,
+      "completions/mean_terminated_length": 2402.533349609375,
+      "completions/min_length": 1864.6,
+      "completions/min_terminated_length": 1864.6,
+      "entropy": 0.08767899870872498,
+      "epoch": 0.8,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17502494156360626,
+      "learning_rate": 1.2100000000000001e-05,
+      "loss": 0.08078988194465637,
+      "num_tokens": 1203294.0,
+      "reward": 6.482049989700317,
+      "reward_std": 3.041076159477234,
+      "rewards/combined_reward_fn/mean": 6.482049989700317,
+      "rewards/combined_reward_fn/std": 3.041076183319092,
+      "step": 80,
+      "step_time": 97.3799867952126
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3385.6,
+      "completions/max_terminated_length": 2757.4,
+      "completions/mean_length": 2499.25,
+      "completions/mean_terminated_length": 2302.983349609375,
+      "completions/min_length": 1863.4,
+      "completions/min_terminated_length": 1863.4,
+      "entropy": 0.09770430624485016,
+      "epoch": 0.85,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12575364112854004,
+      "learning_rate": 1.16e-05,
+      "loss": 0.08057029843330384,
+      "num_tokens": 1275307.0,
+      "reward": 5.090700006484985,
+      "reward_std": 5.10671033859253,
+      "rewards/combined_reward_fn/mean": 5.090700006484985,
+      "rewards/combined_reward_fn/std": 5.106710624694824,
+      "step": 85,
+      "step_time": 101.53690337940353
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 3158.4,
+      "completions/max_terminated_length": 2922.6,
+      "completions/mean_length": 2394.3,
+      "completions/mean_terminated_length": 2307.26669921875,
+      "completions/min_length": 1780.2,
+      "completions/min_terminated_length": 1780.2,
+      "entropy": 0.09393323212862015,
+      "epoch": 0.9,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12616610527038574,
+      "learning_rate": 1.1100000000000002e-05,
+      "loss": 0.06031765937805176,
+      "num_tokens": 1345249.0,
+      "reward": 4.1813901424407955,
+      "reward_std": 4.4276350975036625,
+      "rewards/combined_reward_fn/mean": 4.1813901424407955,
+      "rewards/combined_reward_fn/std": 4.427635371685028,
+      "step": 90,
+      "step_time": 100.51337632600917
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 2942.8,
+      "completions/max_terminated_length": 2810.8,
+      "completions/mean_length": 2343.85,
+      "completions/mean_terminated_length": 2270.8,
+      "completions/min_length": 1906.6,
+      "completions/min_terminated_length": 1906.6,
+      "entropy": 0.09947865307331086,
+      "epoch": 0.95,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22282515466213226,
+      "learning_rate": 1.0600000000000002e-05,
+      "loss": -0.007735538482666016,
+      "num_tokens": 1414158.0,
+      "reward": 7.395149993896484,
+      "reward_std": 3.5335000276565554,
+      "rewards/combined_reward_fn/mean": 7.395149993896484,
+      "rewards/combined_reward_fn/std": 3.533500051498413,
+      "step": 95,
+      "step_time": 92.49767726239516
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3397.0,
+      "completions/max_terminated_length": 2508.2,
+      "completions/mean_length": 2472.05,
+      "completions/mean_terminated_length": 2197.616748046875,
+      "completions/min_length": 1896.2,
+      "completions/min_terminated_length": 1896.2,
+      "entropy": 0.1014406830072403,
+      "epoch": 1.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14389891922473907,
+      "learning_rate": 1.0100000000000002e-05,
+      "loss": 0.10350224971771241,
+      "num_tokens": 1485615.0,
+      "reward": 4.888350105285644,
+      "reward_std": 5.132816410064697,
+      "rewards/combined_reward_fn/mean": 4.888350105285644,
+      "rewards/combined_reward_fn/std": 5.1328165292739865,
+      "step": 100,
+      "step_time": 105.17912857598859
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3233.4,
+      "completions/max_terminated_length": 2584.4,
+      "completions/mean_length": 2312.45,
+      "completions/mean_terminated_length": 2121.500048828125,
+      "completions/min_length": 1802.6,
+      "completions/min_terminated_length": 1802.6,
+      "entropy": 0.09058995842933655,
+      "epoch": 1.05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13503997027873993,
+      "learning_rate": 9.600000000000001e-06,
+      "loss": 0.06539378762245178,
+      "num_tokens": 1553912.0,
+      "reward": 5.310800075531006,
+      "reward_std": 5.286368298530578,
+      "rewards/combined_reward_fn/mean": 5.310800075531006,
+      "rewards/combined_reward_fn/std": 5.2863685131073,
+      "step": 105,
+      "step_time": 101.96216435760725
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3560.0,
+      "completions/max_terminated_length": 2975.6,
+      "completions/mean_length": 2525.1,
+      "completions/mean_terminated_length": 2350.86669921875,
+      "completions/min_length": 1907.2,
+      "completions/min_terminated_length": 1907.2,
+      "entropy": 0.09794304817914963,
+      "epoch": 1.1,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11566527932882309,
+      "learning_rate": 9.100000000000001e-06,
+      "loss": 0.10097475051879883,
+      "num_tokens": 1626398.0,
+      "reward": 5.373299884796142,
+      "reward_std": 3.8057665109634398,
+      "rewards/combined_reward_fn/mean": 5.373299884796142,
+      "rewards/combined_reward_fn/std": 3.8057666063308715,
+      "step": 110,
+      "step_time": 110.98522113121581
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3259.2,
+      "completions/max_terminated_length": 2561.8,
+      "completions/mean_length": 2418.6,
+      "completions/mean_terminated_length": 2230.16669921875,
+      "completions/min_length": 1993.6,
+      "completions/min_terminated_length": 1993.6,
+      "entropy": 0.08254525065422058,
+      "epoch": 1.15,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17645150423049927,
+      "learning_rate": 8.6e-06,
+      "loss": 0.010759654641151428,
+      "num_tokens": 1696810.0,
+      "reward": 3.8589000701904297,
+      "reward_std": 4.830600321292877,
+      "rewards/combined_reward_fn/mean": 3.8589000701904297,
+      "rewards/combined_reward_fn/std": 4.8306004226207735,
+      "step": 115,
+      "step_time": 101.04576327840331
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3757.4,
+      "completions/max_terminated_length": 2404.2,
+      "completions/mean_length": 2464.25,
+      "completions/mean_terminated_length": 2059.8833984375,
+      "completions/min_length": 1768.4,
+      "completions/min_terminated_length": 1768.4,
+      "entropy": 0.08660776913166046,
+      "epoch": 1.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15443815290927887,
+      "learning_rate": 8.1e-06,
+      "loss": 0.22375082969665527,
+      "num_tokens": 1768159.0,
+      "reward": 5.913300037384033,
+      "reward_std": 4.695694637298584,
+      "rewards/combined_reward_fn/mean": 5.913300037384033,
+      "rewards/combined_reward_fn/std": 4.6956947326660154,
+      "step": 120,
+      "step_time": 115.54127626260743
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 2899.8,
+      "completions/max_terminated_length": 2640.4,
+      "completions/mean_length": 2322.3,
+      "completions/mean_terminated_length": 2230.45,
+      "completions/min_length": 1914.6,
+      "completions/min_terminated_length": 1914.6,
+      "entropy": 0.08824309259653092,
+      "epoch": 1.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16980406641960144,
+      "learning_rate": 7.600000000000001e-06,
+      "loss": 0.024670909345149993,
+      "num_tokens": 1836645.0,
+      "reward": 5.819709968566895,
+      "reward_std": 4.154629921913147,
+      "rewards/combined_reward_fn/mean": 5.819709968566895,
+      "rewards/combined_reward_fn/std": 4.154630184173584,
+      "step": 125,
+      "step_time": 90.41040673980024
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3268.4,
+      "completions/max_terminated_length": 2870.2,
+      "completions/mean_length": 2633.25,
+      "completions/mean_terminated_length": 2480.300048828125,
+      "completions/min_length": 2124.8,
+      "completions/min_terminated_length": 2124.8,
+      "entropy": 0.09058777540922165,
+      "epoch": 1.3,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11714135110378265,
+      "learning_rate": 7.100000000000001e-06,
+      "loss": 0.06325926780700683,
+      "num_tokens": 1911426.0,
+      "reward": 6.010169887542725,
+      "reward_std": 4.046402001380921,
+      "rewards/combined_reward_fn/mean": 6.010169887542725,
+      "rewards/combined_reward_fn/std": 4.046402168273926,
+      "step": 130,
+      "step_time": 101.50792271901155
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2831.6,
+      "completions/max_terminated_length": 2831.6,
+      "completions/mean_length": 2293.7,
+      "completions/mean_terminated_length": 2293.7,
+      "completions/min_length": 1818.2,
+      "completions/min_terminated_length": 1818.2,
+      "entropy": 0.10288319885730743,
+      "epoch": 1.35,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1425861418247223,
+      "learning_rate": 6.600000000000001e-06,
+      "loss": -0.03755061030387878,
+      "num_tokens": 1979416.0,
+      "reward": 6.556550073623657,
+      "reward_std": 3.4997296571731566,
+      "rewards/combined_reward_fn/mean": 6.556550073623657,
+      "rewards/combined_reward_fn/std": 3.4997297048568727,
+      "step": 135,
+      "step_time": 88.84195423700149
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3319.4,
+      "completions/max_terminated_length": 2498.6,
+      "completions/mean_length": 2664.25,
+      "completions/mean_terminated_length": 2199.083349609375,
+      "completions/min_length": 1739.6,
+      "completions/min_terminated_length": 1739.6,
+      "entropy": 0.08756150305271149,
+      "epoch": 1.4,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12729056179523468,
+      "learning_rate": 6.1e-06,
+      "loss": 0.05446885228157043,
+      "num_tokens": 2054829.0,
+      "reward": 3.3084999561309814,
+      "reward_std": 6.0273261070251465,
+      "rewards/combined_reward_fn/mean": 3.3084999561309814,
+      "rewards/combined_reward_fn/std": 6.0273261070251465,
+      "step": 140,
+      "step_time": 107.17153390740278
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3832.4,
+      "completions/max_terminated_length": 2908.4,
+      "completions/mean_length": 2760.1,
+      "completions/mean_terminated_length": 2483.6833740234374,
+      "completions/min_length": 2114.4,
+      "completions/min_terminated_length": 2114.4,
+      "entropy": 0.09180340468883515,
+      "epoch": 1.45,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12470444291830063,
+      "learning_rate": 5.600000000000001e-06,
+      "loss": 0.12354047298431396,
+      "num_tokens": 2132031.0,
+      "reward": 4.915040111541748,
+      "reward_std": 5.242267656326294,
+      "rewards/combined_reward_fn/mean": 4.915040111541748,
+      "rewards/combined_reward_fn/std": 5.242267799377442,
+      "step": 145,
+      "step_time": 118.50906647598603
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 3527.6,
+      "completions/max_terminated_length": 2709.4,
+      "completions/mean_length": 2601.1,
+      "completions/mean_terminated_length": 2348.533349609375,
+      "completions/min_length": 1946.6,
+      "completions/min_terminated_length": 1946.6,
+      "entropy": 0.08960584700107574,
+      "epoch": 1.5,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16406011581420898,
+      "learning_rate": 5.1e-06,
+      "loss": 0.12920334339141845,
+      "num_tokens": 2206113.0,
+      "reward": 3.5731899976730346,
+      "reward_std": 5.10755033493042,
+      "rewards/combined_reward_fn/mean": 3.5731899976730346,
+      "rewards/combined_reward_fn/std": 5.107550430297851,
+      "step": 150,
+      "step_time": 111.50722688417882
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3467.0,
+      "completions/max_terminated_length": 2623.2,
+      "completions/mean_length": 2645.45,
+      "completions/mean_terminated_length": 2311.06669921875,
+      "completions/min_length": 2093.2,
+      "completions/min_terminated_length": 2093.2,
+      "entropy": 0.1021485298871994,
+      "epoch": 1.55,
+      "frac_reward_zero_std": 0.2,
+      "grad_norm": 0.0,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 0.0487419068813324,
+      "num_tokens": 2281086.0,
+      "reward": 2.5386499881744387,
+      "reward_std": 4.577268314361572,
+      "rewards/combined_reward_fn/mean": 2.5386499881744387,
+      "rewards/combined_reward_fn/std": 4.577268314361572,
+      "step": 155,
+      "step_time": 113.60341777240392
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 3284.6,
+      "completions/max_terminated_length": 2564.6,
+      "completions/mean_length": 2384.7,
+      "completions/mean_terminated_length": 2168.533349609375,
+      "completions/min_length": 1880.4,
+      "completions/min_terminated_length": 1880.4,
+      "entropy": 0.07901217043399811,
+      "epoch": 1.6,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11951136589050293,
+      "learning_rate": 4.1e-06,
+      "loss": 0.12389217615127564,
+      "num_tokens": 2350824.0,
+      "reward": 6.6975000381469725,
+      "reward_std": 4.296219682693481,
+      "rewards/combined_reward_fn/mean": 6.6975000381469725,
+      "rewards/combined_reward_fn/std": 4.296219825744629,
+      "step": 160,
+      "step_time": 103.55389254660113
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 3210.8,
+      "completions/max_terminated_length": 2835.6,
+      "completions/mean_length": 2370.0,
+      "completions/mean_terminated_length": 2262.1166748046876,
+      "completions/min_length": 1711.2,
+      "completions/min_terminated_length": 1711.2,
+      "entropy": 0.11034855544567108,
+      "epoch": 1.65,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15524832904338837,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 0.023141218721866606,
+      "num_tokens": 2420284.0,
+      "reward": 5.670350027084351,
+      "reward_std": 4.717032456398011,
+      "rewards/combined_reward_fn/mean": 5.670350027084351,
+      "rewards/combined_reward_fn/std": 4.717032456398011,
+      "step": 165,
+      "step_time": 98.23545178200584
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 2942.6,
+      "completions/max_terminated_length": 2779.4,
+      "completions/mean_length": 2444.8,
+      "completions/mean_terminated_length": 2379.5,
+      "completions/min_length": 1915.2,
+      "completions/min_terminated_length": 1915.2,
+      "entropy": 0.09058590531349182,
+      "epoch": 1.7,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17255985736846924,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": -0.03838949203491211,
+      "num_tokens": 2491268.0,
+      "reward": 8.130179595947265,
+      "reward_std": 2.6957954287528993,
+      "rewards/combined_reward_fn/mean": 8.130179595947265,
+      "rewards/combined_reward_fn/std": 2.69579541683197,
+      "step": 170,
+      "step_time": 89.64412425479387
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 4016.0,
+      "completions/max_terminated_length": 3245.4,
+      "completions/mean_length": 2821.8,
+      "completions/mean_terminated_length": 2518.5167236328125,
+      "completions/min_length": 1979.8,
+      "completions/min_terminated_length": 1979.8,
+      "entropy": 0.10169393718242645,
+      "epoch": 1.75,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14736266434192657,
+      "learning_rate": 2.6e-06,
+      "loss": 0.12008615732192993,
+      "num_tokens": 2569752.0,
+      "reward": 4.244350171089172,
+      "reward_std": 5.004626512527466,
+      "rewards/combined_reward_fn/mean": 4.244350171089172,
+      "rewards/combined_reward_fn/std": 5.004626512527466,
+      "step": 175,
+      "step_time": 124.0094484991976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2440.6,
+      "completions/max_terminated_length": 2440.6,
+      "completions/mean_length": 2072.25,
+      "completions/mean_terminated_length": 2072.25,
+      "completions/min_length": 1723.8,
+      "completions/min_terminated_length": 1723.8,
+      "entropy": 0.0946453794836998,
+      "epoch": 1.8,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15898209810256958,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": -0.030627071857452393,
+      "num_tokens": 2633297.0,
+      "reward": 6.971000003814697,
+      "reward_std": 3.39311888217926,
+      "rewards/combined_reward_fn/mean": 6.971000003814697,
+      "rewards/combined_reward_fn/std": 3.3931188583374023,
+      "step": 180,
+      "step_time": 75.84038563302019
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 3888.4,
+      "completions/max_terminated_length": 2815.0,
+      "completions/mean_length": 2714.55,
+      "completions/mean_terminated_length": 2339.3000732421874,
+      "completions/min_length": 1939.6,
+      "completions/min_terminated_length": 1939.6,
+      "entropy": 0.08963364362716675,
+      "epoch": 1.85,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1486842930316925,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.20532758235931398,
+      "num_tokens": 2709620.0,
+      "reward": 5.776600074768067,
+      "reward_std": 4.742965912818908,
+      "rewards/combined_reward_fn/mean": 5.776600074768067,
+      "rewards/combined_reward_fn/std": 4.7429660081863405,
+      "step": 185,
+      "step_time": 120.16855793340365
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 3764.2,
+      "completions/max_terminated_length": 3377.0,
+      "completions/mean_length": 2875.3,
+      "completions/mean_terminated_length": 2650.4,
+      "completions/min_length": 2316.2,
+      "completions/min_terminated_length": 2316.2,
+      "entropy": 0.09077955931425094,
+      "epoch": 1.9,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14634189009666443,
+      "learning_rate": 1.1e-06,
+      "loss": 0.00014334097504615785,
+      "num_tokens": 2789198.0,
+      "reward": 5.053549718856812,
+      "reward_std": 4.128131377696991,
+      "rewards/combined_reward_fn/mean": 5.053549718856812,
+      "rewards/combined_reward_fn/std": 4.128131443262101,
+      "step": 190,
+      "step_time": 118.10398631540593
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 3993.2,
+      "completions/max_terminated_length": 2888.4,
+      "completions/mean_length": 2753.0,
+      "completions/mean_terminated_length": 2300.500048828125,
+      "completions/min_length": 1844.4,
+      "completions/min_terminated_length": 1844.4,
+      "entropy": 0.09003306776285172,
+      "epoch": 1.95,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1261162906885147,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 0.20989387035369872,
+      "num_tokens": 2866242.0,
+      "reward": 5.2975998878479,
+      "reward_std": 4.944919896125794,
+      "rewards/combined_reward_fn/mean": 5.2975998878479,
+      "rewards/combined_reward_fn/std": 4.944919896125794,
+      "step": 195,
+      "step_time": 127.51930990898981
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 2903.2,
+      "completions/max_terminated_length": 2600.4,
+      "completions/mean_length": 2282.7,
+      "completions/mean_terminated_length": 2189.75,
+      "completions/min_length": 1668.2,
+      "completions/min_terminated_length": 1668.2,
+      "entropy": 0.09424156248569489,
+      "epoch": 2.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14964203536510468,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 0.00013558268547058104,
+      "num_tokens": 2933920.0,
+      "reward": 7.005850124359131,
+      "reward_std": 3.2380080938339235,
+      "rewards/combined_reward_fn/mean": 7.005850124359131,
+      "rewards/combined_reward_fn/std": 3.23800812959671,
+      "step": 200,
+      "step_time": 90.21042252739426
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 200,
+  "num_input_tokens_seen": 2933920,
+  "num_train_epochs": 2,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:291194e3ba32f6e08ebdda0990ba6335439bdc4b3136db9a90a3cca222fe0461
+size 6712

metrics/results.json

@@ -0,0 +1,140 @@
+{
+  "training_args": {
+    "model_id": "Qwen/Qwen3-1.7B",
+    "num_episodes": 100,
+    "num_generations": 4,
+    "max_steps": 200,
+    "learning_rate": 2e-05,
+    "batch_size": 4,
+    "max_completion_length": 4096,
+    "output_dir": "./checkpoints/grpo_api_tester",
+    "test_mode": false
+  },
+  "training_time_seconds": 21035.5,
+  "baseline_agents": {
+    "random": {
+      "basic_validation": {
+        "total_reward": 2.73,
+        "bugs_found": 0,
+        "total_bugs": 3,
+        "coverage_pct": 50.0
+      },
+      "edge_cases": {
+        "total_reward": 2.733,
+        "bugs_found": 2,
+        "total_bugs": 9,
+        "coverage_pct": 50.0
+      },
+      "security_workflows": {
+        "total_reward": 2.995,
+        "bugs_found": 3,
+        "total_bugs": 13,
+        "coverage_pct": 50.0
+      }
+    },
+    "sequential": {
+      "basic_validation": {
+        "total_reward": 4.32,
+        "bugs_found": 3,
+        "total_bugs": 3,
+        "coverage_pct": 50.0
+      },
+      "edge_cases": {
+        "total_reward": 4.07,
+        "bugs_found": 4,
+        "total_bugs": 9,
+        "coverage_pct": 50.0
+      },
+      "security_workflows": {
+        "total_reward": 3.645,
+        "bugs_found": 4,
+        "total_bugs": 13,
+        "coverage_pct": 50.0
+      }
+    },
+    "smart": {
+      "basic_validation": {
+        "total_reward": 4.86,
+        "bugs_found": 3,
+        "total_bugs": 3,
+        "coverage_pct": 50.0
+      },
+      "edge_cases": {
+        "total_reward": 5.185,
+        "bugs_found": 8,
+        "total_bugs": 9,
+        "coverage_pct": 50.0
+      },
+      "security_workflows": {
+        "total_reward": 5.135,
+        "bugs_found": 10,
+        "total_bugs": 13,
+        "coverage_pct": 50.0
+      }
+    }
+  },
+  "base_model": {
+    "basic_validation": {
+      "task_id": "basic_validation",
+      "seed": 9999,
+      "steps": 0,
+      "total_reward": 0.0,
+      "bugs_found": 0,
+      "total_bugs": 3,
+      "coverage_pct": 0.0,
+      "bugs_found_ids": []
+    },
+    "edge_cases": {
+      "task_id": "edge_cases",
+      "seed": 9999,
+      "steps": 0,
+      "total_reward": 0.0,
+      "bugs_found": 0,
+      "total_bugs": 9,
+      "coverage_pct": 0.0,
+      "bugs_found_ids": []
+    },
+    "security_workflows": {
+      "task_id": "security_workflows",
+      "seed": 9999,
+      "steps": 0,
+      "total_reward": 0.0,
+      "bugs_found": 0,
+      "total_bugs": 13,
+      "coverage_pct": 0.0,
+      "bugs_found_ids": []
+    }
+  },
+  "trained_model": {
+    "basic_validation": {
+      "task_id": "basic_validation",
+      "seed": 9999,
+      "steps": 0,
+      "total_reward": 0.0,
+      "bugs_found": 0,
+      "total_bugs": 3,
+      "coverage_pct": 0.0,
+      "bugs_found_ids": []
+    },
+    "edge_cases": {
+      "task_id": "edge_cases",
+      "seed": 9999,
+      "steps": 0,
+      "total_reward": 0.0,
+      "bugs_found": 0,
+      "total_bugs": 9,
+      "coverage_pct": 0.0,
+      "bugs_found_ids": []
+    },
+    "security_workflows": {
+      "task_id": "security_workflows",
+      "seed": 9999,
+      "steps": 0,
+      "total_reward": 0.0,
+      "bugs_found": 0,
+      "total_bugs": 13,
+      "coverage_pct": 0.0,
+      "bugs_found_ids": []
+    }
+  }
+}

metrics/results.md

@@ -0,0 +1,26 @@
+# Training Results
+
+**Model**: Qwen/Qwen3-1.7B
+**Training time**: 350.6 minutes
+**Episodes**: 100
+**Max steps**: 200
+
+## Comparison Table
+
+| Agent/Model | Task | Reward | Bugs | Coverage |
+|---|---|---|---|---|
+| random | basic_validation | 2.7300 | 0/3 | 50.0% |
+| random | edge_cases | 2.7330 | 2/9 | 50.0% |
+| random | security_workflows | 2.9950 | 3/13 | 50.0% |
+| sequential | basic_validation | 4.3200 | 3/3 | 50.0% |
+| sequential | edge_cases | 4.0700 | 4/9 | 50.0% |
+| sequential | security_workflows | 3.6450 | 4/13 | 50.0% |
+| smart | basic_validation | 4.8600 | 3/3 | 50.0% |
+| smart | edge_cases | 5.1850 | 8/9 | 50.0% |
+| smart | security_workflows | 5.1350 | 10/13 | 50.0% |
+| **base model** | basic_validation | 0.0000 | 0/3 | 0.0% |
+| **base model** | edge_cases | 0.0000 | 0/9 | 0.0% |
+| **base model** | security_workflows | 0.0000 | 0/13 | 0.0% |
+| **GRPO trained** | basic_validation | 0.0000 (+0.0000) | 0/3 | 0.0% |
+| **GRPO trained** | edge_cases | 0.0000 (+0.0000) | 0/9 | 0.0% |
+| **GRPO trained** | security_workflows | 0.0000 (+0.0000) | 0/13 | 0.0% |

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:291194e3ba32f6e08ebdda0990ba6335439bdc4b3136db9a90a3cca222fe0461
+size 6712