Upload task output 1

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: None
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": null,
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "down_proj",
+    "up_proj",
+    "gate_proj",
+    "q_proj",
+    "v_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af4d3b113c62a33eb1defe9d0552171ad65f80af82ec035625130db5741362c7
+size 239536272

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

loss.txt

@@ -0,0 +1 @@
+50,-0.3934959270060062

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,1319 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.0005,
+  "eval_steps": 500,
+  "global_step": 50,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11287.0,
+      "completions/max_terminated_length": 11287.0,
+      "completions/mean_length": 8394.21875,
+      "completions/mean_terminated_length": 8394.21875,
+      "completions/min_length": 2887.0,
+      "completions/min_terminated_length": 2887.0,
+      "entropy": 0.11788358655758202,
+      "epoch": 1e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.6996961832046509,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.1667,
+      "num_tokens": 297100.0,
+      "reward": -0.03610033914446831,
+      "reward_std": 0.7386711835861206,
+      "rewards/rollout_eval_reward_func/mean": 0.2578760087490082,
+      "rewards/rollout_eval_reward_func/std": 0.30121663212776184,
+      "rewards/rollout_reward_func/mean": -0.03610033914446831,
+      "rewards/rollout_reward_func/std": 0.728388786315918,
+      "sampling/importance_sampling_ratio/max": 1.6487237215042114,
+      "sampling/importance_sampling_ratio/mean": 0.9993462562561035,
+      "sampling/importance_sampling_ratio/min": 0.44016358256340027,
+      "sampling/sampling_logp_difference/max": 0.8206088542938232,
+      "sampling/sampling_logp_difference/mean": 0.011885393410921097,
+      "step": 1,
+      "step_time": 73.96733120499948
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "entropy": 0.11788358655758202,
+      "epoch": 2e-05,
+      "grad_norm": 1.699277400970459,
+      "kl": 0.0,
+      "learning_rate": 2.8571428571428573e-06,
+      "loss": 0.1667,
+      "step": 2,
+      "step_time": 33.627345983999476
+    },
+    {
+      "clip_ratio/high_max": 0.005571161163970828,
+      "clip_ratio/high_mean": 0.0031328028126154095,
+      "clip_ratio/low_mean": 0.007165575778344646,
+      "clip_ratio/low_min": 0.0007102272938936949,
+      "clip_ratio/region_mean": 0.010298378649167717,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11163.0,
+      "completions/max_terminated_length": 11163.0,
+      "completions/mean_length": 8984.84375,
+      "completions/mean_terminated_length": 8984.84375,
+      "completions/min_length": 346.0,
+      "completions/min_terminated_length": 346.0,
+      "entropy": 0.14975268673151731,
+      "epoch": 3e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3866604566574097,
+      "kl": 0.002339825068702339,
+      "learning_rate": 5.7142857142857145e-06,
+      "loss": -0.2053,
+      "num_tokens": 613530.0,
+      "reward": -0.20636191964149475,
+      "reward_std": 0.5412060022354126,
+      "rewards/rollout_eval_reward_func/mean": 0.13351118564605713,
+      "rewards/rollout_eval_reward_func/std": 0.24266032874584198,
+      "rewards/rollout_reward_func/mean": -0.20636191964149475,
+      "rewards/rollout_reward_func/std": 0.542148232460022,
+      "sampling/importance_sampling_ratio/max": 1.7320148944854736,
+      "sampling/importance_sampling_ratio/mean": 1.0001366138458252,
+      "sampling/importance_sampling_ratio/min": 0.5123909115791321,
+      "sampling/sampling_logp_difference/max": 0.6686674356460571,
+      "sampling/sampling_logp_difference/mean": 0.014301672577857971,
+      "step": 3,
+      "step_time": 76.45311136299915
+    },
+    {
+      "clip_ratio/high_max": 0.0063213135581463575,
+      "clip_ratio/high_mean": 0.003855101269437,
+      "clip_ratio/low_mean": 0.0049675037153065205,
+      "clip_ratio/low_min": 0.0006944444612599909,
+      "clip_ratio/region_mean": 0.00882260489743203,
+      "entropy": 0.14919065916910768,
+      "epoch": 4e-05,
+      "grad_norm": 1.35505211353302,
+      "kl": 0.0014064895312912995,
+      "learning_rate": 8.571428571428573e-06,
+      "loss": -0.2073,
+      "step": 4,
+      "step_time": 34.60768302199995
+    },
+    {
+      "clip_ratio/high_max": 0.007777874590829015,
+      "clip_ratio/high_mean": 0.0042565843323245645,
+      "clip_ratio/low_mean": 0.003121181536698714,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0073777658108156174,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11443.0,
+      "completions/max_terminated_length": 11443.0,
+      "completions/mean_length": 8618.625,
+      "completions/mean_terminated_length": 8618.625,
+      "completions/min_length": 336.0,
+      "completions/min_terminated_length": 336.0,
+      "entropy": 0.11723149428144097,
+      "epoch": 5e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.2567578554153442,
+      "kl": 0.0012220569333294407,
+      "learning_rate": 1.1428571428571429e-05,
+      "loss": 0.14,
+      "num_tokens": 917591.0,
+      "reward": -0.0526789128780365,
+      "reward_std": 0.7260563373565674,
+      "rewards/rollout_eval_reward_func/mean": 0.24098068475723267,
+      "rewards/rollout_eval_reward_func/std": 0.30969831347465515,
+      "rewards/rollout_reward_func/mean": -0.0526789128780365,
+      "rewards/rollout_reward_func/std": 0.7283722758293152,
+      "sampling/importance_sampling_ratio/max": 1.8090978860855103,
+      "sampling/importance_sampling_ratio/mean": 0.9998831748962402,
+      "sampling/importance_sampling_ratio/min": 0.46312808990478516,
+      "sampling/sampling_logp_difference/max": 0.7697516679763794,
+      "sampling/sampling_logp_difference/mean": 0.012965342029929161,
+      "step": 5,
+      "step_time": 76.55231227500053
+    },
+    {
+      "clip_ratio/high_max": 0.006731559580657631,
+      "clip_ratio/high_mean": 0.0033657797903288156,
+      "clip_ratio/low_mean": 0.0034470450482331216,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006812824809458107,
+      "entropy": 0.1164796226657927,
+      "epoch": 6e-05,
+      "grad_norm": 1.2304322719573975,
+      "kl": 0.001537940081107081,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 0.1411,
+      "step": 6,
+      "step_time": 34.486522669000806
+    },
+    {
+      "clip_ratio/high_max": 0.004329371266067028,
+      "clip_ratio/high_mean": 0.002164685633033514,
+      "clip_ratio/low_mean": 0.0018710909644141793,
+      "clip_ratio/low_min": 0.0007183908019214869,
+      "clip_ratio/region_mean": 0.004035776626551524,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11026.0,
+      "completions/max_terminated_length": 11026.0,
+      "completions/mean_length": 8904.9375,
+      "completions/mean_terminated_length": 8904.9375,
+      "completions/min_length": 1870.0,
+      "completions/min_terminated_length": 1870.0,
+      "entropy": 0.11141919903457165,
+      "epoch": 7e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.0392634868621826,
+      "kl": 0.0028663520533882547,
+      "learning_rate": 1.7142857142857145e-05,
+      "loss": 0.2956,
+      "num_tokens": 1231697.0,
+      "reward": 0.06170085445046425,
+      "reward_std": 0.6284129619598389,
+      "rewards/rollout_eval_reward_func/mean": 0.1812753975391388,
+      "rewards/rollout_eval_reward_func/std": 0.29651859402656555,
+      "rewards/rollout_reward_func/mean": 0.06170085445046425,
+      "rewards/rollout_reward_func/std": 0.6143175363540649,
+      "sampling/importance_sampling_ratio/max": 3.0,
+      "sampling/importance_sampling_ratio/mean": 1.0020291805267334,
+      "sampling/importance_sampling_ratio/min": 0.4785745441913605,
+      "sampling/sampling_logp_difference/max": 1.289139747619629,
+      "sampling/sampling_logp_difference/mean": 0.012369468808174133,
+      "step": 7,
+      "step_time": 79.23396538400038
+    },
+    {
+      "clip_ratio/high_max": 0.004530098114628345,
+      "clip_ratio/high_mean": 0.0022650490573141724,
+      "clip_ratio/low_mean": 0.0062050665146671236,
+      "clip_ratio/low_min": 0.0007183908019214869,
+      "clip_ratio/region_mean": 0.008470115601085126,
+      "entropy": 0.10932181030511856,
+      "epoch": 8e-05,
+      "grad_norm": 1.4861699342727661,
+      "kl": 0.004455661201063776,
+      "learning_rate": 2e-05,
+      "loss": 0.291,
+      "step": 8,
+      "step_time": 33.553139639999245
+    },
+    {
+      "clip_ratio/high_max": 0.006702587008476257,
+      "clip_ratio/high_mean": 0.00395225512329489,
+      "clip_ratio/low_mean": 0.001561393088195473,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005513648211490363,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11738.0,
+      "completions/max_terminated_length": 11738.0,
+      "completions/mean_length": 7849.21875,
+      "completions/mean_terminated_length": 7849.21875,
+      "completions/min_length": 323.0,
+      "completions/min_terminated_length": 323.0,
+      "entropy": 0.10142783727496862,
+      "epoch": 9e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.1208703517913818,
+      "kl": 0.005338215534720803,
+      "learning_rate": 2.2857142857142858e-05,
+      "loss": -0.0444,
+      "num_tokens": 1511511.0,
+      "reward": -0.1137353852391243,
+      "reward_std": 0.6742830872535706,
+      "rewards/rollout_eval_reward_func/mean": 0.21138212084770203,
+      "rewards/rollout_eval_reward_func/std": 0.2764902412891388,
+      "rewards/rollout_reward_func/mean": -0.1137353852391243,
+      "rewards/rollout_reward_func/std": 0.6757553219795227,
+      "sampling/importance_sampling_ratio/max": 1.9115883111953735,
+      "sampling/importance_sampling_ratio/mean": 0.9999130964279175,
+      "sampling/importance_sampling_ratio/min": 0.6430048942565918,
+      "sampling/sampling_logp_difference/max": 0.6479344367980957,
+      "sampling/sampling_logp_difference/mean": 0.009747568517923355,
+      "step": 9,
+      "step_time": 78.92596304499875
+    },
+    {
+      "clip_ratio/high_max": 0.014946904906537384,
+      "clip_ratio/high_mean": 0.008911944547435269,
+      "clip_ratio/low_mean": 0.008717332733795047,
+      "clip_ratio/low_min": 0.0007022471982054412,
+      "clip_ratio/region_mean": 0.017629277397645637,
+      "entropy": 0.09894524328410625,
+      "epoch": 0.0001,
+      "grad_norm": 0.8779318332672119,
+      "kl": 0.012525228099548258,
+      "learning_rate": 2.5714285714285714e-05,
+      "loss": -0.0474,
+      "step": 10,
+      "step_time": 33.86218772000029
+    },
+    {
+      "clip_ratio/high_max": 0.005196896672714502,
+      "clip_ratio/high_mean": 0.002598448336357251,
+      "clip_ratio/low_mean": 0.00565887795528397,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00825732629164122,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11663.0,
+      "completions/max_terminated_length": 11663.0,
+      "completions/mean_length": 9176.8125,
+      "completions/mean_terminated_length": 9176.8125,
+      "completions/min_length": 2189.0,
+      "completions/min_terminated_length": 2189.0,
+      "entropy": 0.1143840029835701,
+      "epoch": 0.00011,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.9076919555664062,
+      "kl": 0.02556461405765731,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 0.1457,
+      "num_tokens": 1833833.0,
+      "reward": -0.07531464844942093,
+      "reward_std": 0.6547598838806152,
+      "rewards/rollout_eval_reward_func/mean": 0.1659044623374939,
+      "rewards/rollout_eval_reward_func/std": 0.27956825494766235,
+      "rewards/rollout_reward_func/mean": -0.07531464844942093,
+      "rewards/rollout_reward_func/std": 0.6382541060447693,
+      "sampling/importance_sampling_ratio/max": 2.9357423782348633,
+      "sampling/importance_sampling_ratio/mean": 0.9987922310829163,
+      "sampling/importance_sampling_ratio/min": 0.4146113395690918,
+      "sampling/sampling_logp_difference/max": 1.0769603252410889,
+      "sampling/sampling_logp_difference/mean": 0.01623542606830597,
+      "step": 11,
+      "step_time": 85.64898062899738
+    },
+    {
+      "clip_ratio/high_max": 0.007925633108243346,
+      "clip_ratio/high_mean": 0.003962816554121673,
+      "clip_ratio/low_mean": 0.013630853296490386,
+      "clip_ratio/low_min": 0.004900663043372333,
+      "clip_ratio/region_mean": 0.01759366990881972,
+      "entropy": 0.11427765572443604,
+      "epoch": 0.00012,
+      "grad_norm": 1.6511975526809692,
+      "kl": 0.048679544823244214,
+      "learning_rate": 3.142857142857143e-05,
+      "loss": 0.142,
+      "step": 12,
+      "step_time": 34.99860805099979
+    },
+    {
+      "clip_ratio/high_max": 0.003243650193326175,
+      "clip_ratio/high_mean": 0.0016218250966630876,
+      "clip_ratio/low_mean": 0.0024501511070411652,
+      "clip_ratio/low_min": 0.0006944444612599909,
+      "clip_ratio/region_mean": 0.004071976203704253,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11007.0,
+      "completions/max_terminated_length": 11007.0,
+      "completions/mean_length": 8355.625,
+      "completions/mean_terminated_length": 8355.625,
+      "completions/min_length": 874.0,
+      "completions/min_terminated_length": 874.0,
+      "entropy": 0.11024783831089735,
+      "epoch": 0.00013,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.4035089015960693,
+      "kl": 0.08716274873586372,
+      "learning_rate": 3.428571428571429e-05,
+      "loss": 0.1218,
+      "num_tokens": 2130373.0,
+      "reward": 0.020019814372062683,
+      "reward_std": 0.6314284801483154,
+      "rewards/rollout_eval_reward_func/mean": 0.21633638441562653,
+      "rewards/rollout_eval_reward_func/std": 0.31151828169822693,
+      "rewards/rollout_reward_func/mean": 0.020019814372062683,
+      "rewards/rollout_reward_func/std": 0.689924418926239,
+      "sampling/importance_sampling_ratio/max": 3.0,
+      "sampling/importance_sampling_ratio/mean": 1.0034040212631226,
+      "sampling/importance_sampling_ratio/min": 0.20111392438411713,
+      "sampling/sampling_logp_difference/max": 1.6038837432861328,
+      "sampling/sampling_logp_difference/mean": 0.02121809497475624,
+      "step": 13,
+      "step_time": 86.70206161200258
+    },
+    {
+      "clip_ratio/high_max": 0.009814315650146455,
+      "clip_ratio/high_mean": 0.005678762652678415,
+      "clip_ratio/low_mean": 0.012552198284538463,
+      "clip_ratio/low_min": 0.004237600835040212,
+      "clip_ratio/region_mean": 0.01823096099542454,
+      "entropy": 0.10833629406988621,
+      "epoch": 0.00014,
+      "grad_norm": 2.254364490509033,
+      "kl": 0.07821251518907957,
+      "learning_rate": 3.7142857142857143e-05,
+      "loss": 0.1127,
+      "step": 14,
+      "step_time": 33.7453661569989
+    },
+    {
+      "clip_ratio/high_max": 0.004190173640381545,
+      "clip_ratio/high_mean": 0.0020950868201907724,
+      "clip_ratio/low_mean": 0.0028092548600398004,
+      "clip_ratio/low_min": 0.0007022471982054412,
+      "clip_ratio/region_mean": 0.004904341680230573,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10761.0,
+      "completions/max_terminated_length": 10761.0,
+      "completions/mean_length": 8177.6875,
+      "completions/mean_terminated_length": 8177.6875,
+      "completions/min_length": 811.0,
+      "completions/min_terminated_length": 811.0,
+      "entropy": 0.10465786419808865,
+      "epoch": 0.00015,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.6561901569366455,
+      "kl": 0.1072866928880103,
+      "learning_rate": 4e-05,
+      "loss": 0.356,
+      "num_tokens": 2420931.0,
+      "reward": 0.11278900504112244,
+      "reward_std": 0.6452403664588928,
+      "rewards/rollout_eval_reward_func/mean": 0.23056402802467346,
+      "rewards/rollout_eval_reward_func/std": 0.3142653703689575,
+      "rewards/rollout_reward_func/mean": 0.11278900504112244,
+      "rewards/rollout_reward_func/std": 0.6579005718231201,
+      "sampling/importance_sampling_ratio/max": 2.278707504272461,
+      "sampling/importance_sampling_ratio/mean": 0.9995414018630981,
+      "sampling/importance_sampling_ratio/min": 0.18554040789604187,
+      "sampling/sampling_logp_difference/max": 1.6844825744628906,
+      "sampling/sampling_logp_difference/mean": 0.023679137229919434,
+      "step": 15,
+      "step_time": 84.81073227599518
+    },
+    {
+      "clip_ratio/high_max": 0.00427410431439057,
+      "clip_ratio/high_mean": 0.002137052157195285,
+      "clip_ratio/low_mean": 0.013172850711271167,
+      "clip_ratio/low_min": 0.00423778424737975,
+      "clip_ratio/region_mean": 0.015309902926674113,
+      "entropy": 0.1049683466553688,
+      "epoch": 0.00016,
+      "grad_norm": 1.5131983757019043,
+      "kl": 0.12803897575940937,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 0.3499,
+      "step": 16,
+      "step_time": 32.4860796890016
+    },
+    {
+      "clip_ratio/high_max": 0.006543069262988865,
+      "clip_ratio/high_mean": 0.003775566816329956,
+      "clip_ratio/low_mean": 0.0031609196157660335,
+      "clip_ratio/low_min": 0.0006944444612599909,
+      "clip_ratio/region_mean": 0.00693648643209599,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11153.0,
+      "completions/max_terminated_length": 11153.0,
+      "completions/mean_length": 8529.6875,
+      "completions/mean_terminated_length": 8529.6875,
+      "completions/min_length": 330.0,
+      "completions/min_terminated_length": 330.0,
+      "entropy": 0.10206164792180061,
+      "epoch": 0.00017,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3023760318756104,
+      "kl": 0.09016790508758277,
+      "learning_rate": 4.5714285714285716e-05,
+      "loss": -0.055,
+      "num_tokens": 2722874.0,
+      "reward": 0.0452897846698761,
+      "reward_std": 0.5471868515014648,
+      "rewards/rollout_eval_reward_func/mean": 0.16552338004112244,
+      "rewards/rollout_eval_reward_func/std": 0.2769395411014557,
+      "rewards/rollout_reward_func/mean": 0.0452897846698761,
+      "rewards/rollout_reward_func/std": 0.5707083344459534,
+      "sampling/importance_sampling_ratio/max": 1.9813306331634521,
+      "sampling/importance_sampling_ratio/mean": 0.9965455532073975,
+      "sampling/importance_sampling_ratio/min": 0.1743103712797165,
+      "sampling/sampling_logp_difference/max": 1.746917724609375,
+      "sampling/sampling_logp_difference/mean": 0.021249916404485703,
+      "step": 17,
+      "step_time": 93.31380597800035
+    },
+    {
+      "clip_ratio/high_max": 0.013804474321659654,
+      "clip_ratio/high_mean": 0.008459640172077343,
+      "clip_ratio/low_mean": 0.009875382616883144,
+      "clip_ratio/low_min": 0.002192793588619679,
+      "clip_ratio/region_mean": 0.018335022643441334,
+      "entropy": 0.10283505916595459,
+      "epoch": 0.00018,
+      "grad_norm": 1.0806056261062622,
+      "kl": 0.11839380749734119,
+      "learning_rate": 4.8571428571428576e-05,
+      "loss": -0.0586,
+      "step": 18,
+      "step_time": 32.5428760199984
+    },
+    {
+      "clip_ratio/high_max": 0.004807963909115642,
+      "clip_ratio/high_mean": 0.002403981954557821,
+      "clip_ratio/low_mean": 0.004428166459547356,
+      "clip_ratio/low_min": 0.0006944444612599909,
+      "clip_ratio/region_mean": 0.006832148414105177,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10399.0,
+      "completions/max_terminated_length": 10399.0,
+      "completions/mean_length": 7525.5625,
+      "completions/mean_terminated_length": 7525.5625,
+      "completions/min_length": 1309.0,
+      "completions/min_terminated_length": 1309.0,
+      "entropy": 0.10726649826392531,
+      "epoch": 0.00019,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3821837902069092,
+      "kl": 0.09694959409534931,
+      "learning_rate": 5.142857142857143e-05,
+      "loss": 0.2508,
+      "num_tokens": 2992614.0,
+      "reward": 0.22085225582122803,
+      "reward_std": 0.7439246773719788,
+      "rewards/rollout_eval_reward_func/mean": 0.32672765851020813,
+      "rewards/rollout_eval_reward_func/std": 0.3242030739784241,
+      "rewards/rollout_reward_func/mean": 0.22085225582122803,
+      "rewards/rollout_reward_func/std": 0.7110569477081299,
+      "sampling/importance_sampling_ratio/max": 3.0,
+      "sampling/importance_sampling_ratio/mean": 1.0050089359283447,
+      "sampling/importance_sampling_ratio/min": 0.2929533123970032,
+      "sampling/sampling_logp_difference/max": 1.916597604751587,
+      "sampling/sampling_logp_difference/mean": 0.022352777421474457,
+      "step": 19,
+      "step_time": 89.20460286800153
+    },
+    {
+      "clip_ratio/high_max": 0.011035054223611951,
+      "clip_ratio/high_mean": 0.0055175271118059754,
+      "clip_ratio/low_mean": 0.018963277572765946,
+      "clip_ratio/low_min": 0.0055555556900799274,
+      "clip_ratio/region_mean": 0.024480804742779583,
+      "entropy": 0.1047364235855639,
+      "epoch": 0.0002,
+      "grad_norm": 0.9138187170028687,
+      "kl": 0.10579214652534574,
+      "learning_rate": 5.428571428571428e-05,
+      "loss": 0.2408,
+      "step": 20,
+      "step_time": 30.517773796002075
+    },
+    {
+      "clip_ratio/high_max": 0.004395612399093807,
+      "clip_ratio/high_mean": 0.0028961520874872804,
+      "clip_ratio/low_mean": 0.0034880968159995973,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006384248874383047,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11335.0,
+      "completions/max_terminated_length": 11335.0,
+      "completions/mean_length": 8358.6875,
+      "completions/mean_terminated_length": 8358.6875,
+      "completions/min_length": 749.0,
+      "completions/min_terminated_length": 749.0,
+      "entropy": 0.12294450332410634,
+      "epoch": 0.00021,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3461740016937256,
+      "kl": 0.19072263734415174,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": -0.0037,
+      "num_tokens": 3288905.0,
+      "reward": -0.1559383124113083,
+      "reward_std": 0.5600723028182983,
+      "rewards/rollout_eval_reward_func/mean": 0.14430895447731018,
+      "rewards/rollout_eval_reward_func/std": 0.2627376317977905,
+      "rewards/rollout_reward_func/mean": -0.1559383124113083,
+      "rewards/rollout_reward_func/std": 0.5887359380722046,
+      "sampling/importance_sampling_ratio/max": 2.2654411792755127,
+      "sampling/importance_sampling_ratio/mean": 0.9976513385772705,
+      "sampling/importance_sampling_ratio/min": 0.20272883772850037,
+      "sampling/sampling_logp_difference/max": 1.595885992050171,
+      "sampling/sampling_logp_difference/mean": 0.02302836999297142,
+      "step": 21,
+      "step_time": 97.79121274199952
+    },
+    {
+      "clip_ratio/high_max": 0.015567402355372906,
+      "clip_ratio/high_mean": 0.009180392837151885,
+      "clip_ratio/low_mean": 0.017762881761882454,
+      "clip_ratio/low_min": 0.006345431203953922,
+      "clip_ratio/region_mean": 0.026943274919176474,
+      "entropy": 0.12321826652623713,
+      "epoch": 0.00022,
+      "grad_norm": 1.64430570602417,
+      "kl": 0.269085428211838,
+      "learning_rate": 6e-05,
+      "loss": -0.0084,
+      "step": 22,
+      "step_time": 32.807887424998626
+    },
+    {
+      "clip_ratio/high_max": 0.003910088969860226,
+      "clip_ratio/high_mean": 0.001955044484930113,
+      "clip_ratio/low_mean": 0.002580859523732215,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004535904008662328,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11344.0,
+      "completions/max_terminated_length": 11344.0,
+      "completions/mean_length": 8706.71875,
+      "completions/mean_terminated_length": 8706.71875,
+      "completions/min_length": 1846.0,
+      "completions/min_terminated_length": 1846.0,
+      "entropy": 0.1066858924459666,
+      "epoch": 0.00023,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.9362024068832397,
+      "kl": 0.12137257889844477,
+      "learning_rate": 6.285714285714286e-05,
+      "loss": 0.2385,
+      "num_tokens": 3596056.0,
+      "reward": 0.12735633552074432,
+      "reward_std": 0.6423711776733398,
+      "rewards/rollout_eval_reward_func/mean": 0.2489837408065796,
+      "rewards/rollout_eval_reward_func/std": 0.3152793347835541,
+      "rewards/rollout_reward_func/mean": 0.12735633552074432,
+      "rewards/rollout_reward_func/std": 0.7003796696662903,
+      "sampling/importance_sampling_ratio/max": 2.351552963256836,
+      "sampling/importance_sampling_ratio/mean": 1.0003013610839844,
+      "sampling/importance_sampling_ratio/min": 0.3779045045375824,
+      "sampling/sampling_logp_difference/max": 0.9731137752532959,
+      "sampling/sampling_logp_difference/mean": 0.016432534903287888,
+      "step": 23,
+      "step_time": 99.05041999199784
+    },
+    {
+      "clip_ratio/high_max": 0.011161994189023972,
+      "clip_ratio/high_mean": 0.0066947695158887655,
+      "clip_ratio/low_mean": 0.009474401478655636,
+      "clip_ratio/low_min": 0.0021156331058591604,
+      "clip_ratio/region_mean": 0.01616917096544057,
+      "entropy": 0.11043372261337936,
+      "epoch": 0.00024,
+      "grad_norm": 1.812207818031311,
+      "kl": 0.09031509491614997,
+      "learning_rate": 6.571428571428571e-05,
+      "loss": 0.2393,
+      "step": 24,
+      "step_time": 32.823300022000694
+    },
+    {
+      "clip_ratio/high_max": 0.006489152496214956,
+      "clip_ratio/high_mean": 0.003244576248107478,
+      "clip_ratio/low_mean": 0.0018077631539199501,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005052339431131259,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10988.0,
+      "completions/max_terminated_length": 10988.0,
+      "completions/mean_length": 8457.65625,
+      "completions/mean_terminated_length": 8457.65625,
+      "completions/min_length": 1525.0,
+      "completions/min_terminated_length": 1525.0,
+      "entropy": 0.11516976961866021,
+      "epoch": 0.00025,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3577765226364136,
+      "kl": 0.1303024498047307,
+      "learning_rate": 6.857142857142858e-05,
+      "loss": 0.1265,
+      "num_tokens": 3894768.0,
+      "reward": 0.20315799117088318,
+      "reward_std": 0.6316713094711304,
+      "rewards/rollout_eval_reward_func/mean": 0.240091472864151,
+      "rewards/rollout_eval_reward_func/std": 0.31059667468070984,
+      "rewards/rollout_reward_func/mean": 0.20315799117088318,
+      "rewards/rollout_reward_func/std": 0.6295517683029175,
+      "sampling/importance_sampling_ratio/max": 2.256138801574707,
+      "sampling/importance_sampling_ratio/mean": 0.9993441104888916,
+      "sampling/importance_sampling_ratio/min": 0.2282668501138687,
+      "sampling/sampling_logp_difference/max": 1.477239966392517,
+      "sampling/sampling_logp_difference/mean": 0.01688190922141075,
+      "step": 25,
+      "step_time": 96.7960174280006
+    },
+    {
+      "clip_ratio/high_max": 0.016165015986189246,
+      "clip_ratio/high_mean": 0.010140184662304819,
+      "clip_ratio/low_mean": 0.027886587660759687,
+      "clip_ratio/low_min": 0.013233458623290062,
+      "clip_ratio/region_mean": 0.038026772264856845,
+      "entropy": 0.11189563758671284,
+      "epoch": 0.00026,
+      "grad_norm": 2.0246684551239014,
+      "kl": 0.2296465744730085,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 0.1164,
+      "step": 26,
+      "step_time": 32.81865678699978
+    },
+    {
+      "clip_ratio/high_max": 0.0038667930057272315,
+      "clip_ratio/high_mean": 0.0019333965028636158,
+      "clip_ratio/low_mean": 0.003156299644615501,
+      "clip_ratio/low_min": 0.0014046717551536858,
+      "clip_ratio/region_mean": 0.005089696147479117,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11043.0,
+      "completions/max_terminated_length": 11043.0,
+      "completions/mean_length": 8288.09375,
+      "completions/mean_terminated_length": 8288.09375,
+      "completions/min_length": 873.0,
+      "completions/min_terminated_length": 873.0,
+      "entropy": 0.1172712086699903,
+      "epoch": 0.00027,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.0188708305358887,
+      "kl": 0.14066953863948584,
+      "learning_rate": 7.428571428571429e-05,
+      "loss": 0.019,
+      "num_tokens": 4187980.0,
+      "reward": 0.3152431547641754,
+      "reward_std": 0.6300436854362488,
+      "rewards/rollout_eval_reward_func/mean": 0.31758129596710205,
+      "rewards/rollout_eval_reward_func/std": 0.32922983169555664,
+      "rewards/rollout_reward_func/mean": 0.3152431547641754,
+      "rewards/rollout_reward_func/std": 0.6763298511505127,
+      "sampling/importance_sampling_ratio/max": 3.0,
+      "sampling/importance_sampling_ratio/mean": 0.9995622634887695,
+      "sampling/importance_sampling_ratio/min": 0.4497600793838501,
+      "sampling/sampling_logp_difference/max": 1.9223380088806152,
+      "sampling/sampling_logp_difference/mean": 0.017684465274214745,
+      "step": 27,
+      "step_time": 96.53867048800112
+    },
+    {
+      "clip_ratio/high_max": 0.01634173752972856,
+      "clip_ratio/high_mean": 0.00923296029213816,
+      "clip_ratio/low_mean": 0.011995688633760437,
+      "clip_ratio/low_min": 0.0042134831310249865,
+      "clip_ratio/region_mean": 0.021228648955002427,
+      "entropy": 0.11727899359539151,
+      "epoch": 0.00028,
+      "grad_norm": 1.0265206098556519,
+      "kl": 0.1646709272172302,
+      "learning_rate": 7.714285714285715e-05,
+      "loss": 0.0122,
+      "step": 28,
+      "step_time": 32.412335121999604
+    },
+    {
+      "clip_ratio/high_max": 0.0030616914154961705,
+      "clip_ratio/high_mean": 0.0015308457077480853,
+      "clip_ratio/low_mean": 0.0017557066748850048,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003286552324425429,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11171.0,
+      "completions/max_terminated_length": 11171.0,
+      "completions/mean_length": 7933.9375,
+      "completions/mean_terminated_length": 7933.9375,
+      "completions/min_length": 1855.0,
+      "completions/min_terminated_length": 1855.0,
+      "entropy": 0.10820061061531305,
+      "epoch": 0.00029,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.078338623046875,
+      "kl": 0.12938267178833485,
+      "learning_rate": 8e-05,
+      "loss": 0.1072,
+      "num_tokens": 4469763.0,
+      "reward": 0.19558671116828918,
+      "reward_std": 0.6997864842414856,
+      "rewards/rollout_eval_reward_func/mean": 0.2963668704032898,
+      "rewards/rollout_eval_reward_func/std": 0.3109610080718994,
+      "rewards/rollout_reward_func/mean": 0.19558671116828918,
+      "rewards/rollout_reward_func/std": 0.6721439361572266,
+      "sampling/importance_sampling_ratio/max": 1.8601394891738892,
+      "sampling/importance_sampling_ratio/mean": 1.0000590085983276,
+      "sampling/importance_sampling_ratio/min": 0.5378966927528381,
+      "sampling/sampling_logp_difference/max": 0.6206514835357666,
+      "sampling/sampling_logp_difference/mean": 0.01655469462275505,
+      "step": 29,
+      "step_time": 93.66265234299863
+    },
+    {
+      "clip_ratio/high_max": 0.017240845831111073,
+      "clip_ratio/high_mean": 0.009188604657538235,
+      "clip_ratio/low_mean": 0.018311504449229687,
+      "clip_ratio/low_min": 0.004907927650492638,
+      "clip_ratio/region_mean": 0.027500109281390905,
+      "entropy": 0.10658967308700085,
+      "epoch": 0.0003,
+      "grad_norm": 0.5991150140762329,
+      "kl": 0.14486833242699504,
+      "learning_rate": 8.285714285714287e-05,
+      "loss": 0.0961,
+      "step": 30,
+      "step_time": 32.02409046399953
+    },
+    {
+      "clip_ratio/high_max": 0.007639755902346224,
+      "clip_ratio/high_mean": 0.003819877951173112,
+      "clip_ratio/low_mean": 0.0032558340462855995,
+      "clip_ratio/low_min": 0.000811688310932368,
+      "clip_ratio/region_mean": 0.007075712055666372,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11207.0,
+      "completions/max_terminated_length": 11207.0,
+      "completions/mean_length": 7446.4375,
+      "completions/mean_terminated_length": 7446.4375,
+      "completions/min_length": 1941.0,
+      "completions/min_terminated_length": 1941.0,
+      "entropy": 0.1075986991636455,
+      "epoch": 0.00031,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3741053342819214,
+      "kl": 0.1601699753664434,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 0.3119,
+      "num_tokens": 4736188.0,
+      "reward": 0.3060583770275116,
+      "reward_std": 0.7248314619064331,
+      "rewards/rollout_eval_reward_func/mean": 0.3489583134651184,
+      "rewards/rollout_eval_reward_func/std": 0.3138476312160492,
+      "rewards/rollout_reward_func/mean": 0.3060583770275116,
+      "rewards/rollout_reward_func/std": 0.7231479287147522,
+      "sampling/importance_sampling_ratio/max": 2.0959339141845703,
+      "sampling/importance_sampling_ratio/mean": 1.0005576610565186,
+      "sampling/importance_sampling_ratio/min": 0.3962559998035431,
+      "sampling/sampling_logp_difference/max": 0.9256948232650757,
+      "sampling/sampling_logp_difference/mean": 0.015314233489334583,
+      "step": 31,
+      "step_time": 89.359079396997
+    },
+    {
+      "clip_ratio/high_max": 0.018800459045451134,
+      "clip_ratio/high_mean": 0.010931086115306243,
+      "clip_ratio/low_mean": 0.014964523201342672,
+      "clip_ratio/low_min": 0.002763190073892474,
+      "clip_ratio/region_mean": 0.025895609316648915,
+      "entropy": 0.11107803648337722,
+      "epoch": 0.00032,
+      "grad_norm": 0.8436726927757263,
+      "kl": 0.17236854787915945,
+      "learning_rate": 8.857142857142857e-05,
+      "loss": 0.2984,
+      "step": 32,
+      "step_time": 32.26129659800063
+    },
+    {
+      "clip_ratio/high_max": 0.005184833600651473,
+      "clip_ratio/high_mean": 0.0025924168003257364,
+      "clip_ratio/low_mean": 0.0021227904653642327,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004715207265689969,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11234.0,
+      "completions/max_terminated_length": 11234.0,
+      "completions/mean_length": 8056.90625,
+      "completions/mean_terminated_length": 8056.90625,
+      "completions/min_length": 834.0,
+      "completions/min_terminated_length": 834.0,
+      "entropy": 0.10890151979401708,
+      "epoch": 0.00033,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.012204647064209,
+      "kl": 0.13646527798846364,
+      "learning_rate": 9.142857142857143e-05,
+      "loss": 0.2208,
+      "num_tokens": 5021855.0,
+      "reward": 0.28810450434684753,
+      "reward_std": 0.6511638164520264,
+      "rewards/rollout_eval_reward_func/mean": 0.27172255516052246,
+      "rewards/rollout_eval_reward_func/std": 0.3220342993736267,
+      "rewards/rollout_reward_func/mean": 0.28810450434684753,
+      "rewards/rollout_reward_func/std": 0.6260398030281067,
+      "sampling/importance_sampling_ratio/max": 1.8270360231399536,
+      "sampling/importance_sampling_ratio/mean": 1.0004476308822632,
+      "sampling/importance_sampling_ratio/min": 0.2768998444080353,
+      "sampling/sampling_logp_difference/max": 1.2840993404388428,
+      "sampling/sampling_logp_difference/mean": 0.012767134234309196,
+      "step": 33,
+      "step_time": 93.59931743799825
+    },
+    {
+      "clip_ratio/high_max": 0.013980166870169342,
+      "clip_ratio/high_mean": 0.008351352356839925,
+      "clip_ratio/low_mean": 0.016900451679248363,
+      "clip_ratio/low_min": 0.0068813529796898365,
+      "clip_ratio/region_mean": 0.025251803919672966,
+      "entropy": 0.1073108701966703,
+      "epoch": 0.00034,
+      "grad_norm": 0.6274716258049011,
+      "kl": 0.14035740494728088,
+      "learning_rate": 9.428571428571429e-05,
+      "loss": 0.2118,
+      "step": 34,
+      "step_time": 32.919508103002954
+    },
+    {
+      "clip_ratio/high_max": 0.0018894725944846869,
+      "clip_ratio/high_mean": 0.0009447362972423434,
+      "clip_ratio/low_mean": 0.0021555489802267402,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0031002852774690837,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11099.0,
+      "completions/max_terminated_length": 11099.0,
+      "completions/mean_length": 8609.5,
+      "completions/mean_terminated_length": 8609.5,
+      "completions/min_length": 1070.0,
+      "completions/min_terminated_length": 1070.0,
+      "entropy": 0.10947995679453015,
+      "epoch": 0.00035,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.9109163284301758,
+      "kl": 0.24753205897286534,
+      "learning_rate": 9.714285714285715e-05,
+      "loss": 0.1542,
+      "num_tokens": 5325755.0,
+      "reward": 0.4508249759674072,
+      "reward_std": 0.5780479907989502,
+      "rewards/rollout_eval_reward_func/mean": 0.32621949911117554,
+      "rewards/rollout_eval_reward_func/std": 0.3382527828216553,
+      "rewards/rollout_reward_func/mean": 0.4508249759674072,
+      "rewards/rollout_reward_func/std": 0.5698684453964233,
+      "sampling/importance_sampling_ratio/max": 1.7834506034851074,
+      "sampling/importance_sampling_ratio/mean": 1.0010955333709717,
+      "sampling/importance_sampling_ratio/min": 0.366268515586853,
+      "sampling/sampling_logp_difference/max": 1.0043885707855225,
+      "sampling/sampling_logp_difference/mean": 0.013199402019381523,
+      "step": 35,
+      "step_time": 93.34705402800137
+    },
+    {
+      "clip_ratio/high_max": 0.0200398035813123,
+      "clip_ratio/high_mean": 0.01071434619370848,
+      "clip_ratio/low_mean": 0.010386000329162925,
+      "clip_ratio/low_min": 0.0007022471982054412,
+      "clip_ratio/region_mean": 0.021100346348248422,
+      "entropy": 0.11280779354274273,
+      "epoch": 0.00036,
+      "grad_norm": 0.8509212732315063,
+      "kl": 0.24738536309450865,
+      "learning_rate": 0.0001,
+      "loss": 0.1443,
+      "step": 36,
+      "step_time": 33.26461400700191
+    },
+    {
+      "clip_ratio/high_max": 0.0045615925919264555,
+      "clip_ratio/high_mean": 0.002661893842741847,
+      "clip_ratio/low_mean": 0.001787908433470875,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004449802276212722,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10712.0,
+      "completions/max_terminated_length": 10712.0,
+      "completions/mean_length": 7822.9375,
+      "completions/mean_terminated_length": 7822.9375,
+      "completions/min_length": 1061.0,
+      "completions/min_terminated_length": 1061.0,
+      "entropy": 0.09347328078001738,
+      "epoch": 0.00037,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7398293018341064,
+      "kl": 0.14611079543828964,
+      "learning_rate": 9.918053502751771e-05,
+      "loss": 0.0539,
+      "num_tokens": 5604207.0,
+      "reward": 0.2784110903739929,
+      "reward_std": 0.6512280702590942,
+      "rewards/rollout_eval_reward_func/mean": 0.2896341383457184,
+      "rewards/rollout_eval_reward_func/std": 0.32505732774734497,
+      "rewards/rollout_reward_func/mean": 0.2784110903739929,
+      "rewards/rollout_reward_func/std": 0.6340307593345642,
+      "sampling/importance_sampling_ratio/max": 1.785902500152588,
+      "sampling/importance_sampling_ratio/mean": 0.9995973110198975,
+      "sampling/importance_sampling_ratio/min": 0.3934570252895355,
+      "sampling/sampling_logp_difference/max": 0.9327834844589233,
+      "sampling/sampling_logp_difference/mean": 0.01090280432254076,
+      "step": 37,
+      "step_time": 89.90687796200018
+    },
+    {
+      "clip_ratio/high_max": 0.014242319972254336,
+      "clip_ratio/high_mean": 0.007633455039467663,
+      "clip_ratio/low_mean": 0.009654591587604955,
+      "clip_ratio/low_min": 0.001453684933949262,
+      "clip_ratio/region_mean": 0.017288046510657296,
+      "entropy": 0.09478882700204849,
+      "epoch": 0.00038,
+      "grad_norm": 0.5018179416656494,
+      "kl": 0.15739013627171516,
+      "learning_rate": 9.675795466159755e-05,
+      "loss": 0.0474,
+      "step": 38,
+      "step_time": 32.40730591600277
+    },
+    {
+      "clip_ratio/high_max": 0.0033184904023073614,
+      "clip_ratio/high_mean": 0.0016592452011536807,
+      "clip_ratio/low_mean": 0.002118892065482214,
+      "clip_ratio/low_min": 0.0007102272938936949,
+      "clip_ratio/region_mean": 0.0037781372957397252,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10878.0,
+      "completions/max_terminated_length": 10878.0,
+      "completions/mean_length": 8825.25,
+      "completions/mean_terminated_length": 8825.25,
+      "completions/min_length": 2113.0,
+      "completions/min_terminated_length": 2113.0,
+      "entropy": 0.13806713884696364,
+      "epoch": 0.00039,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.113547444343567,
+      "kl": 0.20099136885255575,
+      "learning_rate": 9.283813728906054e-05,
+      "loss": 0.1486,
+      "num_tokens": 5915217.0,
+      "reward": 0.18237435817718506,
+      "reward_std": 0.5858109593391418,
+      "rewards/rollout_eval_reward_func/mean": 0.23246951401233673,
+      "rewards/rollout_eval_reward_func/std": 0.3161698281764984,
+      "rewards/rollout_reward_func/mean": 0.18237435817718506,
+      "rewards/rollout_reward_func/std": 0.6235412955284119,
+      "sampling/importance_sampling_ratio/max": 1.8600788116455078,
+      "sampling/importance_sampling_ratio/mean": 1.0009781122207642,
+      "sampling/importance_sampling_ratio/min": 0.6195499897003174,
+      "sampling/sampling_logp_difference/max": 0.6206188201904297,
+      "sampling/sampling_logp_difference/mean": 0.01361357793211937,
+      "step": 39,
+      "step_time": 95.42290947900165
+    },
+    {
+      "clip_ratio/high_max": 0.01989291445352137,
+      "clip_ratio/high_mean": 0.010865762596949935,
+      "clip_ratio/low_mean": 0.01921083332854323,
+      "clip_ratio/low_min": 0.004245770338457078,
+      "clip_ratio/region_mean": 0.030076596303842962,
+      "entropy": 0.13848060183227062,
+      "epoch": 0.0004,
+      "grad_norm": 0.8446061611175537,
+      "kl": 0.22338007856160402,
+      "learning_rate": 8.759239773845718e-05,
+      "loss": 0.1364,
+      "step": 40,
+      "step_time": 33.852559433999886
+    },
+    {
+      "clip_ratio/high_max": 0.0030503973248414695,
+      "clip_ratio/high_mean": 0.0015251986624207348,
+      "clip_ratio/low_mean": 0.0010654326470103115,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0025906313094310462,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11107.0,
+      "completions/max_terminated_length": 11107.0,
+      "completions/mean_length": 8257.21875,
+      "completions/mean_terminated_length": 8257.21875,
+      "completions/min_length": 2666.0,
+      "completions/min_terminated_length": 2666.0,
+      "entropy": 0.11340034916065633,
+      "epoch": 0.00041,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0817996263504028,
+      "kl": 0.20612840354442596,
+      "learning_rate": 8.125000000000001e-05,
+      "loss": 0.3665,
+      "num_tokens": 6207607.0,
+      "reward": 0.42999449372291565,
+      "reward_std": 0.5903699994087219,
+      "rewards/rollout_eval_reward_func/mean": 0.33269816637039185,
+      "rewards/rollout_eval_reward_func/std": 0.343418687582016,
+      "rewards/rollout_reward_func/mean": 0.42999449372291565,
+      "rewards/rollout_reward_func/std": 0.5738593935966492,
+      "sampling/importance_sampling_ratio/max": 1.4319037199020386,
+      "sampling/importance_sampling_ratio/mean": 0.9990136027336121,
+      "sampling/importance_sampling_ratio/min": 0.4935825765132904,
+      "sampling/sampling_logp_difference/max": 0.7060651779174805,
+      "sampling/sampling_logp_difference/mean": 0.011390198953449726,
+      "step": 41,
+      "step_time": 91.59171992200027
+    },
+    {
+      "clip_ratio/high_max": 0.01730476424563676,
+      "clip_ratio/high_mean": 0.009731545054819435,
+      "clip_ratio/low_mean": 0.01579960942035541,
+      "clip_ratio/low_min": 0.006750052853021771,
+      "clip_ratio/region_mean": 0.025531154591590166,
+      "entropy": 0.11443871166557074,
+      "epoch": 0.00042,
+      "grad_norm": 0.7992480397224426,
+      "kl": 0.27106572315096855,
+      "learning_rate": 7.408813728906053e-05,
+      "loss": 0.3537,
+      "step": 42,
+      "step_time": 33.60049366600106
+    },
+    {
+      "clip_ratio/high_max": 0.0035054103354923427,
+      "clip_ratio/high_mean": 0.0017527051677461714,
+      "clip_ratio/low_mean": 0.0010694226948544383,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0028221279208082706,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10896.0,
+      "completions/max_terminated_length": 10896.0,
+      "completions/mean_length": 7894.15625,
+      "completions/mean_terminated_length": 7894.15625,
+      "completions/min_length": 2890.0,
+      "completions/min_terminated_length": 2890.0,
+      "entropy": 0.13558923918753862,
+      "epoch": 0.00043,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.9851862192153931,
+      "kl": 0.24445611471310258,
+      "learning_rate": 6.6419817372537e-05,
+      "loss": 0.1226,
+      "num_tokens": 6488287.0,
+      "reward": 0.18586361408233643,
+      "reward_std": 0.6816079616546631,
+      "rewards/rollout_eval_reward_func/mean": 0.30538618564605713,
+      "rewards/rollout_eval_reward_func/std": 0.32574331760406494,
+      "rewards/rollout_reward_func/mean": 0.18586361408233643,
+      "rewards/rollout_reward_func/std": 0.676216721534729,
+      "sampling/importance_sampling_ratio/max": 1.8440579175949097,
+      "sampling/importance_sampling_ratio/mean": 1.0000970363616943,
+      "sampling/importance_sampling_ratio/min": 0.6517351269721985,
+      "sampling/sampling_logp_difference/max": 0.6119685173034668,
+      "sampling/sampling_logp_difference/mean": 0.012094310484826565,
+      "step": 43,
+      "step_time": 92.82438364300197
+    },
+    {
+      "clip_ratio/high_max": 0.010269680351484567,
+      "clip_ratio/high_mean": 0.005821653438033536,
+      "clip_ratio/low_mean": 0.015070577035658062,
+      "clip_ratio/low_min": 0.0048273742431774735,
+      "clip_ratio/region_mean": 0.020892230415483937,
+      "entropy": 0.13336073094978929,
+      "epoch": 0.00044,
+      "grad_norm": 0.6729846000671387,
+      "kl": 0.2622186541557312,
+      "learning_rate": 5.858018262746301e-05,
+      "loss": 0.1142,
+      "step": 44,
+      "step_time": 32.630656069
+    },
+    {
+      "clip_ratio/high_max": 0.006278776447288692,
+      "clip_ratio/high_mean": 0.003139388223644346,
+      "clip_ratio/low_mean": 0.0014205463230609894,
+      "clip_ratio/low_min": 0.0007022471982054412,
+      "clip_ratio/region_mean": 0.004559934546705335,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11255.0,
+      "completions/max_terminated_length": 11255.0,
+      "completions/mean_length": 7849.59375,
+      "completions/mean_terminated_length": 7849.59375,
+      "completions/min_length": 841.0,
+      "completions/min_terminated_length": 841.0,
+      "entropy": 0.13749901531264186,
+      "epoch": 0.00045,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.1264039278030396,
+      "kl": 0.22676664125174284,
+      "learning_rate": 5.0911862710939485e-05,
+      "loss": -0.0333,
+      "num_tokens": 6767906.0,
+      "reward": 0.15443384647369385,
+      "reward_std": 0.693654477596283,
+      "rewards/rollout_eval_reward_func/mean": 0.2919207215309143,
+      "rewards/rollout_eval_reward_func/std": 0.29990869760513306,
+      "rewards/rollout_reward_func/mean": 0.15443384647369385,
+      "rewards/rollout_reward_func/std": 0.7213361263275146,
+      "sampling/importance_sampling_ratio/max": 1.6071842908859253,
+      "sampling/importance_sampling_ratio/mean": 0.9978848099708557,
+      "sampling/importance_sampling_ratio/min": 0.48193788528442383,
+      "sampling/sampling_logp_difference/max": 0.7299400568008423,
+      "sampling/sampling_logp_difference/mean": 0.01217048242688179,
+      "step": 45,
+      "step_time": 91.51718380699913
+    },
+    {
+      "clip_ratio/high_max": 0.012777922616805881,
+      "clip_ratio/high_mean": 0.006918622384546325,
+      "clip_ratio/low_mean": 0.014222246245481074,
+      "clip_ratio/low_min": 0.005445442744530737,
+      "clip_ratio/region_mean": 0.0211408686300274,
+      "entropy": 0.1343677118420601,
+      "epoch": 0.00046,
+      "grad_norm": 0.7688785791397095,
+      "kl": 0.24824338778853416,
+      "learning_rate": 4.375000000000001e-05,
+      "loss": -0.0418,
+      "step": 46,
+      "step_time": 33.231114288000754
+    },
+    {
+      "clip_ratio/high_max": 0.002997979463543743,
+      "clip_ratio/high_mean": 0.0018581851327326149,
+      "clip_ratio/low_mean": 0.0012389520416036248,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0030971371743362397,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10980.0,
+      "completions/max_terminated_length": 10980.0,
+      "completions/mean_length": 7450.6875,
+      "completions/mean_terminated_length": 7450.6875,
+      "completions/min_length": 320.0,
+      "completions/min_terminated_length": 320.0,
+      "entropy": 0.11971927247941494,
+      "epoch": 0.00047,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0886417627334595,
+      "kl": 0.27644640300422907,
+      "learning_rate": 3.740760226154283e-05,
+      "loss": 0.0684,
+      "num_tokens": 7034802.0,
+      "reward": 0.32516512274742126,
+      "reward_std": 0.6862237453460693,
+      "rewards/rollout_eval_reward_func/mean": 0.3408282399177551,
+      "rewards/rollout_eval_reward_func/std": 0.31514763832092285,
+      "rewards/rollout_reward_func/mean": 0.32516512274742126,
+      "rewards/rollout_reward_func/std": 0.6982549428939819,
+      "sampling/importance_sampling_ratio/max": 1.4899945259094238,
+      "sampling/importance_sampling_ratio/mean": 0.9988203048706055,
+      "sampling/importance_sampling_ratio/min": 0.5982000827789307,
+      "sampling/sampling_logp_difference/max": 0.5138299465179443,
+      "sampling/sampling_logp_difference/mean": 0.010367423295974731,
+      "step": 47,
+      "step_time": 92.68640525500086
+    },
+    {
+      "clip_ratio/high_max": 0.005615144735202193,
+      "clip_ratio/high_mean": 0.0031981973443180323,
+      "clip_ratio/low_mean": 0.008041755587328225,
+      "clip_ratio/low_min": 0.0007102272938936949,
+      "clip_ratio/region_mean": 0.011239952960750088,
+      "entropy": 0.11511751124635339,
+      "epoch": 0.00048,
+      "grad_norm": 0.677844762802124,
+      "kl": 0.2775820675306022,
+      "learning_rate": 3.2161862710939476e-05,
+      "loss": 0.0609,
+      "step": 48,
+      "step_time": 32.67402728900106
+    },
+    {
+      "clip_ratio/high_max": 0.00474693434080109,
+      "clip_ratio/high_mean": 0.002373467170400545,
+      "clip_ratio/low_mean": 0.0003511235991027206,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0027245907695032656,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11070.0,
+      "completions/max_terminated_length": 11070.0,
+      "completions/mean_length": 7308.03125,
+      "completions/mean_terminated_length": 7308.03125,
+      "completions/min_length": 854.0,
+      "completions/min_terminated_length": 854.0,
+      "entropy": 0.10222014435566962,
+      "epoch": 0.00049,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8171713948249817,
+      "kl": 0.22370027378201485,
+      "learning_rate": 2.8242045338402468e-05,
+      "loss": -0.068,
+      "num_tokens": 7296677.0,
+      "reward": 0.28688520193099976,
+      "reward_std": 0.7849241495132446,
+      "rewards/rollout_eval_reward_func/mean": 0.37195122241973877,
+      "rewards/rollout_eval_reward_func/std": 0.3127909302711487,
+      "rewards/rollout_reward_func/mean": 0.28688520193099976,
+      "rewards/rollout_reward_func/std": 0.7561083436012268,
+      "sampling/importance_sampling_ratio/max": 1.7672066688537598,
+      "sampling/importance_sampling_ratio/mean": 1.0014787912368774,
+      "sampling/importance_sampling_ratio/min": 0.5275546908378601,
+      "sampling/sampling_logp_difference/max": 0.639502763748169,
+      "sampling/sampling_logp_difference/mean": 0.011029250919818878,
+      "step": 49,
+      "step_time": 86.99529032999908
+    },
+    {
+      "clip_ratio/high_max": 0.007674970023799688,
+      "clip_ratio/high_mean": 0.004594452795572579,
+      "clip_ratio/low_mean": 0.008248247409937903,
+      "clip_ratio/low_min": 0.0007183908019214869,
+      "clip_ratio/region_mean": 0.012842700205510482,
+      "entropy": 0.09808745188638568,
+      "epoch": 0.0005,
+      "grad_norm": 0.6178692579269409,
+      "kl": 0.23346312018111348,
+      "learning_rate": 2.5819464972482288e-05,
+      "loss": -0.0742,
+      "step": 50,
+      "step_time": 32.30095837099998
+    },
+    {
+      "epoch": 0.0005,
+      "eval_clip_ratio/high_max": 0.0,
+      "eval_clip_ratio/high_mean": 0.0,
+      "eval_clip_ratio/low_mean": 0.0,
+      "eval_clip_ratio/low_min": 0.0,
+      "eval_clip_ratio/region_mean": 0.0,
+      "eval_completions/clipped_ratio": 0.0,
+      "eval_completions/max_length": 10309.35,
+      "eval_completions/max_terminated_length": 10309.35,
+      "eval_completions/mean_length": 7614.075,
+      "eval_completions/mean_terminated_length": 7614.075,
+      "eval_completions/min_length": 4457.25,
+      "eval_completions/min_terminated_length": 4457.25,
+      "eval_entropy": 0.1146618027240038,
+      "eval_frac_reward_zero_std": 1.0,
+      "eval_kl": 0.2420900955796242,
+      "eval_loss": 0.00033982208697125316,
+      "eval_num_tokens": 7296677.0,
+      "eval_reward": 0.439066107571125,
+      "eval_reward_std": 0.0,
+      "eval_rewards/rollout_eval_reward_func/mean": 0.3934959270060062,
+      "eval_rewards/rollout_eval_reward_func/std": 0.30925278924405575,
+      "eval_rewards/rollout_reward_func/mean": 0.439066107571125,
+      "eval_rewards/rollout_reward_func/std": 0.643099270761013,
+      "eval_runtime": 164.9571,
+      "eval_samples_per_second": 0.061,
+      "eval_sampling/importance_sampling_ratio/max": 1.303001207113266,
+      "eval_sampling/importance_sampling_ratio/mean": 1.0004466891288757,
+      "eval_sampling/importance_sampling_ratio/min": 0.7182389438152313,
+      "eval_sampling/sampling_logp_difference/max": 0.3734502553939819,
+      "eval_sampling/sampling_logp_difference/mean": 0.009935753676109016,
+      "eval_steps_per_second": 0.018,
+      "step": 50
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 50,
+  "num_input_tokens_seen": 7296677,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6023a241ed9e13dc120558482fd59ad8483ff6a757259a53618a637b706d2707
+size 8145