Upload task output 1

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: None
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": null,
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f6259fcfe5c263daa2883554700f5c4a54a49f7950ff88798a47f1c6e8f2546
+size 239536272

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

loss.txt

@@ -0,0 +1 @@
+50,no_eval

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,1234 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.0005,
+  "eval_steps": 500,
+  "global_step": 50,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8966.0,
+      "completions/max_terminated_length": 8966.0,
+      "completions/mean_length": 7392.1875,
+      "completions/mean_terminated_length": 7392.1875,
+      "completions/min_length": 892.0,
+      "completions/min_terminated_length": 892.0,
+      "entropy": 0.11410779180005193,
+      "epoch": 1e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.795484185218811,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": -0.2193,
+      "num_tokens": 263588.0,
+      "reward": -0.3529798984527588,
+      "reward_std": 0.3798896074295044,
+      "rewards/rollout_reward_func/mean": -0.3529798984527588,
+      "rewards/rollout_reward_func/std": 0.4275760054588318,
+      "sampling/importance_sampling_ratio/max": 1.9381288290023804,
+      "sampling/importance_sampling_ratio/mean": 0.9994939565658569,
+      "sampling/importance_sampling_ratio/min": 0.48449021577835083,
+      "sampling/sampling_logp_difference/max": 0.7246580123901367,
+      "sampling/sampling_logp_difference/mean": 0.013468129560351372,
+      "step": 1,
+      "step_time": 93.399304126
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "entropy": 0.11410779180005193,
+      "epoch": 2e-05,
+      "grad_norm": 1.7967665195465088,
+      "kl": 0.0,
+      "learning_rate": 2.8571428571428573e-06,
+      "loss": -0.2193,
+      "step": 2,
+      "step_time": 27.58208410200001
+    },
+    {
+      "clip_ratio/high_max": 0.002206696954090148,
+      "clip_ratio/high_mean": 0.001103348477045074,
+      "clip_ratio/low_mean": 0.0021773561311420053,
+      "clip_ratio/low_min": 0.0007183908019214869,
+      "clip_ratio/region_mean": 0.0032807046081870794,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8789.0,
+      "completions/max_terminated_length": 8789.0,
+      "completions/mean_length": 7273.3125,
+      "completions/mean_terminated_length": 7273.3125,
+      "completions/min_length": 1689.0,
+      "completions/min_terminated_length": 1689.0,
+      "entropy": 0.08583545382134616,
+      "epoch": 3e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.1726843118667603,
+      "kl": 0.0017027981803039438,
+      "learning_rate": 5.7142857142857145e-06,
+      "loss": -0.0032,
+      "num_tokens": 523711.0,
+      "reward": -0.3553134799003601,
+      "reward_std": 0.48743048310279846,
+      "rewards/rollout_reward_func/mean": -0.3553134799003601,
+      "rewards/rollout_reward_func/std": 0.5032535195350647,
+      "sampling/importance_sampling_ratio/max": 2.636791944503784,
+      "sampling/importance_sampling_ratio/mean": 1.0005749464035034,
+      "sampling/importance_sampling_ratio/min": 0.6534329652786255,
+      "sampling/sampling_logp_difference/max": 0.9695630073547363,
+      "sampling/sampling_logp_difference/mean": 0.008272429928183556,
+      "step": 3,
+      "step_time": 91.38279614700019
+    },
+    {
+      "clip_ratio/high_max": 0.00520685565425083,
+      "clip_ratio/high_mean": 0.002603427827125415,
+      "clip_ratio/low_mean": 0.0017682212928775698,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004371649120002985,
+      "entropy": 0.08481810381636024,
+      "epoch": 4e-05,
+      "grad_norm": 2.2690632343292236,
+      "kl": 0.0009417296496394556,
+      "learning_rate": 8.571428571428573e-06,
+      "loss": -0.0051,
+      "step": 4,
+      "step_time": 26.98271870500014
+    },
+    {
+      "clip_ratio/high_max": 0.004254478495568037,
+      "clip_ratio/high_mean": 0.0021272392477840185,
+      "clip_ratio/low_mean": 0.002160204341635108,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0042874435894191265,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8986.0,
+      "completions/max_terminated_length": 8986.0,
+      "completions/mean_length": 7767.59375,
+      "completions/mean_terminated_length": 7767.59375,
+      "completions/min_length": 300.0,
+      "completions/min_terminated_length": 300.0,
+      "entropy": 0.10082893958315253,
+      "epoch": 5e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3848886489868164,
+      "kl": 0.0014980694850237342,
+      "learning_rate": 1.1428571428571429e-05,
+      "loss": -0.1089,
+      "num_tokens": 799123.0,
+      "reward": -0.4331575036048889,
+      "reward_std": 0.3397783935070038,
+      "rewards/rollout_reward_func/mean": -0.4331575036048889,
+      "rewards/rollout_reward_func/std": 0.37834790349006653,
+      "sampling/importance_sampling_ratio/max": 1.6952898502349854,
+      "sampling/importance_sampling_ratio/mean": 0.9987826347351074,
+      "sampling/importance_sampling_ratio/min": 0.2250514030456543,
+      "sampling/sampling_logp_difference/max": 1.4914264678955078,
+      "sampling/sampling_logp_difference/mean": 0.010691944509744644,
+      "step": 5,
+      "step_time": 94.93945038100003
+    },
+    {
+      "clip_ratio/high_max": 0.006353594711981714,
+      "clip_ratio/high_mean": 0.003176797355990857,
+      "clip_ratio/low_mean": 0.0018021696887444705,
+      "clip_ratio/low_min": 0.0007183908019214869,
+      "clip_ratio/region_mean": 0.004978967044735327,
+      "entropy": 0.10040339734405279,
+      "epoch": 6e-05,
+      "grad_norm": 1.3835108280181885,
+      "kl": 0.00129023771660286,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": -0.1114,
+      "step": 6,
+      "step_time": 27.51428913800021
+    },
+    {
+      "clip_ratio/high_max": 0.007119529938790947,
+      "clip_ratio/high_mean": 0.0035597649693954736,
+      "clip_ratio/low_mean": 0.0018344622512813658,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0053942272788845,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9295.0,
+      "completions/max_terminated_length": 9295.0,
+      "completions/mean_length": 8119.90625,
+      "completions/mean_terminated_length": 8119.90625,
+      "completions/min_length": 425.0,
+      "completions/min_terminated_length": 425.0,
+      "entropy": 0.08391103229951113,
+      "epoch": 7e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.6203776597976685,
+      "kl": 0.0014097213361310423,
+      "learning_rate": 1.7142857142857145e-05,
+      "loss": -0.1002,
+      "num_tokens": 1086692.0,
+      "reward": -0.5209137201309204,
+      "reward_std": 0.2494075447320938,
+      "rewards/rollout_reward_func/mean": -0.5209137201309204,
+      "rewards/rollout_reward_func/std": 0.2681976854801178,
+      "sampling/importance_sampling_ratio/max": 1.7586421966552734,
+      "sampling/importance_sampling_ratio/mean": 1.0001193284988403,
+      "sampling/importance_sampling_ratio/min": 0.5591319799423218,
+      "sampling/sampling_logp_difference/max": 0.5813696980476379,
+      "sampling/sampling_logp_difference/mean": 0.010238477028906345,
+      "step": 7,
+      "step_time": 99.45176190400002
+    },
+    {
+      "clip_ratio/high_max": 0.0050131616299040616,
+      "clip_ratio/high_mean": 0.0025065808149520308,
+      "clip_ratio/low_mean": 0.0048430965398438275,
+      "clip_ratio/low_min": 0.0014367816038429737,
+      "clip_ratio/region_mean": 0.00734967747121118,
+      "entropy": 0.08137266687117517,
+      "epoch": 8e-05,
+      "grad_norm": 1.417209267616272,
+      "kl": 0.0033996296788245672,
+      "learning_rate": 2e-05,
+      "loss": -0.1028,
+      "step": 8,
+      "step_time": 29.014337608000233
+    },
+    {
+      "clip_ratio/high_max": 0.0042058578110300004,
+      "clip_ratio/high_mean": 0.0024501511361449957,
+      "clip_ratio/low_mean": 0.0024779988452792168,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0049281499814242125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9264.0,
+      "completions/max_terminated_length": 9264.0,
+      "completions/mean_length": 7546.03125,
+      "completions/mean_terminated_length": 7546.03125,
+      "completions/min_length": 860.0,
+      "completions/min_terminated_length": 860.0,
+      "entropy": 0.08567450987175107,
+      "epoch": 9e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.5940141677856445,
+      "kl": 0.010301298352715094,
+      "learning_rate": 2.2857142857142858e-05,
+      "loss": -0.3628,
+      "num_tokens": 1355432.0,
+      "reward": -0.40637093782424927,
+      "reward_std": 0.3687950074672699,
+      "rewards/rollout_reward_func/mean": -0.40637093782424927,
+      "rewards/rollout_reward_func/std": 0.3848126232624054,
+      "sampling/importance_sampling_ratio/max": 1.819852352142334,
+      "sampling/importance_sampling_ratio/mean": 1.0000147819519043,
+      "sampling/importance_sampling_ratio/min": 0.4127380847930908,
+      "sampling/sampling_logp_difference/max": 0.8849420547485352,
+      "sampling/sampling_logp_difference/mean": 0.01255122572183609,
+      "step": 9,
+      "step_time": 92.58362530700037
+    },
+    {
+      "clip_ratio/high_max": 0.010624476184602827,
+      "clip_ratio/high_mean": 0.006026733753969893,
+      "clip_ratio/low_mean": 0.01009578033699654,
+      "clip_ratio/low_min": 0.0042697106837295,
+      "clip_ratio/region_mean": 0.01612251391634345,
+      "entropy": 0.08229666040278971,
+      "epoch": 0.0001,
+      "grad_norm": 1.487949252128601,
+      "kl": 0.037161100313824136,
+      "learning_rate": 2.5714285714285714e-05,
+      "loss": -0.3665,
+      "step": 10,
+      "step_time": 28.38116288400022
+    },
+    {
+      "clip_ratio/high_max": 0.0049977844464592636,
+      "clip_ratio/high_mean": 0.0024988922232296318,
+      "clip_ratio/low_mean": 0.0011143402953166515,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0036132325185462832,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9244.0,
+      "completions/max_terminated_length": 9244.0,
+      "completions/mean_length": 7200.8125,
+      "completions/mean_terminated_length": 7200.8125,
+      "completions/min_length": 297.0,
+      "completions/min_terminated_length": 297.0,
+      "entropy": 0.06809334014542401,
+      "epoch": 0.00011,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.8031041622161865,
+      "kl": 0.015580801184114534,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": -0.3986,
+      "num_tokens": 1613108.0,
+      "reward": -0.4156220555305481,
+      "reward_std": 0.5291995406150818,
+      "rewards/rollout_reward_func/mean": -0.4156220555305481,
+      "rewards/rollout_reward_func/std": 0.5197964906692505,
+      "sampling/importance_sampling_ratio/max": 2.4048261642456055,
+      "sampling/importance_sampling_ratio/mean": 0.9991490244865417,
+      "sampling/importance_sampling_ratio/min": 0.2525537610054016,
+      "sampling/sampling_logp_difference/max": 1.3761311769485474,
+      "sampling/sampling_logp_difference/mean": 0.009278691373765469,
+      "step": 11,
+      "step_time": 91.42414895300033
+    },
+    {
+      "clip_ratio/high_max": 0.00853408733382821,
+      "clip_ratio/high_mean": 0.004985434468835592,
+      "clip_ratio/low_mean": 0.006528367113787681,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.011513801524415612,
+      "entropy": 0.06515359785407782,
+      "epoch": 0.00012,
+      "grad_norm": 0.9123954176902771,
+      "kl": 0.05925005234166747,
+      "learning_rate": 3.142857142857143e-05,
+      "loss": -0.4041,
+      "step": 12,
+      "step_time": 27.261401052000338
+    },
+    {
+      "clip_ratio/high_max": 0.004881663771811873,
+      "clip_ratio/high_mean": 0.0027880541456397623,
+      "clip_ratio/low_mean": 0.0037891828105784953,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0065772370144259185,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9011.0,
+      "completions/max_terminated_length": 9011.0,
+      "completions/mean_length": 7481.90625,
+      "completions/mean_terminated_length": 7481.90625,
+      "completions/min_length": 2417.0,
+      "completions/min_terminated_length": 2417.0,
+      "entropy": 0.06863200129009783,
+      "epoch": 0.00013,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.003671407699585,
+      "kl": 0.039789786795154214,
+      "learning_rate": 3.428571428571429e-05,
+      "loss": -0.203,
+      "num_tokens": 1880242.0,
+      "reward": -0.5097812414169312,
+      "reward_std": 0.4184800386428833,
+      "rewards/rollout_reward_func/mean": -0.5097812414169312,
+      "rewards/rollout_reward_func/std": 0.4387035071849823,
+      "sampling/importance_sampling_ratio/max": 3.0,
+      "sampling/importance_sampling_ratio/mean": 1.0012085437774658,
+      "sampling/importance_sampling_ratio/min": 0.21886371076107025,
+      "sampling/sampling_logp_difference/max": 2.316737651824951,
+      "sampling/sampling_logp_difference/mean": 0.01692187413573265,
+      "step": 13,
+      "step_time": 93.44268519900038
+    },
+    {
+      "clip_ratio/high_max": 0.012604236777406186,
+      "clip_ratio/high_mean": 0.006302118388703093,
+      "clip_ratio/low_mean": 0.01540282418136485,
+      "clip_ratio/low_min": 0.003511236049234867,
+      "clip_ratio/region_mean": 0.021704942628275603,
+      "entropy": 0.06422114372253418,
+      "epoch": 0.00014,
+      "grad_norm": 1.1214606761932373,
+      "kl": 0.07548845067503862,
+      "learning_rate": 3.7142857142857143e-05,
+      "loss": -0.2136,
+      "step": 14,
+      "step_time": 26.921327204999443
+    },
+    {
+      "clip_ratio/high_max": 0.003742929140571505,
+      "clip_ratio/high_mean": 0.0022186867718119174,
+      "clip_ratio/low_mean": 0.0041773931588977575,
+      "clip_ratio/low_min": 0.0014451349852606654,
+      "clip_ratio/region_mean": 0.0063960799016058445,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9200.0,
+      "completions/max_terminated_length": 9200.0,
+      "completions/mean_length": 7406.59375,
+      "completions/mean_terminated_length": 7406.59375,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
+      "entropy": 0.046332313562743366,
+      "epoch": 0.00015,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3658106327056885,
+      "kl": 0.08860734064364806,
+      "learning_rate": 4e-05,
+      "loss": 0.0669,
+      "num_tokens": 2144669.0,
+      "reward": -0.3528895974159241,
+      "reward_std": 0.443051815032959,
+      "rewards/rollout_reward_func/mean": -0.3528895974159241,
+      "rewards/rollout_reward_func/std": 0.48428139090538025,
+      "sampling/importance_sampling_ratio/max": 1.8482800722122192,
+      "sampling/importance_sampling_ratio/mean": 0.9966132640838623,
+      "sampling/importance_sampling_ratio/min": 0.118045873939991,
+      "sampling/sampling_logp_difference/max": 2.1366820335388184,
+      "sampling/sampling_logp_difference/mean": 0.016083233058452606,
+      "step": 15,
+      "step_time": 91.83712665100029
+    },
+    {
+      "clip_ratio/high_max": 0.004916455829516053,
+      "clip_ratio/high_mean": 0.0024582279147580266,
+      "clip_ratio/low_mean": 0.008916221850086004,
+      "clip_ratio/low_min": 0.0021388453897088766,
+      "clip_ratio/region_mean": 0.011374449823051691,
+      "entropy": 0.04555429570609704,
+      "epoch": 0.00016,
+      "grad_norm": 1.3114889860153198,
+      "kl": 0.1314735203923192,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 0.0618,
+      "step": 16,
+      "step_time": 27.011356363000004
+    },
+    {
+      "clip_ratio/high_max": 0.008057098719291389,
+      "clip_ratio/high_mean": 0.00437577159027569,
+      "clip_ratio/low_mean": 0.004278601991245523,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.008654373581521213,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9228.0,
+      "completions/max_terminated_length": 9228.0,
+      "completions/mean_length": 7819.875,
+      "completions/mean_terminated_length": 7819.875,
+      "completions/min_length": 3760.0,
+      "completions/min_terminated_length": 3760.0,
+      "entropy": 0.07362106395885348,
+      "epoch": 0.00017,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.2947819232940674,
+      "kl": 0.13101027125958353,
+      "learning_rate": 4.5714285714285716e-05,
+      "loss": -0.0547,
+      "num_tokens": 2422510.0,
+      "reward": -0.35005444288253784,
+      "reward_std": 0.49921366572380066,
+      "rewards/rollout_reward_func/mean": -0.35005444288253784,
+      "rewards/rollout_reward_func/std": 0.522201657295227,
+      "sampling/importance_sampling_ratio/max": 3.0,
+      "sampling/importance_sampling_ratio/mean": 1.0008654594421387,
+      "sampling/importance_sampling_ratio/min": 0.04268655180931091,
+      "sampling/sampling_logp_difference/max": 3.1538712978363037,
+      "sampling/sampling_logp_difference/mean": 0.027018871158361435,
+      "step": 17,
+      "step_time": 96.36866129600003
+    },
+    {
+      "clip_ratio/high_max": 0.011791770870331675,
+      "clip_ratio/high_mean": 0.0066107546736020595,
+      "clip_ratio/low_mean": 0.01208667631726712,
+      "clip_ratio/low_min": 0.0028746770112775266,
+      "clip_ratio/region_mean": 0.01869743096176535,
+      "entropy": 0.0709565794095397,
+      "epoch": 0.00018,
+      "grad_norm": 3.9747724533081055,
+      "kl": 0.45611770218238235,
+      "learning_rate": 4.8571428571428576e-05,
+      "loss": -0.0527,
+      "step": 18,
+      "step_time": 27.170892336000406
+    },
+    {
+      "clip_ratio/high_max": 0.006368288770318031,
+      "clip_ratio/high_mean": 0.0031841443851590157,
+      "clip_ratio/low_mean": 0.0007183908019214869,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0039025351288728416,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9443.0,
+      "completions/max_terminated_length": 9443.0,
+      "completions/mean_length": 7551.3125,
+      "completions/mean_terminated_length": 7551.3125,
+      "completions/min_length": 2300.0,
+      "completions/min_terminated_length": 2300.0,
+      "entropy": 0.05112030985765159,
+      "epoch": 0.00019,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.3253093957901,
+      "kl": 0.06368285935604945,
+      "learning_rate": 5.142857142857143e-05,
+      "loss": -0.1504,
+      "num_tokens": 2691652.0,
+      "reward": -0.4411429762840271,
+      "reward_std": 0.4299367666244507,
+      "rewards/rollout_reward_func/mean": -0.4411429762840271,
+      "rewards/rollout_reward_func/std": 0.4561052918434143,
+      "sampling/importance_sampling_ratio/max": 1.6581709384918213,
+      "sampling/importance_sampling_ratio/mean": 0.997830867767334,
+      "sampling/importance_sampling_ratio/min": 0.07465386390686035,
+      "sampling/sampling_logp_difference/max": 2.594892978668213,
+      "sampling/sampling_logp_difference/mean": 0.013442842289805412,
+      "step": 19,
+      "step_time": 93.42405358899964
+    },
+    {
+      "clip_ratio/high_max": 0.01606024947250262,
+      "clip_ratio/high_mean": 0.008740443765418604,
+      "clip_ratio/low_mean": 0.006419806071789935,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.015160249720793217,
+      "entropy": 0.053090699831955135,
+      "epoch": 0.0002,
+      "grad_norm": 1.2332507371902466,
+      "kl": 0.1118333391204942,
+      "learning_rate": 5.428571428571428e-05,
+      "loss": -0.156,
+      "step": 20,
+      "step_time": 27.701954217999173
+    },
+    {
+      "clip_ratio/high_max": 0.005651562649291009,
+      "clip_ratio/high_mean": 0.0028257813246455044,
+      "clip_ratio/low_mean": 0.0027939277642872185,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005619709118036553,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9189.0,
+      "completions/max_terminated_length": 9189.0,
+      "completions/mean_length": 7761.09375,
+      "completions/mean_terminated_length": 7761.09375,
+      "completions/min_length": 1417.0,
+      "completions/min_terminated_length": 1417.0,
+      "entropy": 0.06741820118622854,
+      "epoch": 0.00021,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.6862173080444336,
+      "kl": 0.4830274658452254,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": -0.2263,
+      "num_tokens": 2967426.0,
+      "reward": -0.28399354219436646,
+      "reward_std": 0.4300430417060852,
+      "rewards/rollout_reward_func/mean": -0.28399354219436646,
+      "rewards/rollout_reward_func/std": 0.5347884297370911,
+      "sampling/importance_sampling_ratio/max": 2.1812915802001953,
+      "sampling/importance_sampling_ratio/mean": 0.997977614402771,
+      "sampling/importance_sampling_ratio/min": 0.4058607518672943,
+      "sampling/sampling_logp_difference/max": 0.9017452001571655,
+      "sampling/sampling_logp_difference/mean": 0.014318476431071758,
+      "step": 21,
+      "step_time": 94.09382517800077
+    },
+    {
+      "clip_ratio/high_max": 0.009377924376167357,
+      "clip_ratio/high_mean": 0.006085653760237619,
+      "clip_ratio/low_mean": 0.012686202622717246,
+      "clip_ratio/low_min": 0.002905860950704664,
+      "clip_ratio/region_mean": 0.018771856353851035,
+      "entropy": 0.07064696005545557,
+      "epoch": 0.00022,
+      "grad_norm": 6.065097332000732,
+      "kl": 0.08845078520243987,
+      "learning_rate": 6e-05,
+      "loss": -0.2227,
+      "step": 22,
+      "step_time": 27.40954218999991
+    },
+    {
+      "clip_ratio/high_max": 0.0028251263429410756,
+      "clip_ratio/high_mean": 0.0017597853729967028,
+      "clip_ratio/low_mean": 0.0020223904284648597,
+      "clip_ratio/low_min": 0.0005252101109363139,
+      "clip_ratio/region_mean": 0.0037821758014615625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8721.0,
+      "completions/max_terminated_length": 8721.0,
+      "completions/mean_length": 7136.59375,
+      "completions/mean_terminated_length": 7136.59375,
+      "completions/min_length": 959.0,
+      "completions/min_terminated_length": 959.0,
+      "entropy": 0.06426455150358379,
+      "epoch": 0.00023,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.0993857383728027,
+      "kl": 0.1524171840865165,
+      "learning_rate": 6.285714285714286e-05,
+      "loss": -0.0482,
+      "num_tokens": 3222856.0,
+      "reward": -0.365644633769989,
+      "reward_std": 0.5254338979721069,
+      "rewards/rollout_reward_func/mean": -0.365644633769989,
+      "rewards/rollout_reward_func/std": 0.5246545076370239,
+      "sampling/importance_sampling_ratio/max": 2.320268392562866,
+      "sampling/importance_sampling_ratio/mean": 0.9996501803398132,
+      "sampling/importance_sampling_ratio/min": 0.09908817708492279,
+      "sampling/sampling_logp_difference/max": 2.3117451667785645,
+      "sampling/sampling_logp_difference/mean": 0.011567876674234867,
+      "step": 23,
+      "step_time": 89.85609540400014
+    },
+    {
+      "clip_ratio/high_max": 0.009038183547090739,
+      "clip_ratio/high_mean": 0.005584432714385912,
+      "clip_ratio/low_mean": 0.005201028659939766,
+      "clip_ratio/low_min": 0.0015668768319301307,
+      "clip_ratio/region_mean": 0.010785461287014186,
+      "entropy": 0.06729507446289062,
+      "epoch": 0.00024,
+      "grad_norm": 1.5627487897872925,
+      "kl": 0.1648054391844198,
+      "learning_rate": 6.571428571428571e-05,
+      "loss": -0.0548,
+      "step": 24,
+      "step_time": 25.88749120899979
+    },
+    {
+      "clip_ratio/high_max": 0.005618693074211478,
+      "clip_ratio/high_mean": 0.002809346537105739,
+      "clip_ratio/low_mean": 0.001929406455019489,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004738752992125228,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8856.0,
+      "completions/max_terminated_length": 8856.0,
+      "completions/mean_length": 7467.65625,
+      "completions/mean_terminated_length": 7467.65625,
+      "completions/min_length": 402.0,
+      "completions/min_terminated_length": 402.0,
+      "entropy": 0.07340177637524903,
+      "epoch": 0.00025,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.2772353887557983,
+      "kl": 0.0859682586742565,
+      "learning_rate": 6.857142857142858e-05,
+      "loss": -0.2314,
+      "num_tokens": 3488411.0,
+      "reward": -0.3830329477787018,
+      "reward_std": 0.5107574462890625,
+      "rewards/rollout_reward_func/mean": -0.3830329477787018,
+      "rewards/rollout_reward_func/std": 0.5173285007476807,
+      "sampling/importance_sampling_ratio/max": 2.1774182319641113,
+      "sampling/importance_sampling_ratio/mean": 0.999387264251709,
+      "sampling/importance_sampling_ratio/min": 0.29978206753730774,
+      "sampling/sampling_logp_difference/max": 1.2046995162963867,
+      "sampling/sampling_logp_difference/mean": 0.011983634904026985,
+      "step": 25,
+      "step_time": 92.61141428400015
+    },
+    {
+      "clip_ratio/high_max": 0.014493243885226548,
+      "clip_ratio/high_mean": 0.008288288547191769,
+      "clip_ratio/low_mean": 0.013486813055351377,
+      "clip_ratio/low_min": 0.0028249489841982722,
+      "clip_ratio/region_mean": 0.021775101457023993,
+      "entropy": 0.0781157671008259,
+      "epoch": 0.00026,
+      "grad_norm": 0.8266052603721619,
+      "kl": 0.08862257207510993,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": -0.2407,
+      "step": 26,
+      "step_time": 26.541104049000296
+    },
+    {
+      "clip_ratio/high_max": 0.0022112573496997356,
+      "clip_ratio/high_mean": 0.001456752244848758,
+      "clip_ratio/low_mean": 0.0013966916594654322,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00285344390431419,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9222.0,
+      "completions/max_terminated_length": 9222.0,
+      "completions/mean_length": 7990.125,
+      "completions/mean_terminated_length": 7990.125,
+      "completions/min_length": 2192.0,
+      "completions/min_terminated_length": 2192.0,
+      "entropy": 0.08974491525441408,
+      "epoch": 0.00027,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0755014419555664,
+      "kl": 0.08955412585055456,
+      "learning_rate": 7.428571428571429e-05,
+      "loss": -0.0612,
+      "num_tokens": 3770720.0,
+      "reward": -0.1771392822265625,
+      "reward_std": 0.4869571924209595,
+      "rewards/rollout_reward_func/mean": -0.1771392822265625,
+      "rewards/rollout_reward_func/std": 0.46993595361709595,
+      "sampling/importance_sampling_ratio/max": 1.7862199544906616,
+      "sampling/importance_sampling_ratio/mean": 1.0007758140563965,
+      "sampling/importance_sampling_ratio/min": 0.294488787651062,
+      "sampling/sampling_logp_difference/max": 1.2225143909454346,
+      "sampling/sampling_logp_difference/mean": 0.011052686721086502,
+      "step": 27,
+      "step_time": 95.3786270669998
+    },
+    {
+      "clip_ratio/high_max": 0.012634392420295626,
+      "clip_ratio/high_mean": 0.00806891280808486,
+      "clip_ratio/low_mean": 0.005300049553625286,
+      "clip_ratio/low_min": 0.0014046717551536858,
+      "clip_ratio/region_mean": 0.013368962419917807,
+      "entropy": 0.09402668941766024,
+      "epoch": 0.00028,
+      "grad_norm": 0.9924318790435791,
+      "kl": 0.0753595522546675,
+      "learning_rate": 7.714285714285715e-05,
+      "loss": -0.0693,
+      "step": 28,
+      "step_time": 27.647992973000328
+    },
+    {
+      "clip_ratio/high_max": 0.0035870341234840453,
+      "clip_ratio/high_mean": 0.002140739292372018,
+      "clip_ratio/low_mean": 0.0017440025694668293,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003884741832735017,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9067.0,
+      "completions/max_terminated_length": 9067.0,
+      "completions/mean_length": 7836.9375,
+      "completions/mean_terminated_length": 7836.9375,
+      "completions/min_length": 1928.0,
+      "completions/min_terminated_length": 1928.0,
+      "entropy": 0.09281386900693178,
+      "epoch": 0.00029,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.068334937095642,
+      "kl": 0.0688294384162873,
+      "learning_rate": 8e-05,
+      "loss": 0.0442,
+      "num_tokens": 4047979.0,
+      "reward": -0.25341448187828064,
+      "reward_std": 0.49919599294662476,
+      "rewards/rollout_reward_func/mean": -0.25341448187828064,
+      "rewards/rollout_reward_func/std": 0.500785231590271,
+      "sampling/importance_sampling_ratio/max": 1.868089199066162,
+      "sampling/importance_sampling_ratio/mean": 1.0002467632293701,
+      "sampling/importance_sampling_ratio/min": 0.5016597509384155,
+      "sampling/sampling_logp_difference/max": 0.6898331642150879,
+      "sampling/sampling_logp_difference/mean": 0.009339751675724983,
+      "step": 29,
+      "step_time": 94.94639882799993
+    },
+    {
+      "clip_ratio/high_max": 0.015361635130830109,
+      "clip_ratio/high_mean": 0.0076808175654150546,
+      "clip_ratio/low_mean": 0.007387197489151731,
+      "clip_ratio/low_min": 0.0006944444612599909,
+      "clip_ratio/region_mean": 0.015068015054566786,
+      "entropy": 0.09539854992181063,
+      "epoch": 0.0003,
+      "grad_norm": 0.7568414807319641,
+      "kl": 0.06834399094805121,
+      "learning_rate": 8.285714285714287e-05,
+      "loss": 0.0376,
+      "step": 30,
+      "step_time": 26.633041288999266
+    },
+    {
+      "clip_ratio/high_max": 0.004851312725804746,
+      "clip_ratio/high_mean": 0.002425656362902373,
+      "clip_ratio/low_mean": 0.003376906446646899,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00580256269313395,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9536.0,
+      "completions/max_terminated_length": 9536.0,
+      "completions/mean_length": 8069.3125,
+      "completions/mean_terminated_length": 8069.3125,
+      "completions/min_length": 2100.0,
+      "completions/min_terminated_length": 2100.0,
+      "entropy": 0.1009219279512763,
+      "epoch": 0.00031,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0075691938400269,
+      "kl": 0.06613797519821674,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 0.0875,
+      "num_tokens": 4332991.0,
+      "reward": -0.2427319884300232,
+      "reward_std": 0.5179601907730103,
+      "rewards/rollout_reward_func/mean": -0.2427319884300232,
+      "rewards/rollout_reward_func/std": 0.5765778422355652,
+      "sampling/importance_sampling_ratio/max": 1.7835460901260376,
+      "sampling/importance_sampling_ratio/mean": 0.9999290704727173,
+      "sampling/importance_sampling_ratio/min": 0.6331738829612732,
+      "sampling/sampling_logp_difference/max": 0.5786035060882568,
+      "sampling/sampling_logp_difference/mean": 0.009435810148715973,
+      "step": 31,
+      "step_time": 94.81605570200009
+    },
+    {
+      "clip_ratio/high_max": 0.017235137231182307,
+      "clip_ratio/high_mean": 0.008985215710708871,
+      "clip_ratio/low_mean": 0.012812130036763847,
+      "clip_ratio/low_min": 0.0027777778450399637,
+      "clip_ratio/region_mean": 0.021797345718368888,
+      "entropy": 0.10036895563825965,
+      "epoch": 0.00032,
+      "grad_norm": 0.7584834098815918,
+      "kl": 0.11454511061310768,
+      "learning_rate": 8.857142857142857e-05,
+      "loss": 0.0788,
+      "step": 32,
+      "step_time": 28.761855419000312
+    },
+    {
+      "clip_ratio/high_max": 0.0031494522700086236,
+      "clip_ratio/high_mean": 0.0015747261350043118,
+      "clip_ratio/low_mean": 0.0024462497676722705,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004020975931780413,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9354.0,
+      "completions/max_terminated_length": 9354.0,
+      "completions/mean_length": 7928.5,
+      "completions/mean_terminated_length": 7928.5,
+      "completions/min_length": 3150.0,
+      "completions/min_terminated_length": 3150.0,
+      "entropy": 0.1094990405254066,
+      "epoch": 0.00033,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8852567672729492,
+      "kl": 0.0793593015987426,
+      "learning_rate": 9.142857142857143e-05,
+      "loss": 0.1924,
+      "num_tokens": 4613150.0,
+      "reward": -0.1065119206905365,
+      "reward_std": 0.6632397174835205,
+      "rewards/rollout_reward_func/mean": -0.1065119206905365,
+      "rewards/rollout_reward_func/std": 0.6485887765884399,
+      "sampling/importance_sampling_ratio/max": 1.593437671661377,
+      "sampling/importance_sampling_ratio/mean": 0.9991821050643921,
+      "sampling/importance_sampling_ratio/min": 0.5760508179664612,
+      "sampling/sampling_logp_difference/max": 0.551559329032898,
+      "sampling/sampling_logp_difference/mean": 0.0090299928560853,
+      "step": 33,
+      "step_time": 94.56970139799978
+    },
+    {
+      "clip_ratio/high_max": 0.010831523861270398,
+      "clip_ratio/high_mean": 0.006442332087317482,
+      "clip_ratio/low_mean": 0.014884777745464817,
+      "clip_ratio/low_min": 0.004198966489639133,
+      "clip_ratio/region_mean": 0.02132710992009379,
+      "entropy": 0.1035804757848382,
+      "epoch": 0.00034,
+      "grad_norm": 6.932408332824707,
+      "kl": 0.36925772204995155,
+      "learning_rate": 9.428571428571429e-05,
+      "loss": 0.19,
+      "step": 34,
+      "step_time": 28.189770212999974
+    },
+    {
+      "clip_ratio/high_max": 0.004503837670199573,
+      "clip_ratio/high_mean": 0.0022519188350997865,
+      "clip_ratio/low_mean": 0.001957070722710341,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004208989586913958,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9253.0,
+      "completions/max_terminated_length": 9253.0,
+      "completions/mean_length": 8121.40625,
+      "completions/mean_terminated_length": 8121.40625,
+      "completions/min_length": 3211.0,
+      "completions/min_terminated_length": 3211.0,
+      "entropy": 0.08986494969576597,
+      "epoch": 0.00035,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0189476013183594,
+      "kl": 0.11070865276269615,
+      "learning_rate": 9.714285714285715e-05,
+      "loss": -0.1358,
+      "num_tokens": 4899944.0,
+      "reward": -0.3683607578277588,
+      "reward_std": 0.39094337821006775,
+      "rewards/rollout_reward_func/mean": -0.3683607578277588,
+      "rewards/rollout_reward_func/std": 0.3886735141277313,
+      "sampling/importance_sampling_ratio/max": 1.678568720817566,
+      "sampling/importance_sampling_ratio/mean": 0.9991096258163452,
+      "sampling/importance_sampling_ratio/min": 0.2759600579738617,
+      "sampling/sampling_logp_difference/max": 1.287499189376831,
+      "sampling/sampling_logp_difference/mean": 0.009727813303470612,
+      "step": 35,
+      "step_time": 95.49759846299912
+    },
+    {
+      "clip_ratio/high_max": 0.016388497198931873,
+      "clip_ratio/high_mean": 0.008888693118933588,
+      "clip_ratio/low_mean": 0.012551680905744433,
+      "clip_ratio/low_min": 0.002083333383779973,
+      "clip_ratio/region_mean": 0.021440374111989513,
+      "entropy": 0.08508480247110128,
+      "epoch": 0.00036,
+      "grad_norm": 0.7093645334243774,
+      "kl": 0.13321141549386084,
+      "learning_rate": 0.0001,
+      "loss": -0.142,
+      "step": 36,
+      "step_time": 28.9255163910002
+    },
+    {
+      "clip_ratio/high_max": 0.0037438530707731843,
+      "clip_ratio/high_mean": 0.0018719265353865921,
+      "clip_ratio/low_mean": 0.0017557953542564064,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0036277218896429986,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8904.0,
+      "completions/max_terminated_length": 8904.0,
+      "completions/mean_length": 7886.75,
+      "completions/mean_terminated_length": 7886.75,
+      "completions/min_length": 2730.0,
+      "completions/min_terminated_length": 2730.0,
+      "entropy": 0.0856572980992496,
+      "epoch": 0.00037,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8132036328315735,
+      "kl": 0.10761717008426785,
+      "learning_rate": 9.918053502751771e-05,
+      "loss": -0.0071,
+      "num_tokens": 5179039.0,
+      "reward": -0.2326236367225647,
+      "reward_std": 0.5471592545509338,
+      "rewards/rollout_reward_func/mean": -0.2326236367225647,
+      "rewards/rollout_reward_func/std": 0.5504636168479919,
+      "sampling/importance_sampling_ratio/max": 1.7522045373916626,
+      "sampling/importance_sampling_ratio/mean": 0.9999704360961914,
+      "sampling/importance_sampling_ratio/min": 0.6301739811897278,
+      "sampling/sampling_logp_difference/max": 0.5608747005462646,
+      "sampling/sampling_logp_difference/mean": 0.00934191420674324,
+      "step": 37,
+      "step_time": 91.66157414699956
+    },
+    {
+      "clip_ratio/high_max": 0.009254805103410035,
+      "clip_ratio/high_mean": 0.004974624811438844,
+      "clip_ratio/low_mean": 0.0146892913908232,
+      "clip_ratio/low_min": 0.002091136120725423,
+      "clip_ratio/region_mean": 0.019663916405988857,
+      "entropy": 0.08511333051137626,
+      "epoch": 0.00038,
+      "grad_norm": 1.016090750694275,
+      "kl": 0.13888527592644095,
+      "learning_rate": 9.675795466159755e-05,
+      "loss": -0.0123,
+      "step": 38,
+      "step_time": 27.996872589999384
+    },
+    {
+      "clip_ratio/high_max": 0.004166666767559946,
+      "clip_ratio/high_mean": 0.002083333383779973,
+      "clip_ratio/low_mean": 0.003956988512072712,
+      "clip_ratio/low_min": 0.0013966916594654322,
+      "clip_ratio/region_mean": 0.006040321895852685,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9174.0,
+      "completions/max_terminated_length": 9174.0,
+      "completions/mean_length": 7773.78125,
+      "completions/mean_terminated_length": 7773.78125,
+      "completions/min_length": 644.0,
+      "completions/min_terminated_length": 644.0,
+      "entropy": 0.08465631771832705,
+      "epoch": 0.00039,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.2169103622436523,
+      "kl": 0.08935846015810966,
+      "learning_rate": 9.283813728906054e-05,
+      "loss": -0.1228,
+      "num_tokens": 5455087.0,
+      "reward": -0.23303870856761932,
+      "reward_std": 0.587963879108429,
+      "rewards/rollout_reward_func/mean": -0.23303870856761932,
+      "rewards/rollout_reward_func/std": 0.6264134645462036,
+      "sampling/importance_sampling_ratio/max": 2.5134615898132324,
+      "sampling/importance_sampling_ratio/mean": 1.000549077987671,
+      "sampling/importance_sampling_ratio/min": 0.555368959903717,
+      "sampling/sampling_logp_difference/max": 0.9216609001159668,
+      "sampling/sampling_logp_difference/mean": 0.009383916854858398,
+      "step": 39,
+      "step_time": 87.98878976900005
+    },
+    {
+      "clip_ratio/high_max": 0.007900029362645,
+      "clip_ratio/high_mean": 0.004326520691392943,
+      "clip_ratio/low_mean": 0.011018106888514012,
+      "clip_ratio/low_min": 0.0018596546142362058,
+      "clip_ratio/region_mean": 0.015344627550803125,
+      "entropy": 0.08598045213147998,
+      "epoch": 0.0004,
+      "grad_norm": 1.855177640914917,
+      "kl": 0.08451172313652933,
+      "learning_rate": 8.759239773845718e-05,
+      "loss": -0.1252,
+      "step": 40,
+      "step_time": 28.765726263999568
+    },
+    {
+      "clip_ratio/high_max": 0.005807758076116443,
+      "clip_ratio/high_mean": 0.003251101268688217,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003251101268688217,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8897.0,
+      "completions/max_terminated_length": 8897.0,
+      "completions/mean_length": 7154.53125,
+      "completions/mean_terminated_length": 7154.53125,
+      "completions/min_length": 2028.0,
+      "completions/min_terminated_length": 2028.0,
+      "entropy": 0.07370180264115334,
+      "epoch": 0.00041,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6197995543479919,
+      "kl": 0.08875341480597854,
+      "learning_rate": 8.125000000000001e-05,
+      "loss": -0.0075,
+      "num_tokens": 5710693.0,
+      "reward": -0.11052435636520386,
+      "reward_std": 0.6305065155029297,
+      "rewards/rollout_reward_func/mean": -0.11052435636520386,
+      "rewards/rollout_reward_func/std": 0.6378242373466492,
+      "sampling/importance_sampling_ratio/max": 1.3583569526672363,
+      "sampling/importance_sampling_ratio/mean": 0.9987045526504517,
+      "sampling/importance_sampling_ratio/min": 0.3902639150619507,
+      "sampling/sampling_logp_difference/max": 0.940932035446167,
+      "sampling/sampling_logp_difference/mean": 0.007857292890548706,
+      "step": 41,
+      "step_time": 89.4707287680003
+    },
+    {
+      "clip_ratio/high_max": 0.011104670935310423,
+      "clip_ratio/high_mean": 0.005552335467655212,
+      "clip_ratio/low_mean": 0.0056608943559695035,
+      "clip_ratio/low_min": 0.0024853801587596536,
+      "clip_ratio/region_mean": 0.011213229881832376,
+      "entropy": 0.07147298031486571,
+      "epoch": 0.00042,
+      "grad_norm": 0.532979428768158,
+      "kl": 0.1001066651660949,
+      "learning_rate": 7.408813728906053e-05,
+      "loss": -0.0097,
+      "step": 42,
+      "step_time": 27.083737006999854
+    },
+    {
+      "clip_ratio/high_max": 0.002190170984249562,
+      "clip_ratio/high_mean": 0.001095085492124781,
+      "clip_ratio/low_mean": 0.0017723122437018901,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002867397735826671,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9047.0,
+      "completions/max_terminated_length": 9047.0,
+      "completions/mean_length": 7559.5625,
+      "completions/mean_terminated_length": 7559.5625,
+      "completions/min_length": 714.0,
+      "completions/min_terminated_length": 714.0,
+      "entropy": 0.07850536378100514,
+      "epoch": 0.00043,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6568584442138672,
+      "kl": 0.13246144936420023,
+      "learning_rate": 6.6419817372537e-05,
+      "loss": -0.164,
+      "num_tokens": 5979343.0,
+      "reward": -0.18616318702697754,
+      "reward_std": 0.5167393088340759,
+      "rewards/rollout_reward_func/mean": -0.18616318702697754,
+      "rewards/rollout_reward_func/std": 0.5579463243484497,
+      "sampling/importance_sampling_ratio/max": 1.8209000825881958,
+      "sampling/importance_sampling_ratio/mean": 1.0000110864639282,
+      "sampling/importance_sampling_ratio/min": 0.5961325764656067,
+      "sampling/sampling_logp_difference/max": 0.5993309020996094,
+      "sampling/sampling_logp_difference/mean": 0.007760843262076378,
+      "step": 43,
+      "step_time": 89.20096494900008
+    },
+    {
+      "clip_ratio/high_max": 0.011211407894734293,
+      "clip_ratio/high_mean": 0.0063001484086271375,
+      "clip_ratio/low_mean": 0.00652579654706642,
+      "clip_ratio/low_min": 0.0014211886445991695,
+      "clip_ratio/region_mean": 0.012825944955693558,
+      "entropy": 0.07914698123931885,
+      "epoch": 0.00044,
+      "grad_norm": 0.4922528564929962,
+      "kl": 0.1358487531542778,
+      "learning_rate": 5.858018262746301e-05,
+      "loss": -0.1705,
+      "step": 44,
+      "step_time": 28.31045281400202
+    },
+    {
+      "clip_ratio/high_max": 0.0007102272938936949,
+      "clip_ratio/high_mean": 0.00035511364694684744,
+      "clip_ratio/low_mean": 0.0017025089473463595,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002057622594293207,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8960.0,
+      "completions/max_terminated_length": 8960.0,
+      "completions/mean_length": 7259.9375,
+      "completions/mean_terminated_length": 7259.9375,
+      "completions/min_length": 1793.0,
+      "completions/min_terminated_length": 1793.0,
+      "entropy": 0.0675961917731911,
+      "epoch": 0.00045,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4965215027332306,
+      "kl": 0.12192867766134441,
+      "learning_rate": 5.0911862710939485e-05,
+      "loss": 0.0928,
+      "num_tokens": 6238685.0,
+      "reward": -0.19794124364852905,
+      "reward_std": 0.6908308267593384,
+      "rewards/rollout_reward_func/mean": -0.19794124364852905,
+      "rewards/rollout_reward_func/std": 0.6924669742584229,
+      "sampling/importance_sampling_ratio/max": 1.7397383451461792,
+      "sampling/importance_sampling_ratio/mean": 1.0005090236663818,
+      "sampling/importance_sampling_ratio/min": 0.6195511817932129,
+      "sampling/sampling_logp_difference/max": 0.5537347793579102,
+      "sampling/sampling_logp_difference/mean": 0.006288852542638779,
+      "step": 45,
+      "step_time": 88.08988413000043
+    },
+    {
+      "clip_ratio/high_max": 0.0034880051389336586,
+      "clip_ratio/high_mean": 0.0017440025694668293,
+      "clip_ratio/low_mean": 0.0030215850565582514,
+      "clip_ratio/low_min": 0.0006944444612599909,
+      "clip_ratio/region_mean": 0.004765587626025081,
+      "entropy": 0.06737200380302966,
+      "epoch": 0.00046,
+      "grad_norm": 0.537150502204895,
+      "kl": 0.12145327543839812,
+      "learning_rate": 4.375000000000001e-05,
+      "loss": 0.092,
+      "step": 46,
+      "step_time": 27.73081736800077
+    },
+    {
+      "clip_ratio/high_max": 0.001753766497131437,
+      "clip_ratio/high_mean": 0.0012241055082995445,
+      "clip_ratio/low_mean": 0.001392790290992707,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0026168957992922515,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9253.0,
+      "completions/max_terminated_length": 9253.0,
+      "completions/mean_length": 7692.15625,
+      "completions/mean_terminated_length": 7692.15625,
+      "completions/min_length": 293.0,
+      "completions/min_terminated_length": 293.0,
+      "entropy": 0.0828928763512522,
+      "epoch": 0.00047,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5421693921089172,
+      "kl": 0.13829960371367633,
+      "learning_rate": 3.740760226154283e-05,
+      "loss": 0.1124,
+      "num_tokens": 6512018.0,
+      "reward": -0.13337820768356323,
+      "reward_std": 0.6952483057975769,
+      "rewards/rollout_reward_func/mean": -0.13337820768356323,
+      "rewards/rollout_reward_func/std": 0.6867809295654297,
+      "sampling/importance_sampling_ratio/max": 1.9118043184280396,
+      "sampling/importance_sampling_ratio/mean": 1.000080943107605,
+      "sampling/importance_sampling_ratio/min": 0.6484359502792358,
+      "sampling/sampling_logp_difference/max": 0.6480474472045898,
+      "sampling/sampling_logp_difference/mean": 0.007844729349017143,
+      "step": 47,
+      "step_time": 89.50516965200086
+    },
+    {
+      "clip_ratio/high_max": 0.0034878277801908553,
+      "clip_ratio/high_mean": 0.002519218367524445,
+      "clip_ratio/low_mean": 0.00764669178170152,
+      "clip_ratio/low_min": 0.0013888889225199819,
+      "clip_ratio/region_mean": 0.010165910149225965,
+      "entropy": 0.08108305186033249,
+      "epoch": 0.00048,
+      "grad_norm": 0.5435065031051636,
+      "kl": 0.15144605189561844,
+      "learning_rate": 3.2161862710939476e-05,
+      "loss": 0.1097,
+      "step": 48,
+      "step_time": 28.342568254000525
+    },
+    {
+      "clip_ratio/high_max": 0.004411048837937415,
+      "clip_ratio/high_mean": 0.0022055244189687073,
+      "clip_ratio/low_mean": 0.0010455680603627115,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003251092479331419,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9355.0,
+      "completions/max_terminated_length": 9355.0,
+      "completions/mean_length": 7641.09375,
+      "completions/mean_terminated_length": 7641.09375,
+      "completions/min_length": 395.0,
+      "completions/min_terminated_length": 395.0,
+      "entropy": 0.06896222359500825,
+      "epoch": 0.00049,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7906138300895691,
+      "kl": 0.17947973730042577,
+      "learning_rate": 2.8242045338402468e-05,
+      "loss": 0.0025,
+      "num_tokens": 6783110.0,
+      "reward": -0.24048486351966858,
+      "reward_std": 0.5240083932876587,
+      "rewards/rollout_reward_func/mean": -0.24048486351966858,
+      "rewards/rollout_reward_func/std": 0.6190164685249329,
+      "sampling/importance_sampling_ratio/max": 1.339056372642517,
+      "sampling/importance_sampling_ratio/mean": 1.000028371810913,
+      "sampling/importance_sampling_ratio/min": 0.6448177099227905,
+      "sampling/sampling_logp_difference/max": 0.43878769874572754,
+      "sampling/sampling_logp_difference/mean": 0.0061599984765052795,
+      "step": 49,
+      "step_time": 91.6421234169984
+    },
+    {
+      "clip_ratio/high_max": 0.00502964504994452,
+      "clip_ratio/high_mean": 0.00251482252497226,
+      "clip_ratio/low_mean": 0.00382724724477157,
+      "clip_ratio/low_min": 0.0006944444612599909,
+      "clip_ratio/region_mean": 0.00634206976974383,
+      "entropy": 0.0660756464349106,
+      "epoch": 0.0005,
+      "grad_norm": 0.6092560887336731,
+      "kl": 0.1758871516212821,
+      "learning_rate": 2.5819464972482288e-05,
+      "loss": -0.0001,
+      "step": 50,
+      "step_time": 28.18612482699973
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 50,
+  "num_input_tokens_seen": 6783110,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:019b24481996b285ffc979c3d1c3f694891c8900b84f610887b4f2a87d8967c6
+size 8145