diff --git a/.gitattributes b/.gitattributes index 918b0424e523d5c00a2767ae813eefe0be640b9c..bb20f492208d06b5b9511093b971a3073c083af6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -41,3 +41,10 @@ logs/figures/E0_entropy_live.png filter=lfs diff=lfs merge=lfs -text logs/figures/E0_live_dashboard.png filter=lfs diff=lfs merge=lfs -text outputs/pairs_4b/multipos_train.jsonl filter=lfs diff=lfs merge=lfs -text outputs/pool_4b/pool_train.jsonl filter=lfs diff=lfs merge=lfs -text +outputs/E0-baseline/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text +outputs/E0-baseline/checkpoint-150/tokenizer.json filter=lfs diff=lfs merge=lfs -text +outputs/E0-baseline/checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text +outputs/E0-baseline/checkpoint-250/tokenizer.json filter=lfs diff=lfs merge=lfs -text +outputs/E0-baseline/checkpoint-300/tokenizer.json filter=lfs diff=lfs merge=lfs -text +outputs/E0-baseline/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text +outputs/E0-baseline/final/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/outputs/E0-baseline/README.md b/outputs/E0-baseline/README.md new file mode 100644 index 0000000000000000000000000000000000000000..136e3133b402c3abe3d2f9b07cf2a4c1760955a9 --- /dev/null +++ b/outputs/E0-baseline/README.md @@ -0,0 +1,67 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +library_name: transformers +model_name: E0-baseline +tags: +- generated_from_trainer +- grpo +- trl +licence: license +--- + +# Model Card for E0-baseline + +This model is a fine-tuned version of [Qwen/Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/pranav2278/div-grpo/runs/hhvcvkst) + + + +This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300). + +### Framework versions + +- TRL: 1.10.0 +- Transformers: 5.15.0 +- Pytorch: 2.13.0 +- Datasets: 5.0.1 +- Tokenizers: 0.22.2 + +## Citations + +Cite GRPO as: + +```bibtex +@article{shao2024deepseekmath, + title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}}, + author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo}, + year = 2024, + eprint = {arXiv:2402.03300}, +} +``` + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-100/README.md b/outputs/E0-baseline/checkpoint-100/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88990785a4695219513364eafae195070ddd3182 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-100/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507 +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.20.0 \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-100/adapter_config.json b/outputs/E0-baseline/checkpoint-100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3b383bef044f277e3ff79f0fc5addabd3c441593 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-100/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.20.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-100/adapter_model.safetensors b/outputs/E0-baseline/checkpoint-100/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05daa47f09a66ab17b99077f753bfbf317369bff --- /dev/null +++ b/outputs/E0-baseline/checkpoint-100/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58b3ac37aaa4179bc96d7e703dc29bc19755704e1761cdb06e990b9f8c77be82 +size 264308896 diff --git a/outputs/E0-baseline/checkpoint-100/chat_template.jinja b/outputs/E0-baseline/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-100/chat_template.jinja @@ -0,0 +1,61 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-100/tokenizer.json b/outputs/E0-baseline/checkpoint-100/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-100/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/outputs/E0-baseline/checkpoint-100/tokenizer_config.json b/outputs/E0-baseline/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..86d67326516569b4f1cec3ac7b51b8903062c648 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-100/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/outputs/E0-baseline/checkpoint-100/trainer_state.json b/outputs/E0-baseline/checkpoint-100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bcd115d3a944f3a2c737207f291f91604449cdff --- /dev/null +++ b/outputs/E0-baseline/checkpoint-100/trainer_state.json @@ -0,0 +1,3334 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2, + "eval_steps": 500, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 529.0, + "completions/mean_terminated_length": 529.0, + "completions/min_length": 482.0, + "completions/min_terminated_length": 482.0, + "entropy": 1.2025159299373627, + "epoch": 0.002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22668755054473877, + "kl": 0.0, + "learning_rate": 0.0, + "loss": 0.2398601919412613, + "num_tokens": 10400.0, + "reward": 6.375, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6477458477020264, + "sampling/importance_sampling_ratio/mean": 0.5064857602119446, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40894174575805664, + "sampling/sampling_logp_difference/mean": 0.026567919179797173, + "step": 1, + "step_time": 12.760562099982053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 644.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 562.25, + "completions/mean_terminated_length": 562.25, + "completions/min_length": 497.0, + "completions/min_terminated_length": 497.0, + "entropy": 1.1930748969316483, + "epoch": 0.004, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12967805564403534, + "kl": 0.0, + "learning_rate": 3e-06, + "loss": -0.08571265637874603, + "num_tokens": 20924.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 1.0037232637405396, + "sampling/importance_sampling_ratio/mean": 0.41275694966316223, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45699238777160645, + "sampling/sampling_logp_difference/mean": 0.025086138397455215, + "step": 2, + "step_time": 11.406366533134133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 483.625, + "completions/mean_terminated_length": 483.625, + "completions/min_length": 407.0, + "completions/min_terminated_length": 407.0, + "entropy": 1.1096689626574516, + "epoch": 0.006, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22966289520263672, + "kl": 0.0008355328354809899, + "learning_rate": 6e-06, + "loss": 0.020913563668727875, + "num_tokens": 30222.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.927120566368103, + "sampling/importance_sampling_ratio/mean": 0.6114993095397949, + "sampling/importance_sampling_ratio/min": 0.11067161709070206, + "sampling/sampling_logp_difference/max": 0.4620504379272461, + "sampling/sampling_logp_difference/mean": 0.024864500388503075, + "step": 3, + "step_time": 26.480680393986404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 608.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 524.75, + "completions/mean_terminated_length": 524.75, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.211122527718544, + "epoch": 0.008, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30378466844558716, + "kl": 0.0008403196770814247, + "learning_rate": 9e-06, + "loss": -0.12959149479866028, + "num_tokens": 40410.0, + "reward": 6.25, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.406888961791992, + "sampling/importance_sampling_ratio/mean": 0.5457419753074646, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3680229187011719, + "sampling/sampling_logp_difference/mean": 0.02656388282775879, + "step": 4, + "step_time": 22.95301443943754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 487.5625, + "completions/mean_terminated_length": 487.5625, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "entropy": 1.247180238366127, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5199307799339294, + "kl": 0.0008723233368073124, + "learning_rate": 1.2e-05, + "loss": 0.11524428427219391, + "num_tokens": 49915.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.1544158458709717, + "sampling/importance_sampling_ratio/mean": 0.963020920753479, + "sampling/importance_sampling_ratio/min": 0.04948288947343826, + "sampling/sampling_logp_difference/max": 0.4774587154388428, + "sampling/sampling_logp_difference/mean": 0.02625642716884613, + "step": 5, + "step_time": 22.37928077671677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 533.0, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 394.0, + "completions/min_terminated_length": 394.0, + "entropy": 1.1693861335515976, + "epoch": 0.012, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27643197774887085, + "kl": 0.0009261858467652928, + "learning_rate": 1.5e-05, + "loss": 0.15093231201171875, + "num_tokens": 60219.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 2.125091791152954, + "sampling/importance_sampling_ratio/mean": 0.7733402252197266, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7049552202224731, + "sampling/sampling_logp_difference/mean": 0.025986071676015854, + "step": 6, + "step_time": 21.00841654697433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 505.6875, + "completions/mean_terminated_length": 505.6875, + "completions/min_length": 425.0, + "completions/min_terminated_length": 425.0, + "entropy": 1.2473183795809746, + "epoch": 0.014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2166663259267807, + "kl": 0.0009701631606731098, + "learning_rate": 1.8e-05, + "loss": -0.08582288026809692, + "num_tokens": 69902.0, + "reward": 7.1875, + "reward_std": 0.75, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.8287386894226074, + "sampling/importance_sampling_ratio/mean": 0.5286832451820374, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.38030898571014404, + "sampling/sampling_logp_difference/mean": 0.0264718160033226, + "step": 7, + "step_time": 46.596127359196544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.1040107272565365, + "epoch": 0.016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18874908983707428, + "kl": 0.0010721117541834246, + "learning_rate": 2.1e-05, + "loss": -0.1946130096912384, + "num_tokens": 79501.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.3561792373657227, + "sampling/importance_sampling_ratio/mean": 0.6918502449989319, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.26114892959594727, + "sampling/sampling_logp_difference/mean": 0.02554757334291935, + "step": 8, + "step_time": 25.433595282491297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 554.0, + "completions/max_terminated_length": 554.0, + "completions/mean_length": 490.1875, + "completions/mean_terminated_length": 490.1875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1377170644700527, + "epoch": 0.018, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.204215869307518, + "kl": 0.002002388624532614, + "learning_rate": 2.4e-05, + "loss": -0.08130021393299103, + "num_tokens": 88976.0, + "reward": 6.5, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.0982444286346436, + "sampling/importance_sampling_ratio/mean": 0.5873216986656189, + "sampling/importance_sampling_ratio/min": 0.04890051856637001, + "sampling/sampling_logp_difference/max": 0.8512144088745117, + "sampling/sampling_logp_difference/mean": 0.02638406865298748, + "step": 9, + "step_time": 18.427699581719935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 500.1875, + "completions/mean_terminated_length": 500.1875, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.1230391450226307, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1985069215297699, + "kl": 0.0026735300780273974, + "learning_rate": 2.7000000000000002e-05, + "loss": -0.12387816607952118, + "num_tokens": 98603.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.983199954032898, + "sampling/importance_sampling_ratio/mean": 0.5639468431472778, + "sampling/importance_sampling_ratio/min": 0.012905921787023544, + "sampling/sampling_logp_difference/max": 0.3653905391693115, + "sampling/sampling_logp_difference/mean": 0.025383003056049347, + "step": 10, + "step_time": 14.99981931829825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 459.6875, + "completions/mean_terminated_length": 459.6875, + "completions/min_length": 379.0, + "completions/min_terminated_length": 379.0, + "entropy": 1.213625729084015, + "epoch": 0.022, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3695620596408844, + "kl": 0.00424640768324025, + "learning_rate": 3e-05, + "loss": -0.06913074851036072, + "num_tokens": 107734.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.000136375427246, + "sampling/importance_sampling_ratio/mean": 0.8144867420196533, + "sampling/importance_sampling_ratio/min": 0.06302778422832489, + "sampling/sampling_logp_difference/max": 0.3647327423095703, + "sampling/sampling_logp_difference/mean": 0.026674197986721992, + "step": 11, + "step_time": 30.46549107739702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.191277615725994, + "epoch": 0.024, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35457733273506165, + "kl": 0.007200263120466843, + "learning_rate": 3e-05, + "loss": 0.22097699344158173, + "num_tokens": 117199.0, + "reward": 5.9375, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 1.5660414695739746, + "sampling/importance_sampling_ratio/mean": 0.649204432964325, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3921785354614258, + "sampling/sampling_logp_difference/mean": 0.02726689912378788, + "step": 12, + "step_time": 15.719243939965963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 464.3125, + "completions/mean_terminated_length": 464.3125, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "entropy": 1.24251464381814, + "epoch": 0.026, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30047014355659485, + "kl": 0.0058551667898427695, + "learning_rate": 3e-05, + "loss": -0.07746122777462006, + "num_tokens": 126556.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6028401851654053, + "sampling/importance_sampling_ratio/mean": 0.7561360597610474, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4946432113647461, + "sampling/sampling_logp_difference/mean": 0.02639186754822731, + "step": 13, + "step_time": 18.08984712138772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 444.8125, + "completions/mean_terminated_length": 444.8125, + "completions/min_length": 389.0, + "completions/min_terminated_length": 389.0, + "entropy": 1.1501125395298004, + "epoch": 0.028, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24105942249298096, + "kl": 0.012796793889719993, + "learning_rate": 3e-05, + "loss": 0.10855278372764587, + "num_tokens": 135417.0, + "reward": 3.1875, + "reward_std": 3.310966730117798, + "rewards/quality_reward/mean": 3.1875, + "rewards/quality_reward/std": 3.310966730117798, + "sampling/importance_sampling_ratio/max": 2.541518211364746, + "sampling/importance_sampling_ratio/mean": 0.5903321504592896, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.8390979766845703, + "sampling/sampling_logp_difference/mean": 0.02838001400232315, + "step": 14, + "step_time": 20.055794408079237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 497.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 442.25, + "completions/mean_terminated_length": 442.25, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 1.1262825280427933, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19040776789188385, + "kl": 0.010701361010433175, + "learning_rate": 3e-05, + "loss": -0.007966680452227592, + "num_tokens": 144205.0, + "reward": 5.875, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.0754293203353882, + "sampling/importance_sampling_ratio/mean": 0.41446638107299805, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3982200622558594, + "sampling/sampling_logp_difference/mean": 0.027210108935832977, + "step": 15, + "step_time": 14.176074750721455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 478.125, + "completions/mean_terminated_length": 478.125, + "completions/min_length": 433.0, + "completions/min_terminated_length": 433.0, + "entropy": 1.2985924184322357, + "epoch": 0.032, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23625652492046356, + "kl": 0.0213887135614641, + "learning_rate": 3e-05, + "loss": -0.21546132862567902, + "num_tokens": 153543.0, + "reward": 6.5, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 2.0074336528778076, + "sampling/importance_sampling_ratio/mean": 0.49076417088508606, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5095968246459961, + "sampling/sampling_logp_difference/mean": 0.028833162039518356, + "step": 16, + "step_time": 15.848205763846636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 469.25, + "completions/mean_terminated_length": 469.25, + "completions/min_length": 423.0, + "completions/min_terminated_length": 423.0, + "entropy": 1.3148910626769066, + "epoch": 0.034, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17765119671821594, + "kl": 0.02128879475640133, + "learning_rate": 3e-05, + "loss": -0.0828079879283905, + "num_tokens": 163043.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 1.5988941192626953, + "sampling/importance_sampling_ratio/mean": 0.5021570324897766, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.591062068939209, + "sampling/sampling_logp_difference/mean": 0.030804751440882683, + "step": 17, + "step_time": 28.313011147081852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 447.0, + "completions/mean_terminated_length": 447.0, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 1.390664003789425, + "epoch": 0.036, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37244123220443726, + "kl": 0.019650122558232397, + "learning_rate": 3e-05, + "loss": -0.01184748113155365, + "num_tokens": 172379.0, + "reward": 6.0, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.675238847732544, + "sampling/importance_sampling_ratio/mean": 0.9581233263015747, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5762512683868408, + "sampling/sampling_logp_difference/mean": 0.03126702085137367, + "step": 18, + "step_time": 30.991567107848823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 498.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 447.75, + "completions/mean_terminated_length": 447.75, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 1.3287223279476166, + "epoch": 0.038, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23734751343727112, + "kl": 0.022738213243428618, + "learning_rate": 3e-05, + "loss": 0.040024783462285995, + "num_tokens": 181239.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.095722198486328, + "sampling/importance_sampling_ratio/mean": 0.6408629417419434, + "sampling/importance_sampling_ratio/min": 0.1203346848487854, + "sampling/sampling_logp_difference/max": 0.4722297191619873, + "sampling/sampling_logp_difference/mean": 0.030378391966223717, + "step": 19, + "step_time": 18.615950418636203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 452.3125, + "completions/mean_terminated_length": 452.3125, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 1.1001618690788746, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34637194871902466, + "kl": 0.015380491153337061, + "learning_rate": 3e-05, + "loss": 0.1691148728132248, + "num_tokens": 190068.0, + "reward": 6.125, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.6566128730773926, + "sampling/importance_sampling_ratio/mean": 0.7767290472984314, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.6531033515930176, + "sampling/sampling_logp_difference/mean": 0.030463142320513725, + "step": 20, + "step_time": 15.741292077116668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 457.5625, + "completions/mean_terminated_length": 457.5625, + "completions/min_length": 390.0, + "completions/min_terminated_length": 390.0, + "entropy": 1.3708399310708046, + "epoch": 0.042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2514922618865967, + "kl": 0.018277494702488184, + "learning_rate": 3e-05, + "loss": -0.13425321877002716, + "num_tokens": 198941.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.5612297058105469, + "sampling/importance_sampling_ratio/mean": 0.601022481918335, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4818992614746094, + "sampling/sampling_logp_difference/mean": 0.0315740592777729, + "step": 21, + "step_time": 17.24192419089377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 421.4375, + "completions/mean_terminated_length": 421.4375, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 1.136269599199295, + "epoch": 0.044, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2268020063638687, + "kl": 0.017973962530959398, + "learning_rate": 3e-05, + "loss": 0.010622119531035423, + "num_tokens": 207300.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.7986358404159546, + "sampling/importance_sampling_ratio/mean": 0.46136727929115295, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5335149765014648, + "sampling/sampling_logp_difference/mean": 0.02804401144385338, + "step": 22, + "step_time": 19.37282825494185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 452.6875, + "completions/mean_terminated_length": 452.6875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3299130946397781, + "epoch": 0.046, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33994877338409424, + "kl": 0.021804221265483648, + "learning_rate": 3e-05, + "loss": -0.24404363334178925, + "num_tokens": 216343.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.452249526977539, + "sampling/importance_sampling_ratio/mean": 0.747193455696106, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6328549385070801, + "sampling/sampling_logp_difference/mean": 0.02918298915028572, + "step": 23, + "step_time": 15.356728344224393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 462.4375, + "completions/mean_terminated_length": 462.4375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.379701942205429, + "epoch": 0.048, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3076202869415283, + "kl": 0.020299295720178634, + "learning_rate": 3e-05, + "loss": -0.09123071283102036, + "num_tokens": 225550.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.5845099687576294, + "sampling/importance_sampling_ratio/mean": 0.6625345945358276, + "sampling/importance_sampling_ratio/min": 0.04495502635836601, + "sampling/sampling_logp_difference/max": 0.36547183990478516, + "sampling/sampling_logp_difference/mean": 0.028946854174137115, + "step": 24, + "step_time": 14.95462113339454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.2471638694405556, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22180576622486115, + "kl": 0.018309170845896006, + "learning_rate": 3e-05, + "loss": -0.1412944793701172, + "num_tokens": 235005.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.6407876014709473, + "sampling/importance_sampling_ratio/mean": 0.7712795734405518, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4363563060760498, + "sampling/sampling_logp_difference/mean": 0.02898716926574707, + "step": 25, + "step_time": 16.46686205593869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 455.75, + "completions/mean_terminated_length": 455.75, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 1.28530602902174, + "epoch": 0.052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37535253167152405, + "kl": 0.020504546992015094, + "learning_rate": 3e-05, + "loss": 0.10839397460222244, + "num_tokens": 243953.0, + "reward": 6.25, + "reward_std": 1.0645813941955566, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.0645813941955566, + "sampling/importance_sampling_ratio/max": 2.0567266941070557, + "sampling/importance_sampling_ratio/mean": 0.5800145864486694, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4523625373840332, + "sampling/sampling_logp_difference/mean": 0.027610119432210922, + "step": 26, + "step_time": 22.121026155073196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.0, + "completions/max_terminated_length": 514.0, + "completions/mean_length": 462.75, + "completions/mean_terminated_length": 462.75, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.2253629937767982, + "epoch": 0.054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2340388149023056, + "kl": 0.020236384589225054, + "learning_rate": 3e-05, + "loss": 0.04823978245258331, + "num_tokens": 253237.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.1560932397842407, + "sampling/importance_sampling_ratio/mean": 0.40753045678138733, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35022830963134766, + "sampling/sampling_logp_difference/mean": 0.028367744758725166, + "step": 27, + "step_time": 15.10967448912561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 538.0, + "completions/max_terminated_length": 538.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "entropy": 1.3779077678918839, + "epoch": 0.056, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5531017184257507, + "kl": 0.01706669357372448, + "learning_rate": 3e-05, + "loss": 0.0933581069111824, + "num_tokens": 262454.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.5711586475372314, + "sampling/importance_sampling_ratio/mean": 0.7730721831321716, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.343827486038208, + "sampling/sampling_logp_difference/mean": 0.028883326798677444, + "step": 28, + "step_time": 38.59647103771567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 477.0, + "completions/mean_terminated_length": 477.0, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.3791070505976677, + "epoch": 0.058, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33771538734436035, + "kl": 0.02141271048458293, + "learning_rate": 3e-05, + "loss": 0.010216176509857178, + "num_tokens": 271918.0, + "reward": 5.75, + "reward_std": 1.2909945249557495, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.2909945249557495, + "sampling/importance_sampling_ratio/max": 2.4999797344207764, + "sampling/importance_sampling_ratio/mean": 1.0250636339187622, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3258817195892334, + "sampling/sampling_logp_difference/mean": 0.029029540717601776, + "step": 29, + "step_time": 23.610272648278624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 471.1875, + "completions/mean_terminated_length": 471.1875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.4652926102280617, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21955685317516327, + "kl": 0.019562674744520336, + "learning_rate": 3e-05, + "loss": -0.014814459718763828, + "num_tokens": 281305.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 2.802098274230957, + "sampling/importance_sampling_ratio/mean": 0.866391658782959, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6354451179504395, + "sampling/sampling_logp_difference/mean": 0.03177585452795029, + "step": 30, + "step_time": 16.961607525125146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 490.5, + "completions/mean_terminated_length": 490.5, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 1.1957123205065727, + "epoch": 0.062, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12176825106143951, + "kl": 0.026934220048133284, + "learning_rate": 3e-05, + "loss": -0.08307373523712158, + "num_tokens": 291409.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.610858678817749, + "sampling/importance_sampling_ratio/mean": 0.6937527656555176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4253416061401367, + "sampling/sampling_logp_difference/mean": 0.02871524728834629, + "step": 31, + "step_time": 15.012207658961415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 462.0, + "completions/mean_terminated_length": 462.0, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3018206283450127, + "epoch": 0.064, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4979296624660492, + "kl": 0.03539742121938616, + "learning_rate": 3e-05, + "loss": 0.0017175457905977964, + "num_tokens": 300649.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8527640104293823, + "sampling/importance_sampling_ratio/mean": 0.7824680209159851, + "sampling/importance_sampling_ratio/min": 0.07447884231805801, + "sampling/sampling_logp_difference/max": 0.5221483707427979, + "sampling/sampling_logp_difference/mean": 0.029107660055160522, + "step": 32, + "step_time": 22.78309725271538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 474.0, + "completions/mean_terminated_length": 474.0, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.321175642311573, + "epoch": 0.066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22016967833042145, + "kl": 0.029592803912237287, + "learning_rate": 3e-05, + "loss": 0.05625719577074051, + "num_tokens": 309889.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.9875097274780273, + "sampling/importance_sampling_ratio/mean": 0.7832435369491577, + "sampling/importance_sampling_ratio/min": 0.18006731569766998, + "sampling/sampling_logp_difference/max": 0.38585615158081055, + "sampling/sampling_logp_difference/mean": 0.027828343212604523, + "step": 33, + "step_time": 44.51360382232815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 459.9375, + "completions/mean_terminated_length": 459.9375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.2222414836287498, + "epoch": 0.068, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2873040437698364, + "kl": 0.02840927499346435, + "learning_rate": 3e-05, + "loss": -0.037432070821523666, + "num_tokens": 318904.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 2.6647069454193115, + "sampling/importance_sampling_ratio/mean": 0.5744763016700745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.41987133026123047, + "sampling/sampling_logp_difference/mean": 0.029470665380358696, + "step": 34, + "step_time": 133.03877451224253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 522.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 462.1875, + "completions/mean_terminated_length": 462.1875, + "completions/min_length": 413.0, + "completions/min_terminated_length": 413.0, + "entropy": 1.1665974482893944, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2490653246641159, + "kl": 0.025841041060630232, + "learning_rate": 3e-05, + "loss": -0.20422951877117157, + "num_tokens": 328011.0, + "reward": 6.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 2.510730266571045, + "sampling/importance_sampling_ratio/mean": 0.7954420447349548, + "sampling/importance_sampling_ratio/min": 0.020566223189234734, + "sampling/sampling_logp_difference/max": 0.36430132389068604, + "sampling/sampling_logp_difference/mean": 0.026844775304198265, + "step": 35, + "step_time": 21.5843787365593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 549.0, + "completions/max_terminated_length": 549.0, + "completions/mean_length": 492.0, + "completions/mean_terminated_length": 492.0, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3467887043952942, + "epoch": 0.072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15268851816654205, + "kl": 0.023660800594370812, + "learning_rate": 3e-05, + "loss": -0.11188814043998718, + "num_tokens": 337731.0, + "reward": 6.5625, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.6270908117294312, + "sampling/importance_sampling_ratio/mean": 0.614537239074707, + "sampling/importance_sampling_ratio/min": 0.10853960365056992, + "sampling/sampling_logp_difference/max": 0.4023854732513428, + "sampling/sampling_logp_difference/mean": 0.028961554169654846, + "step": 36, + "step_time": 18.843947287183255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.0, + "completions/max_terminated_length": 513.0, + "completions/mean_length": 460.75, + "completions/mean_terminated_length": 460.75, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "entropy": 1.2476315572857857, + "epoch": 0.074, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42548227310180664, + "kl": 0.022181478852871805, + "learning_rate": 3e-05, + "loss": 0.37223517894744873, + "num_tokens": 346815.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.6624510288238525, + "sampling/importance_sampling_ratio/mean": 0.7942180633544922, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4037543535232544, + "sampling/sampling_logp_difference/mean": 0.028780322521924973, + "step": 37, + "step_time": 42.04662919091061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 452.0, + "completions/mean_terminated_length": 452.0, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "entropy": 1.1745503433048725, + "epoch": 0.076, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16980381309986115, + "kl": 0.017483733594417572, + "learning_rate": 3e-05, + "loss": -0.09029137343168259, + "num_tokens": 355711.0, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "sampling/importance_sampling_ratio/max": 2.3201518058776855, + "sampling/importance_sampling_ratio/mean": 0.721072793006897, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.30433225631713867, + "sampling/sampling_logp_difference/mean": 0.026646045967936516, + "step": 38, + "step_time": 38.63075139513239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 466.8125, + "completions/mean_terminated_length": 466.8125, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2664988860487938, + "epoch": 0.078, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21305795013904572, + "kl": 0.021121050289366394, + "learning_rate": 3e-05, + "loss": -0.03154226019978523, + "num_tokens": 364860.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.0297553539276123, + "sampling/importance_sampling_ratio/mean": 0.674609363079071, + "sampling/importance_sampling_ratio/min": 0.11245065927505493, + "sampling/sampling_logp_difference/max": 0.37443917989730835, + "sampling/sampling_logp_difference/mean": 0.028257746249437332, + "step": 39, + "step_time": 30.028073193039745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 485.1875, + "completions/mean_terminated_length": 485.1875, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.3458357080817223, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12235487997531891, + "kl": 0.018535695446189493, + "learning_rate": 3e-05, + "loss": -0.035816628485918045, + "num_tokens": 374607.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.242042303085327, + "sampling/importance_sampling_ratio/mean": 0.5344723463058472, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36985254287719727, + "sampling/sampling_logp_difference/mean": 0.029600802809000015, + "step": 40, + "step_time": 15.446288945619017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 498.875, + "completions/mean_terminated_length": 498.875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3402792811393738, + "epoch": 0.082, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15642686188220978, + "kl": 0.013967045990284532, + "learning_rate": 3e-05, + "loss": 0.0011727213859558105, + "num_tokens": 384317.0, + "reward": 6.375, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5691092014312744, + "sampling/importance_sampling_ratio/mean": 0.3696203827857971, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6487679481506348, + "sampling/sampling_logp_difference/mean": 0.03018251620233059, + "step": 41, + "step_time": 18.15720977121964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.3317564576864243, + "epoch": 0.084, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3236794173717499, + "kl": 0.01707366103073582, + "learning_rate": 3e-05, + "loss": 0.10363321751356125, + "num_tokens": 393934.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.0568668842315674, + "sampling/importance_sampling_ratio/mean": 0.6415404677391052, + "sampling/importance_sampling_ratio/min": 0.07682990282773972, + "sampling/sampling_logp_difference/max": 0.7769032716751099, + "sampling/sampling_logp_difference/mean": 0.02936164103448391, + "step": 42, + "step_time": 15.066733688581735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 510.9375, + "completions/mean_terminated_length": 510.9375, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2099780030548573, + "epoch": 0.086, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1771102100610733, + "kl": 0.01784718461567536, + "learning_rate": 3e-05, + "loss": -0.027566466480493546, + "num_tokens": 403893.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.3430161476135254, + "sampling/importance_sampling_ratio/mean": 0.7769261598587036, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36153650283813477, + "sampling/sampling_logp_difference/mean": 0.028799494728446007, + "step": 43, + "step_time": 16.90863296529278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 494.1875, + "completions/mean_terminated_length": 494.1875, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "entropy": 1.2924985438585281, + "epoch": 0.088, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3432070314884186, + "kl": 0.014529847539961338, + "learning_rate": 3e-05, + "loss": -0.04157561436295509, + "num_tokens": 413312.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.726816177368164, + "sampling/importance_sampling_ratio/mean": 0.8989821672439575, + "sampling/importance_sampling_ratio/min": 0.07410597801208496, + "sampling/sampling_logp_difference/max": 0.31444358825683594, + "sampling/sampling_logp_difference/mean": 0.028122060000896454, + "step": 44, + "step_time": 16.7880685236305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 543.5, + "completions/mean_terminated_length": 543.5, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.206408604979515, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36233776807785034, + "kl": 0.015796773659531027, + "learning_rate": 3e-05, + "loss": 0.029176680371165276, + "num_tokens": 423624.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.677195429801941, + "sampling/importance_sampling_ratio/mean": 0.6537867188453674, + "sampling/importance_sampling_ratio/min": 0.09822077304124832, + "sampling/sampling_logp_difference/max": 0.31381869316101074, + "sampling/sampling_logp_difference/mean": 0.02717999368906021, + "step": 45, + "step_time": 23.35960763087496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 643.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 534.5625, + "completions/mean_terminated_length": 534.5625, + "completions/min_length": 453.0, + "completions/min_terminated_length": 453.0, + "entropy": 1.2577891275286674, + "epoch": 0.092, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20219561457633972, + "kl": 0.014481160265859216, + "learning_rate": 3e-05, + "loss": 0.18850615620613098, + "num_tokens": 433817.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.203894853591919, + "sampling/importance_sampling_ratio/mean": 0.697495698928833, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35237741470336914, + "sampling/sampling_logp_difference/mean": 0.02723248302936554, + "step": 46, + "step_time": 19.56032704282552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 538.25, + "completions/mean_terminated_length": 538.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.3271892964839935, + "epoch": 0.094, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22262753546237946, + "kl": 0.012554377142805606, + "learning_rate": 3e-05, + "loss": 0.06984467804431915, + "num_tokens": 444045.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 2.2261502742767334, + "sampling/importance_sampling_ratio/mean": 0.6712950468063354, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4496123790740967, + "sampling/sampling_logp_difference/mean": 0.028838323429226875, + "step": 47, + "step_time": 21.226045075803995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 514.25, + "completions/mean_terminated_length": 514.25, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "entropy": 1.1054812744259834, + "epoch": 0.096, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3244606554508209, + "kl": 0.0157591889728792, + "learning_rate": 3e-05, + "loss": 0.09024985134601593, + "num_tokens": 453945.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.6745388507843018, + "sampling/importance_sampling_ratio/mean": 0.7607913017272949, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650559186935425, + "sampling/sampling_logp_difference/mean": 0.026116060093045235, + "step": 48, + "step_time": 17.565261672716588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 585.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 502.4375, + "completions/mean_terminated_length": 502.4375, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.2281213253736496, + "epoch": 0.098, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4033137857913971, + "kl": 0.015613102470524609, + "learning_rate": 3e-05, + "loss": -0.2898017466068268, + "num_tokens": 463576.0, + "reward": 6.4375, + "reward_std": 1.2632629871368408, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.2632629871368408, + "sampling/importance_sampling_ratio/max": 2.6974196434020996, + "sampling/importance_sampling_ratio/mean": 0.7124314308166504, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3675417900085449, + "sampling/sampling_logp_difference/mean": 0.028341906145215034, + "step": 49, + "step_time": 29.838082558009773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 530.1875, + "completions/mean_terminated_length": 530.1875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.1955150067806244, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23144562542438507, + "kl": 0.01374751579714939, + "learning_rate": 3e-05, + "loss": -0.19065965712070465, + "num_tokens": 473915.0, + "reward": 6.3125, + "reward_std": 1.1954776048660278, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.1954776048660278, + "sampling/importance_sampling_ratio/max": 2.54063081741333, + "sampling/importance_sampling_ratio/mean": 0.8162041902542114, + "sampling/importance_sampling_ratio/min": 0.1628064066171646, + "sampling/sampling_logp_difference/max": 0.2777421474456787, + "sampling/sampling_logp_difference/mean": 0.02853373810648918, + "step": 50, + "step_time": 16.047010839451104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 549.875, + "completions/mean_terminated_length": 549.875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1870752647519112, + "epoch": 0.102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4241364896297455, + "kl": 0.013923745544161648, + "learning_rate": 3e-05, + "loss": -0.1761355698108673, + "num_tokens": 484577.0, + "reward": 6.5625, + "reward_std": 1.0935417413711548, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 1.0935417413711548, + "sampling/importance_sampling_ratio/max": 2.929507255554199, + "sampling/importance_sampling_ratio/mean": 0.6905896663665771, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5649824142456055, + "sampling/sampling_logp_difference/mean": 0.028025619685649872, + "step": 51, + "step_time": 44.488836522214115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 552.375, + "completions/mean_terminated_length": 552.375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.098166175186634, + "epoch": 0.104, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.32851356267929077, + "kl": 0.01297539210645482, + "learning_rate": 3e-05, + "loss": -0.06503897905349731, + "num_tokens": 495015.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.9652340412139893, + "sampling/importance_sampling_ratio/mean": 0.9612224102020264, + "sampling/importance_sampling_ratio/min": 0.040177375078201294, + "sampling/sampling_logp_difference/max": 0.3454720973968506, + "sampling/sampling_logp_difference/mean": 0.02687419019639492, + "step": 52, + "step_time": 20.21497478755191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 578.375, + "completions/mean_terminated_length": 578.375, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2311968132853508, + "epoch": 0.106, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21921825408935547, + "kl": 0.017025968758389354, + "learning_rate": 3e-05, + "loss": -0.18975484371185303, + "num_tokens": 505909.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.0094237327575684, + "sampling/importance_sampling_ratio/mean": 0.5587533116340637, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.39138758182525635, + "sampling/sampling_logp_difference/mean": 0.028095029294490814, + "step": 53, + "step_time": 31.140278964303434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 473.25, + "completions/mean_terminated_length": 473.25, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 1.2682743221521378, + "epoch": 0.108, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27410373091697693, + "kl": 0.018500705540645868, + "learning_rate": 3e-05, + "loss": 0.14847250282764435, + "num_tokens": 515073.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.875750780105591, + "sampling/importance_sampling_ratio/mean": 0.7429271936416626, + "sampling/importance_sampling_ratio/min": 0.09779425710439682, + "sampling/sampling_logp_difference/max": 0.5433444976806641, + "sampling/sampling_logp_difference/mean": 0.02810005284845829, + "step": 54, + "step_time": 18.365382733754814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 549.0, + "completions/mean_terminated_length": 549.0, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.3073157891631126, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29323700070381165, + "kl": 0.016703857632819563, + "learning_rate": 3e-05, + "loss": 0.05967015400528908, + "num_tokens": 525377.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 1.4832638502120972, + "sampling/importance_sampling_ratio/mean": 0.6094669103622437, + "sampling/importance_sampling_ratio/min": 0.08072065562009811, + "sampling/sampling_logp_difference/max": 0.39328718185424805, + "sampling/sampling_logp_difference/mean": 0.02906947024166584, + "step": 55, + "step_time": 30.47015379369259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 576.625, + "completions/mean_terminated_length": 576.625, + "completions/min_length": 500.0, + "completions/min_terminated_length": 500.0, + "entropy": 1.2820357605814934, + "epoch": 0.112, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.200644388794899, + "kl": 0.018819268501829356, + "learning_rate": 3e-05, + "loss": -0.04828515648841858, + "num_tokens": 536163.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.3941831588745117, + "sampling/importance_sampling_ratio/mean": 0.5633801221847534, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3664684295654297, + "sampling/sampling_logp_difference/mean": 0.02962428703904152, + "step": 56, + "step_time": 16.172011949121952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 640.25, + "completions/mean_terminated_length": 640.25, + "completions/min_length": 556.0, + "completions/min_terminated_length": 556.0, + "entropy": 1.4191219061613083, + "epoch": 0.114, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19633841514587402, + "kl": 0.02060725452611223, + "learning_rate": 3e-05, + "loss": -0.12029920518398285, + "num_tokens": 548143.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.3877830505371094, + "sampling/importance_sampling_ratio/mean": 0.6956661343574524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33841991424560547, + "sampling/sampling_logp_difference/mean": 0.028747636824846268, + "step": 57, + "step_time": 26.892430102452636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 646.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 560.4375, + "completions/mean_terminated_length": 560.4375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.3337246850132942, + "epoch": 0.116, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24178451299667358, + "kl": 0.023009511292912066, + "learning_rate": 3e-05, + "loss": -0.02738652005791664, + "num_tokens": 558710.0, + "reward": 6.0625, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.4787031412124634, + "sampling/importance_sampling_ratio/mean": 0.4813012480735779, + "sampling/importance_sampling_ratio/min": 0.05691095441579819, + "sampling/sampling_logp_difference/max": 0.3029825687408447, + "sampling/sampling_logp_difference/mean": 0.029777564108371735, + "step": 58, + "step_time": 17.878377372398973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 697.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 595.25, + "completions/mean_terminated_length": 595.25, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.3107040077447891, + "epoch": 0.118, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1262614130973816, + "kl": 0.022026430582627654, + "learning_rate": 3e-05, + "loss": 0.04775794595479965, + "num_tokens": 569826.0, + "reward": 6.4375, + "reward_std": 1.0307763814926147, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.0307763814926147, + "sampling/importance_sampling_ratio/max": 2.1841602325439453, + "sampling/importance_sampling_ratio/mean": 0.5139275193214417, + "sampling/importance_sampling_ratio/min": 0.026369251310825348, + "sampling/sampling_logp_difference/max": 0.42272377014160156, + "sampling/sampling_logp_difference/mean": 0.028494788333773613, + "step": 59, + "step_time": 24.42572767334059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 549.1875, + "completions/mean_terminated_length": 549.1875, + "completions/min_length": 489.0, + "completions/min_terminated_length": 489.0, + "entropy": 1.1738965958356857, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.588756799697876, + "kl": 0.025482238037511706, + "learning_rate": 3e-05, + "loss": 0.2925710678100586, + "num_tokens": 580229.0, + "reward": 6.0625, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 2.8465583324432373, + "sampling/importance_sampling_ratio/mean": 1.1227651834487915, + "sampling/importance_sampling_ratio/min": 0.1096419170498848, + "sampling/sampling_logp_difference/max": 0.4628920555114746, + "sampling/sampling_logp_difference/mean": 0.02885228767991066, + "step": 60, + "step_time": 21.57787229306996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 692.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 577.3125, + "completions/mean_terminated_length": 577.3125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.1913195550441742, + "epoch": 0.122, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2779391407966614, + "kl": 0.02629381464794278, + "learning_rate": 3e-05, + "loss": 0.12304374575614929, + "num_tokens": 591178.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.1122686862945557, + "sampling/importance_sampling_ratio/mean": 0.650765061378479, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.47726941108703613, + "sampling/sampling_logp_difference/mean": 0.027112768962979317, + "step": 61, + "step_time": 23.71764762001112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 628.3125, + "completions/mean_terminated_length": 628.3125, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3224291801452637, + "epoch": 0.124, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1645125448703766, + "kl": 0.025764177553355694, + "learning_rate": 3e-05, + "loss": 0.17419062554836273, + "num_tokens": 603055.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1662254333496094, + "sampling/importance_sampling_ratio/mean": 0.5809424519538879, + "sampling/importance_sampling_ratio/min": 0.03490918502211571, + "sampling/sampling_logp_difference/max": 0.4525270462036133, + "sampling/sampling_logp_difference/mean": 0.028948483988642693, + "step": 62, + "step_time": 35.74759274255484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 597.375, + "completions/mean_terminated_length": 597.375, + "completions/min_length": 478.0, + "completions/min_terminated_length": 478.0, + "entropy": 1.1552416533231735, + "epoch": 0.126, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20707836747169495, + "kl": 0.026473846402950585, + "learning_rate": 3e-05, + "loss": -0.019145065918564796, + "num_tokens": 614341.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.0268709659576416, + "sampling/importance_sampling_ratio/mean": 0.7066210508346558, + "sampling/importance_sampling_ratio/min": 0.1595209240913391, + "sampling/sampling_logp_difference/max": 0.42907285690307617, + "sampling/sampling_logp_difference/mean": 0.028000790625810623, + "step": 63, + "step_time": 39.37250621803105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 568.5, + "completions/mean_terminated_length": 568.5, + "completions/min_length": 509.0, + "completions/min_terminated_length": 509.0, + "entropy": 1.2810933291912079, + "epoch": 0.128, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21552008390426636, + "kl": 0.029041914734989405, + "learning_rate": 3e-05, + "loss": 0.037037670612335205, + "num_tokens": 625165.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.1896748542785645, + "sampling/importance_sampling_ratio/mean": 0.6408567428588867, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.374176025390625, + "sampling/sampling_logp_difference/mean": 0.02858484350144863, + "step": 64, + "step_time": 18.69576471252367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 622.75, + "completions/mean_terminated_length": 622.75, + "completions/min_length": 545.0, + "completions/min_terminated_length": 545.0, + "entropy": 1.32467532902956, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2287176102399826, + "kl": 0.023987084976397455, + "learning_rate": 3e-05, + "loss": 0.0731797143816948, + "num_tokens": 636633.0, + "reward": 6.375, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 1.4288272857666016, + "sampling/importance_sampling_ratio/mean": 0.5977773666381836, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4503474235534668, + "sampling/sampling_logp_difference/mean": 0.02755960263311863, + "step": 65, + "step_time": 27.502957582939416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 767.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 639.25, + "completions/mean_terminated_length": 639.25, + "completions/min_length": 554.0, + "completions/min_terminated_length": 554.0, + "entropy": 1.400998167693615, + "epoch": 0.132, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27425676584243774, + "kl": 0.028104660217650235, + "learning_rate": 3e-05, + "loss": 0.10324293375015259, + "num_tokens": 648597.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.971261501312256, + "sampling/importance_sampling_ratio/mean": 0.6433250904083252, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4819211959838867, + "sampling/sampling_logp_difference/mean": 0.029852069914340973, + "step": 66, + "step_time": 26.79405551031232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 541.375, + "completions/mean_terminated_length": 541.375, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.2635268718004227, + "epoch": 0.134, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13181555271148682, + "kl": 0.03373931790702045, + "learning_rate": 3e-05, + "loss": -0.11447598040103912, + "num_tokens": 658875.0, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 1.1883972883224487, + "sampling/importance_sampling_ratio/mean": 0.4192371368408203, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.44372403621673584, + "sampling/sampling_logp_difference/mean": 0.02911720983684063, + "step": 67, + "step_time": 27.6137525443919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 842.0, + "completions/max_terminated_length": 842.0, + "completions/mean_length": 689.4375, + "completions/mean_terminated_length": 689.4375, + "completions/min_length": 544.0, + "completions/min_terminated_length": 544.0, + "entropy": 1.2496536895632744, + "epoch": 0.136, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20560002326965332, + "kl": 0.026702777307946235, + "learning_rate": 3e-05, + "loss": -0.10130438208580017, + "num_tokens": 671690.0, + "reward": 5.875, + "reward_std": 2.0289571285247803, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 2.0289571285247803, + "sampling/importance_sampling_ratio/max": 2.8383262157440186, + "sampling/importance_sampling_ratio/mean": 0.9476768374443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35082435607910156, + "sampling/sampling_logp_difference/mean": 0.028364315629005432, + "step": 68, + "step_time": 29.35345455724746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 701.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 614.25, + "completions/mean_terminated_length": 614.25, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.1948458775877953, + "epoch": 0.138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20431844890117645, + "kl": 0.03377161466050893, + "learning_rate": 3e-05, + "loss": -0.0560678169131279, + "num_tokens": 683046.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.162766933441162, + "sampling/importance_sampling_ratio/mean": 0.5678162574768066, + "sampling/importance_sampling_ratio/min": 0.05678822472691536, + "sampling/sampling_logp_difference/max": 0.5758893489837646, + "sampling/sampling_logp_difference/mean": 0.028125843033194542, + "step": 69, + "step_time": 27.574916049838066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 588.3125, + "completions/mean_terminated_length": 588.3125, + "completions/min_length": 532.0, + "completions/min_terminated_length": 532.0, + "entropy": 1.2782762721180916, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26451998949050903, + "kl": 0.03907236340455711, + "learning_rate": 3e-05, + "loss": 0.17035090923309326, + "num_tokens": 694323.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.950176477432251, + "sampling/importance_sampling_ratio/mean": 0.45171743631362915, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46894216537475586, + "sampling/sampling_logp_difference/mean": 0.02863166108727455, + "step": 70, + "step_time": 19.817490340210497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 597.125, + "completions/mean_terminated_length": 597.125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2051330730319023, + "epoch": 0.142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5549097657203674, + "kl": 0.03670362115371972, + "learning_rate": 3e-05, + "loss": 0.4998631179332733, + "num_tokens": 705773.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.9404170513153076, + "sampling/importance_sampling_ratio/mean": 0.7522475123405457, + "sampling/importance_sampling_ratio/min": 0.05456363409757614, + "sampling/sampling_logp_difference/max": 0.4324514865875244, + "sampling/sampling_logp_difference/mean": 0.028340937569737434, + "step": 71, + "step_time": 41.66373607888818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 691.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 604.125, + "completions/mean_terminated_length": 604.125, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.18794547021389, + "epoch": 0.144, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10015435516834259, + "kl": 0.03911226138006896, + "learning_rate": 3e-05, + "loss": -0.02419177070260048, + "num_tokens": 717159.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1231037378311157, + "sampling/importance_sampling_ratio/mean": 0.4037884473800659, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6400742530822754, + "sampling/sampling_logp_difference/mean": 0.028367817401885986, + "step": 72, + "step_time": 23.86539705330506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 721.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 597.1875, + "completions/mean_terminated_length": 597.1875, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.2641174346208572, + "epoch": 0.146, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07918722927570343, + "kl": 0.03177848691120744, + "learning_rate": 3e-05, + "loss": -0.027635926380753517, + "num_tokens": 728402.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.5195796489715576, + "sampling/importance_sampling_ratio/mean": 0.4324396848678589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6819734573364258, + "sampling/sampling_logp_difference/mean": 0.030029678717255592, + "step": 73, + "step_time": 29.29490938829258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 640.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 574.375, + "completions/mean_terminated_length": 574.375, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3364054411649704, + "epoch": 0.148, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3853444755077362, + "kl": 0.03433372196741402, + "learning_rate": 3e-05, + "loss": -0.031142480671405792, + "num_tokens": 739632.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6853926181793213, + "sampling/importance_sampling_ratio/mean": 0.9200767874717712, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42021608352661133, + "sampling/sampling_logp_difference/mean": 0.030795859172940254, + "step": 74, + "step_time": 34.52008486771956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 618.25, + "completions/mean_terminated_length": 618.25, + "completions/min_length": 539.0, + "completions/min_terminated_length": 539.0, + "entropy": 1.365300178527832, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18101376295089722, + "kl": 0.02779634529724717, + "learning_rate": 3e-05, + "loss": -0.2718795835971832, + "num_tokens": 751164.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.8397568464279175, + "sampling/importance_sampling_ratio/mean": 0.5582400560379028, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42606019973754883, + "sampling/sampling_logp_difference/mean": 0.028895940631628036, + "step": 75, + "step_time": 18.76476171752438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 799.0, + "completions/max_terminated_length": 799.0, + "completions/mean_length": 603.375, + "completions/mean_terminated_length": 603.375, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2181015871465206, + "epoch": 0.152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1852622777223587, + "kl": 0.03176840906962752, + "learning_rate": 3e-05, + "loss": -0.10660596191883087, + "num_tokens": 762370.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.317336082458496, + "sampling/importance_sampling_ratio/mean": 0.550554633140564, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.481325626373291, + "sampling/sampling_logp_difference/mean": 0.028557566925883293, + "step": 76, + "step_time": 27.090129756834358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 572.9375, + "completions/mean_terminated_length": 572.9375, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2986655682325363, + "epoch": 0.154, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1531844586133957, + "kl": 0.03523328877054155, + "learning_rate": 3e-05, + "loss": -0.20856647193431854, + "num_tokens": 773169.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 2.855010986328125, + "sampling/importance_sampling_ratio/mean": 0.8239375352859497, + "sampling/importance_sampling_ratio/min": 0.1521405428647995, + "sampling/sampling_logp_difference/max": 0.4692528247833252, + "sampling/sampling_logp_difference/mean": 0.029906686395406723, + "step": 77, + "step_time": 25.004970545414835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 607.1875, + "completions/mean_terminated_length": 607.1875, + "completions/min_length": 540.0, + "completions/min_terminated_length": 540.0, + "entropy": 1.1003647185862064, + "epoch": 0.156, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14113759994506836, + "kl": 0.025135049945674837, + "learning_rate": 3e-05, + "loss": -0.10802068561315536, + "num_tokens": 784724.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.0096027851104736, + "sampling/importance_sampling_ratio/mean": 0.613497257232666, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4082927703857422, + "sampling/sampling_logp_difference/mean": 0.027884263545274734, + "step": 78, + "step_time": 29.614154959097505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 568.0625, + "completions/mean_terminated_length": 568.0625, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.266264721751213, + "epoch": 0.158, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18073000013828278, + "kl": 0.028119354974478483, + "learning_rate": 3e-05, + "loss": -0.0687609314918518, + "num_tokens": 795517.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.277299404144287, + "sampling/importance_sampling_ratio/mean": 0.3485238552093506, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4658241271972656, + "sampling/sampling_logp_difference/mean": 0.029626762494444847, + "step": 79, + "step_time": 23.5287338164635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 565.8125, + "completions/mean_terminated_length": 565.8125, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2773499339818954, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.31362995505332947, + "kl": 0.028471783734858036, + "learning_rate": 3e-05, + "loss": 0.059164684265851974, + "num_tokens": 806258.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.5347814559936523, + "sampling/importance_sampling_ratio/mean": 0.7027873396873474, + "sampling/importance_sampling_ratio/min": 0.06356429308652878, + "sampling/sampling_logp_difference/max": 0.4123659133911133, + "sampling/sampling_logp_difference/mean": 0.02946357987821102, + "step": 80, + "step_time": 24.251087154261768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 600.0, + "completions/max_terminated_length": 600.0, + "completions/mean_length": 536.1875, + "completions/mean_terminated_length": 536.1875, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.211024284362793, + "epoch": 0.162, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1959177404642105, + "kl": 0.02152467134874314, + "learning_rate": 3e-05, + "loss": 0.14755703508853912, + "num_tokens": 816717.0, + "reward": 6.0, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.764387369155884, + "sampling/importance_sampling_ratio/mean": 0.8167816400527954, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.28648805618286133, + "sampling/sampling_logp_difference/mean": 0.02814806066453457, + "step": 81, + "step_time": 22.752198832575232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 568.625, + "completions/mean_terminated_length": 568.625, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "entropy": 1.2584010139107704, + "epoch": 0.164, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16224367916584015, + "kl": 0.02812566957436502, + "learning_rate": 3e-05, + "loss": -0.1586945354938507, + "num_tokens": 827591.0, + "reward": 6.5, + "reward_std": 1.26491117477417, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.26491117477417, + "sampling/importance_sampling_ratio/max": 2.4172537326812744, + "sampling/importance_sampling_ratio/mean": 0.7026975154876709, + "sampling/importance_sampling_ratio/min": 0.1125984862446785, + "sampling/sampling_logp_difference/max": 0.3966444730758667, + "sampling/sampling_logp_difference/mean": 0.02937215007841587, + "step": 82, + "step_time": 22.57465209439397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 783.0, + "completions/max_terminated_length": 783.0, + "completions/mean_length": 583.5625, + "completions/mean_terminated_length": 583.5625, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2498536258935928, + "epoch": 0.166, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28551891446113586, + "kl": 0.023335880250670016, + "learning_rate": 3e-05, + "loss": 0.09868354350328445, + "num_tokens": 838760.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 1.9430233240127563, + "sampling/importance_sampling_ratio/mean": 0.7391272783279419, + "sampling/importance_sampling_ratio/min": 0.2032935619354248, + "sampling/sampling_logp_difference/max": 0.3390723466873169, + "sampling/sampling_logp_difference/mean": 0.02833949774503708, + "step": 83, + "step_time": 24.9633763525635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 581.0625, + "completions/mean_terminated_length": 581.0625, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.1582137942314148, + "epoch": 0.168, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1089889332652092, + "kl": 0.02546319324756041, + "learning_rate": 3e-05, + "loss": -0.04368923604488373, + "num_tokens": 849945.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.6800583600997925, + "sampling/importance_sampling_ratio/mean": 0.4864083528518677, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48818397521972656, + "sampling/sampling_logp_difference/mean": 0.02942320704460144, + "step": 84, + "step_time": 22.7196712391451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 553.3125, + "completions/mean_terminated_length": 553.3125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.1762890554964542, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18219847977161407, + "kl": 0.023275951389223337, + "learning_rate": 3e-05, + "loss": 0.055040232837200165, + "num_tokens": 860734.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.2940757274627686, + "sampling/importance_sampling_ratio/mean": 0.6381184458732605, + "sampling/importance_sampling_ratio/min": 0.08803392946720123, + "sampling/sampling_logp_difference/max": 0.3728243112564087, + "sampling/sampling_logp_difference/mean": 0.028103860095143318, + "step": 85, + "step_time": 28.569310082122684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 538.5, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.1561524420976639, + "epoch": 0.172, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2631295323371887, + "kl": 0.027657793601974845, + "learning_rate": 3e-05, + "loss": 0.019699495285749435, + "num_tokens": 871182.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.950981378555298, + "sampling/importance_sampling_ratio/mean": 0.5496660470962524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.358644962310791, + "sampling/sampling_logp_difference/mean": 0.02922222763299942, + "step": 86, + "step_time": 19.95468496438116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 688.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 528.625, + "completions/mean_terminated_length": 528.625, + "completions/min_length": 418.0, + "completions/min_terminated_length": 418.0, + "entropy": 1.382395550608635, + "epoch": 0.174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25233277678489685, + "kl": 0.025461523793637753, + "learning_rate": 3e-05, + "loss": -0.0012568621896207333, + "num_tokens": 881272.0, + "reward": 6.3125, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 1.940340280532837, + "sampling/importance_sampling_ratio/mean": 0.44232380390167236, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3433331251144409, + "sampling/sampling_logp_difference/mean": 0.030555889010429382, + "step": 87, + "step_time": 29.044239778537303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 537.0, + "completions/mean_terminated_length": 537.0, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.1740282103419304, + "epoch": 0.176, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1609290987253189, + "kl": 0.023582924506627023, + "learning_rate": 3e-05, + "loss": -0.0040507810190320015, + "num_tokens": 891608.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.8592076301574707, + "sampling/importance_sampling_ratio/mean": 0.4413543939590454, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3994784355163574, + "sampling/sampling_logp_difference/mean": 0.028627343475818634, + "step": 88, + "step_time": 24.642031190916896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 562.0, + "completions/mean_terminated_length": 562.0, + "completions/min_length": 490.0, + "completions/min_terminated_length": 490.0, + "entropy": 1.3354259580373764, + "epoch": 0.178, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25136521458625793, + "kl": 0.03104234568309039, + "learning_rate": 3e-05, + "loss": -0.10014888644218445, + "num_tokens": 902472.0, + "reward": 6.5, + "reward_std": 1.154700517654419, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.154700517654419, + "sampling/importance_sampling_ratio/max": 2.0172529220581055, + "sampling/importance_sampling_ratio/mean": 0.5528109073638916, + "sampling/importance_sampling_ratio/min": 0.031755149364471436, + "sampling/sampling_logp_difference/max": 0.48168420791625977, + "sampling/sampling_logp_difference/mean": 0.029525987803936005, + "step": 89, + "step_time": 23.934129936154932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 543.6875, + "completions/mean_terminated_length": 543.6875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.3072879239916801, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2416294664144516, + "kl": 0.02474795945454389, + "learning_rate": 3e-05, + "loss": 0.02994484454393387, + "num_tokens": 913003.0, + "reward": 6.125, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4122473001480103, + "sampling/importance_sampling_ratio/mean": 0.5672672390937805, + "sampling/importance_sampling_ratio/min": 0.1312582641839981, + "sampling/sampling_logp_difference/max": 0.36547136306762695, + "sampling/sampling_logp_difference/mean": 0.030115650966763496, + "step": 90, + "step_time": 16.719651044346392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 587.5, + "completions/mean_terminated_length": 587.5, + "completions/min_length": 491.0, + "completions/min_terminated_length": 491.0, + "entropy": 1.2642723061144352, + "epoch": 0.182, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11401186883449554, + "kl": 0.018764875654596835, + "learning_rate": 3e-05, + "loss": 0.17740829288959503, + "num_tokens": 923971.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.223915934562683, + "sampling/importance_sampling_ratio/mean": 0.4373893141746521, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.43628501892089844, + "sampling/sampling_logp_difference/mean": 0.027218280360102654, + "step": 91, + "step_time": 21.256958127953112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 519.3125, + "completions/mean_terminated_length": 519.3125, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.2509336844086647, + "epoch": 0.184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14000695943832397, + "kl": 0.017409664229489863, + "learning_rate": 3e-05, + "loss": -0.1092228814959526, + "num_tokens": 933880.0, + "reward": 7.125, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 2.4079525470733643, + "sampling/importance_sampling_ratio/mean": 0.6406391263008118, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3843420743942261, + "sampling/sampling_logp_difference/mean": 0.02841360680758953, + "step": 92, + "step_time": 20.99564675288275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 681.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 587.8125, + "completions/mean_terminated_length": 587.8125, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.1215453520417213, + "epoch": 0.186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16520461440086365, + "kl": 0.028165725176222622, + "learning_rate": 3e-05, + "loss": -0.15029624104499817, + "num_tokens": 945269.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.7825064659118652, + "sampling/importance_sampling_ratio/mean": 0.5902000069618225, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3571600914001465, + "sampling/sampling_logp_difference/mean": 0.02762974239885807, + "step": 93, + "step_time": 17.922352592926472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 566.0, + "completions/mean_terminated_length": 566.0, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.2864234894514084, + "epoch": 0.188, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24750091135501862, + "kl": 0.02070689742686227, + "learning_rate": 3e-05, + "loss": 0.2850193381309509, + "num_tokens": 956021.0, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "sampling/importance_sampling_ratio/max": 1.9146449565887451, + "sampling/importance_sampling_ratio/mean": 0.6849822402000427, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4927506446838379, + "sampling/sampling_logp_difference/mean": 0.029227914288640022, + "step": 94, + "step_time": 23.034203104209155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 516.6875, + "completions/mean_terminated_length": 516.6875, + "completions/min_length": 486.0, + "completions/min_terminated_length": 486.0, + "entropy": 1.2404684573411942, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11928244680166245, + "kl": 0.023086783126927912, + "learning_rate": 3e-05, + "loss": -0.032361116260290146, + "num_tokens": 965920.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.216193437576294, + "sampling/importance_sampling_ratio/mean": 0.32463955879211426, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37001991271972656, + "sampling/sampling_logp_difference/mean": 0.02843114361166954, + "step": 95, + "step_time": 15.103232022374868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 650.0, + "completions/max_terminated_length": 650.0, + "completions/mean_length": 548.4375, + "completions/mean_terminated_length": 548.4375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.2861761301755905, + "epoch": 0.192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2920665442943573, + "kl": 0.017841250344645232, + "learning_rate": 3e-05, + "loss": 0.1640928089618683, + "num_tokens": 976695.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.8954812288284302, + "sampling/importance_sampling_ratio/mean": 0.754618763923645, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31093156337738037, + "sampling/sampling_logp_difference/mean": 0.02842729538679123, + "step": 96, + "step_time": 40.74571412149817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 553.5625, + "completions/mean_terminated_length": 553.5625, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.1928813084959984, + "epoch": 0.194, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29983460903167725, + "kl": 0.020173200638964772, + "learning_rate": 3e-05, + "loss": 0.05926981568336487, + "num_tokens": 987120.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.0362496376037598, + "sampling/importance_sampling_ratio/mean": 0.6645779609680176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40996313095092773, + "sampling/sampling_logp_difference/mean": 0.02854262851178646, + "step": 97, + "step_time": 17.38945102225989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 546.1875, + "completions/mean_terminated_length": 546.1875, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "entropy": 1.365786962211132, + "epoch": 0.196, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12385546416044235, + "kl": 0.02262102661188692, + "learning_rate": 3e-05, + "loss": -0.09927551448345184, + "num_tokens": 997395.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2717533111572266, + "sampling/importance_sampling_ratio/mean": 0.5988417267799377, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35561084747314453, + "sampling/sampling_logp_difference/mean": 0.029298899695277214, + "step": 98, + "step_time": 23.35145698580891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 593.4375, + "completions/mean_terminated_length": 593.4375, + "completions/min_length": 508.0, + "completions/min_terminated_length": 508.0, + "entropy": 1.1754724085330963, + "epoch": 0.198, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2662176787853241, + "kl": 0.02589411090593785, + "learning_rate": 3e-05, + "loss": 0.2574020326137543, + "num_tokens": 1008458.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1308085918426514, + "sampling/importance_sampling_ratio/mean": 0.6420408487319946, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42550981044769287, + "sampling/sampling_logp_difference/mean": 0.02820964716374874, + "step": 99, + "step_time": 21.02622760878876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 538.4375, + "completions/mean_terminated_length": 538.4375, + "completions/min_length": 483.0, + "completions/min_terminated_length": 483.0, + "entropy": 1.3136962801218033, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2549664378166199, + "kl": 0.024644543533213437, + "learning_rate": 3e-05, + "loss": 0.06747792661190033, + "num_tokens": 1018793.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.3327062129974365, + "sampling/importance_sampling_ratio/mean": 0.7165549993515015, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4304838180541992, + "sampling/sampling_logp_difference/mean": 0.0299112256616354, + "step": 100, + "step_time": 16.768271412234753 + } + ], + "logging_steps": 1, + "max_steps": 300, + "num_input_tokens_seen": 1018793, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/outputs/E0-baseline/checkpoint-100/training_args.bin b/outputs/E0-baseline/checkpoint-100/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c3a2693682d14c331dbd12e8872552fc28d54b49 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-100/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6 +size 7697 diff --git a/outputs/E0-baseline/checkpoint-150/README.md b/outputs/E0-baseline/checkpoint-150/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88990785a4695219513364eafae195070ddd3182 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-150/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507 +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.20.0 \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-150/adapter_config.json b/outputs/E0-baseline/checkpoint-150/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3b383bef044f277e3ff79f0fc5addabd3c441593 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-150/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.20.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-150/adapter_model.safetensors b/outputs/E0-baseline/checkpoint-150/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c108fea9a1394e43e22bda548581f650eb16fd08 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-150/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbafc11133a226488bab7d56c436ec4755811aa888b20c9438b4949404543856 +size 264308896 diff --git a/outputs/E0-baseline/checkpoint-150/chat_template.jinja b/outputs/E0-baseline/checkpoint-150/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-150/chat_template.jinja @@ -0,0 +1,61 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-150/tokenizer.json b/outputs/E0-baseline/checkpoint-150/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-150/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/outputs/E0-baseline/checkpoint-150/tokenizer_config.json b/outputs/E0-baseline/checkpoint-150/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..86d67326516569b4f1cec3ac7b51b8903062c648 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-150/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/outputs/E0-baseline/checkpoint-150/trainer_state.json b/outputs/E0-baseline/checkpoint-150/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e8225480713af49fb142cc8675f18e99b1b26119 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-150/trainer_state.json @@ -0,0 +1,4984 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3, + "eval_steps": 500, + "global_step": 150, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 529.0, + "completions/mean_terminated_length": 529.0, + "completions/min_length": 482.0, + "completions/min_terminated_length": 482.0, + "entropy": 1.2025159299373627, + "epoch": 0.002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22668755054473877, + "kl": 0.0, + "learning_rate": 0.0, + "loss": 0.2398601919412613, + "num_tokens": 10400.0, + "reward": 6.375, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6477458477020264, + "sampling/importance_sampling_ratio/mean": 0.5064857602119446, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40894174575805664, + "sampling/sampling_logp_difference/mean": 0.026567919179797173, + "step": 1, + "step_time": 12.760562099982053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 644.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 562.25, + "completions/mean_terminated_length": 562.25, + "completions/min_length": 497.0, + "completions/min_terminated_length": 497.0, + "entropy": 1.1930748969316483, + "epoch": 0.004, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12967805564403534, + "kl": 0.0, + "learning_rate": 3e-06, + "loss": -0.08571265637874603, + "num_tokens": 20924.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 1.0037232637405396, + "sampling/importance_sampling_ratio/mean": 0.41275694966316223, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45699238777160645, + "sampling/sampling_logp_difference/mean": 0.025086138397455215, + "step": 2, + "step_time": 11.406366533134133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 483.625, + "completions/mean_terminated_length": 483.625, + "completions/min_length": 407.0, + "completions/min_terminated_length": 407.0, + "entropy": 1.1096689626574516, + "epoch": 0.006, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22966289520263672, + "kl": 0.0008355328354809899, + "learning_rate": 6e-06, + "loss": 0.020913563668727875, + "num_tokens": 30222.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.927120566368103, + "sampling/importance_sampling_ratio/mean": 0.6114993095397949, + "sampling/importance_sampling_ratio/min": 0.11067161709070206, + "sampling/sampling_logp_difference/max": 0.4620504379272461, + "sampling/sampling_logp_difference/mean": 0.024864500388503075, + "step": 3, + "step_time": 26.480680393986404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 608.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 524.75, + "completions/mean_terminated_length": 524.75, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.211122527718544, + "epoch": 0.008, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30378466844558716, + "kl": 0.0008403196770814247, + "learning_rate": 9e-06, + "loss": -0.12959149479866028, + "num_tokens": 40410.0, + "reward": 6.25, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.406888961791992, + "sampling/importance_sampling_ratio/mean": 0.5457419753074646, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3680229187011719, + "sampling/sampling_logp_difference/mean": 0.02656388282775879, + "step": 4, + "step_time": 22.95301443943754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 487.5625, + "completions/mean_terminated_length": 487.5625, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "entropy": 1.247180238366127, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5199307799339294, + "kl": 0.0008723233368073124, + "learning_rate": 1.2e-05, + "loss": 0.11524428427219391, + "num_tokens": 49915.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.1544158458709717, + "sampling/importance_sampling_ratio/mean": 0.963020920753479, + "sampling/importance_sampling_ratio/min": 0.04948288947343826, + "sampling/sampling_logp_difference/max": 0.4774587154388428, + "sampling/sampling_logp_difference/mean": 0.02625642716884613, + "step": 5, + "step_time": 22.37928077671677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 533.0, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 394.0, + "completions/min_terminated_length": 394.0, + "entropy": 1.1693861335515976, + "epoch": 0.012, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27643197774887085, + "kl": 0.0009261858467652928, + "learning_rate": 1.5e-05, + "loss": 0.15093231201171875, + "num_tokens": 60219.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 2.125091791152954, + "sampling/importance_sampling_ratio/mean": 0.7733402252197266, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7049552202224731, + "sampling/sampling_logp_difference/mean": 0.025986071676015854, + "step": 6, + "step_time": 21.00841654697433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 505.6875, + "completions/mean_terminated_length": 505.6875, + "completions/min_length": 425.0, + "completions/min_terminated_length": 425.0, + "entropy": 1.2473183795809746, + "epoch": 0.014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2166663259267807, + "kl": 0.0009701631606731098, + "learning_rate": 1.8e-05, + "loss": -0.08582288026809692, + "num_tokens": 69902.0, + "reward": 7.1875, + "reward_std": 0.75, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.8287386894226074, + "sampling/importance_sampling_ratio/mean": 0.5286832451820374, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.38030898571014404, + "sampling/sampling_logp_difference/mean": 0.0264718160033226, + "step": 7, + "step_time": 46.596127359196544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.1040107272565365, + "epoch": 0.016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18874908983707428, + "kl": 0.0010721117541834246, + "learning_rate": 2.1e-05, + "loss": -0.1946130096912384, + "num_tokens": 79501.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.3561792373657227, + "sampling/importance_sampling_ratio/mean": 0.6918502449989319, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.26114892959594727, + "sampling/sampling_logp_difference/mean": 0.02554757334291935, + "step": 8, + "step_time": 25.433595282491297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 554.0, + "completions/max_terminated_length": 554.0, + "completions/mean_length": 490.1875, + "completions/mean_terminated_length": 490.1875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1377170644700527, + "epoch": 0.018, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.204215869307518, + "kl": 0.002002388624532614, + "learning_rate": 2.4e-05, + "loss": -0.08130021393299103, + "num_tokens": 88976.0, + "reward": 6.5, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.0982444286346436, + "sampling/importance_sampling_ratio/mean": 0.5873216986656189, + "sampling/importance_sampling_ratio/min": 0.04890051856637001, + "sampling/sampling_logp_difference/max": 0.8512144088745117, + "sampling/sampling_logp_difference/mean": 0.02638406865298748, + "step": 9, + "step_time": 18.427699581719935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 500.1875, + "completions/mean_terminated_length": 500.1875, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.1230391450226307, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1985069215297699, + "kl": 0.0026735300780273974, + "learning_rate": 2.7000000000000002e-05, + "loss": -0.12387816607952118, + "num_tokens": 98603.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.983199954032898, + "sampling/importance_sampling_ratio/mean": 0.5639468431472778, + "sampling/importance_sampling_ratio/min": 0.012905921787023544, + "sampling/sampling_logp_difference/max": 0.3653905391693115, + "sampling/sampling_logp_difference/mean": 0.025383003056049347, + "step": 10, + "step_time": 14.99981931829825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 459.6875, + "completions/mean_terminated_length": 459.6875, + "completions/min_length": 379.0, + "completions/min_terminated_length": 379.0, + "entropy": 1.213625729084015, + "epoch": 0.022, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3695620596408844, + "kl": 0.00424640768324025, + "learning_rate": 3e-05, + "loss": -0.06913074851036072, + "num_tokens": 107734.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.000136375427246, + "sampling/importance_sampling_ratio/mean": 0.8144867420196533, + "sampling/importance_sampling_ratio/min": 0.06302778422832489, + "sampling/sampling_logp_difference/max": 0.3647327423095703, + "sampling/sampling_logp_difference/mean": 0.026674197986721992, + "step": 11, + "step_time": 30.46549107739702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.191277615725994, + "epoch": 0.024, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35457733273506165, + "kl": 0.007200263120466843, + "learning_rate": 3e-05, + "loss": 0.22097699344158173, + "num_tokens": 117199.0, + "reward": 5.9375, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 1.5660414695739746, + "sampling/importance_sampling_ratio/mean": 0.649204432964325, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3921785354614258, + "sampling/sampling_logp_difference/mean": 0.02726689912378788, + "step": 12, + "step_time": 15.719243939965963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 464.3125, + "completions/mean_terminated_length": 464.3125, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "entropy": 1.24251464381814, + "epoch": 0.026, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30047014355659485, + "kl": 0.0058551667898427695, + "learning_rate": 3e-05, + "loss": -0.07746122777462006, + "num_tokens": 126556.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6028401851654053, + "sampling/importance_sampling_ratio/mean": 0.7561360597610474, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4946432113647461, + "sampling/sampling_logp_difference/mean": 0.02639186754822731, + "step": 13, + "step_time": 18.08984712138772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 444.8125, + "completions/mean_terminated_length": 444.8125, + "completions/min_length": 389.0, + "completions/min_terminated_length": 389.0, + "entropy": 1.1501125395298004, + "epoch": 0.028, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24105942249298096, + "kl": 0.012796793889719993, + "learning_rate": 3e-05, + "loss": 0.10855278372764587, + "num_tokens": 135417.0, + "reward": 3.1875, + "reward_std": 3.310966730117798, + "rewards/quality_reward/mean": 3.1875, + "rewards/quality_reward/std": 3.310966730117798, + "sampling/importance_sampling_ratio/max": 2.541518211364746, + "sampling/importance_sampling_ratio/mean": 0.5903321504592896, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.8390979766845703, + "sampling/sampling_logp_difference/mean": 0.02838001400232315, + "step": 14, + "step_time": 20.055794408079237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 497.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 442.25, + "completions/mean_terminated_length": 442.25, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 1.1262825280427933, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19040776789188385, + "kl": 0.010701361010433175, + "learning_rate": 3e-05, + "loss": -0.007966680452227592, + "num_tokens": 144205.0, + "reward": 5.875, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.0754293203353882, + "sampling/importance_sampling_ratio/mean": 0.41446638107299805, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3982200622558594, + "sampling/sampling_logp_difference/mean": 0.027210108935832977, + "step": 15, + "step_time": 14.176074750721455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 478.125, + "completions/mean_terminated_length": 478.125, + "completions/min_length": 433.0, + "completions/min_terminated_length": 433.0, + "entropy": 1.2985924184322357, + "epoch": 0.032, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23625652492046356, + "kl": 0.0213887135614641, + "learning_rate": 3e-05, + "loss": -0.21546132862567902, + "num_tokens": 153543.0, + "reward": 6.5, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 2.0074336528778076, + "sampling/importance_sampling_ratio/mean": 0.49076417088508606, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5095968246459961, + "sampling/sampling_logp_difference/mean": 0.028833162039518356, + "step": 16, + "step_time": 15.848205763846636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 469.25, + "completions/mean_terminated_length": 469.25, + "completions/min_length": 423.0, + "completions/min_terminated_length": 423.0, + "entropy": 1.3148910626769066, + "epoch": 0.034, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17765119671821594, + "kl": 0.02128879475640133, + "learning_rate": 3e-05, + "loss": -0.0828079879283905, + "num_tokens": 163043.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 1.5988941192626953, + "sampling/importance_sampling_ratio/mean": 0.5021570324897766, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.591062068939209, + "sampling/sampling_logp_difference/mean": 0.030804751440882683, + "step": 17, + "step_time": 28.313011147081852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 447.0, + "completions/mean_terminated_length": 447.0, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 1.390664003789425, + "epoch": 0.036, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37244123220443726, + "kl": 0.019650122558232397, + "learning_rate": 3e-05, + "loss": -0.01184748113155365, + "num_tokens": 172379.0, + "reward": 6.0, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.675238847732544, + "sampling/importance_sampling_ratio/mean": 0.9581233263015747, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5762512683868408, + "sampling/sampling_logp_difference/mean": 0.03126702085137367, + "step": 18, + "step_time": 30.991567107848823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 498.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 447.75, + "completions/mean_terminated_length": 447.75, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 1.3287223279476166, + "epoch": 0.038, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23734751343727112, + "kl": 0.022738213243428618, + "learning_rate": 3e-05, + "loss": 0.040024783462285995, + "num_tokens": 181239.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.095722198486328, + "sampling/importance_sampling_ratio/mean": 0.6408629417419434, + "sampling/importance_sampling_ratio/min": 0.1203346848487854, + "sampling/sampling_logp_difference/max": 0.4722297191619873, + "sampling/sampling_logp_difference/mean": 0.030378391966223717, + "step": 19, + "step_time": 18.615950418636203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 452.3125, + "completions/mean_terminated_length": 452.3125, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 1.1001618690788746, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34637194871902466, + "kl": 0.015380491153337061, + "learning_rate": 3e-05, + "loss": 0.1691148728132248, + "num_tokens": 190068.0, + "reward": 6.125, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.6566128730773926, + "sampling/importance_sampling_ratio/mean": 0.7767290472984314, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.6531033515930176, + "sampling/sampling_logp_difference/mean": 0.030463142320513725, + "step": 20, + "step_time": 15.741292077116668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 457.5625, + "completions/mean_terminated_length": 457.5625, + "completions/min_length": 390.0, + "completions/min_terminated_length": 390.0, + "entropy": 1.3708399310708046, + "epoch": 0.042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2514922618865967, + "kl": 0.018277494702488184, + "learning_rate": 3e-05, + "loss": -0.13425321877002716, + "num_tokens": 198941.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.5612297058105469, + "sampling/importance_sampling_ratio/mean": 0.601022481918335, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4818992614746094, + "sampling/sampling_logp_difference/mean": 0.0315740592777729, + "step": 21, + "step_time": 17.24192419089377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 421.4375, + "completions/mean_terminated_length": 421.4375, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 1.136269599199295, + "epoch": 0.044, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2268020063638687, + "kl": 0.017973962530959398, + "learning_rate": 3e-05, + "loss": 0.010622119531035423, + "num_tokens": 207300.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.7986358404159546, + "sampling/importance_sampling_ratio/mean": 0.46136727929115295, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5335149765014648, + "sampling/sampling_logp_difference/mean": 0.02804401144385338, + "step": 22, + "step_time": 19.37282825494185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 452.6875, + "completions/mean_terminated_length": 452.6875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3299130946397781, + "epoch": 0.046, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33994877338409424, + "kl": 0.021804221265483648, + "learning_rate": 3e-05, + "loss": -0.24404363334178925, + "num_tokens": 216343.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.452249526977539, + "sampling/importance_sampling_ratio/mean": 0.747193455696106, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6328549385070801, + "sampling/sampling_logp_difference/mean": 0.02918298915028572, + "step": 23, + "step_time": 15.356728344224393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 462.4375, + "completions/mean_terminated_length": 462.4375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.379701942205429, + "epoch": 0.048, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3076202869415283, + "kl": 0.020299295720178634, + "learning_rate": 3e-05, + "loss": -0.09123071283102036, + "num_tokens": 225550.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.5845099687576294, + "sampling/importance_sampling_ratio/mean": 0.6625345945358276, + "sampling/importance_sampling_ratio/min": 0.04495502635836601, + "sampling/sampling_logp_difference/max": 0.36547183990478516, + "sampling/sampling_logp_difference/mean": 0.028946854174137115, + "step": 24, + "step_time": 14.95462113339454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.2471638694405556, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22180576622486115, + "kl": 0.018309170845896006, + "learning_rate": 3e-05, + "loss": -0.1412944793701172, + "num_tokens": 235005.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.6407876014709473, + "sampling/importance_sampling_ratio/mean": 0.7712795734405518, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4363563060760498, + "sampling/sampling_logp_difference/mean": 0.02898716926574707, + "step": 25, + "step_time": 16.46686205593869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 455.75, + "completions/mean_terminated_length": 455.75, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 1.28530602902174, + "epoch": 0.052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37535253167152405, + "kl": 0.020504546992015094, + "learning_rate": 3e-05, + "loss": 0.10839397460222244, + "num_tokens": 243953.0, + "reward": 6.25, + "reward_std": 1.0645813941955566, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.0645813941955566, + "sampling/importance_sampling_ratio/max": 2.0567266941070557, + "sampling/importance_sampling_ratio/mean": 0.5800145864486694, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4523625373840332, + "sampling/sampling_logp_difference/mean": 0.027610119432210922, + "step": 26, + "step_time": 22.121026155073196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.0, + "completions/max_terminated_length": 514.0, + "completions/mean_length": 462.75, + "completions/mean_terminated_length": 462.75, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.2253629937767982, + "epoch": 0.054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2340388149023056, + "kl": 0.020236384589225054, + "learning_rate": 3e-05, + "loss": 0.04823978245258331, + "num_tokens": 253237.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.1560932397842407, + "sampling/importance_sampling_ratio/mean": 0.40753045678138733, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35022830963134766, + "sampling/sampling_logp_difference/mean": 0.028367744758725166, + "step": 27, + "step_time": 15.10967448912561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 538.0, + "completions/max_terminated_length": 538.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "entropy": 1.3779077678918839, + "epoch": 0.056, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5531017184257507, + "kl": 0.01706669357372448, + "learning_rate": 3e-05, + "loss": 0.0933581069111824, + "num_tokens": 262454.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.5711586475372314, + "sampling/importance_sampling_ratio/mean": 0.7730721831321716, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.343827486038208, + "sampling/sampling_logp_difference/mean": 0.028883326798677444, + "step": 28, + "step_time": 38.59647103771567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 477.0, + "completions/mean_terminated_length": 477.0, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.3791070505976677, + "epoch": 0.058, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33771538734436035, + "kl": 0.02141271048458293, + "learning_rate": 3e-05, + "loss": 0.010216176509857178, + "num_tokens": 271918.0, + "reward": 5.75, + "reward_std": 1.2909945249557495, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.2909945249557495, + "sampling/importance_sampling_ratio/max": 2.4999797344207764, + "sampling/importance_sampling_ratio/mean": 1.0250636339187622, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3258817195892334, + "sampling/sampling_logp_difference/mean": 0.029029540717601776, + "step": 29, + "step_time": 23.610272648278624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 471.1875, + "completions/mean_terminated_length": 471.1875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.4652926102280617, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21955685317516327, + "kl": 0.019562674744520336, + "learning_rate": 3e-05, + "loss": -0.014814459718763828, + "num_tokens": 281305.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 2.802098274230957, + "sampling/importance_sampling_ratio/mean": 0.866391658782959, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6354451179504395, + "sampling/sampling_logp_difference/mean": 0.03177585452795029, + "step": 30, + "step_time": 16.961607525125146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 490.5, + "completions/mean_terminated_length": 490.5, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 1.1957123205065727, + "epoch": 0.062, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12176825106143951, + "kl": 0.026934220048133284, + "learning_rate": 3e-05, + "loss": -0.08307373523712158, + "num_tokens": 291409.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.610858678817749, + "sampling/importance_sampling_ratio/mean": 0.6937527656555176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4253416061401367, + "sampling/sampling_logp_difference/mean": 0.02871524728834629, + "step": 31, + "step_time": 15.012207658961415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 462.0, + "completions/mean_terminated_length": 462.0, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3018206283450127, + "epoch": 0.064, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4979296624660492, + "kl": 0.03539742121938616, + "learning_rate": 3e-05, + "loss": 0.0017175457905977964, + "num_tokens": 300649.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8527640104293823, + "sampling/importance_sampling_ratio/mean": 0.7824680209159851, + "sampling/importance_sampling_ratio/min": 0.07447884231805801, + "sampling/sampling_logp_difference/max": 0.5221483707427979, + "sampling/sampling_logp_difference/mean": 0.029107660055160522, + "step": 32, + "step_time": 22.78309725271538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 474.0, + "completions/mean_terminated_length": 474.0, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.321175642311573, + "epoch": 0.066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22016967833042145, + "kl": 0.029592803912237287, + "learning_rate": 3e-05, + "loss": 0.05625719577074051, + "num_tokens": 309889.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.9875097274780273, + "sampling/importance_sampling_ratio/mean": 0.7832435369491577, + "sampling/importance_sampling_ratio/min": 0.18006731569766998, + "sampling/sampling_logp_difference/max": 0.38585615158081055, + "sampling/sampling_logp_difference/mean": 0.027828343212604523, + "step": 33, + "step_time": 44.51360382232815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 459.9375, + "completions/mean_terminated_length": 459.9375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.2222414836287498, + "epoch": 0.068, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2873040437698364, + "kl": 0.02840927499346435, + "learning_rate": 3e-05, + "loss": -0.037432070821523666, + "num_tokens": 318904.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 2.6647069454193115, + "sampling/importance_sampling_ratio/mean": 0.5744763016700745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.41987133026123047, + "sampling/sampling_logp_difference/mean": 0.029470665380358696, + "step": 34, + "step_time": 133.03877451224253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 522.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 462.1875, + "completions/mean_terminated_length": 462.1875, + "completions/min_length": 413.0, + "completions/min_terminated_length": 413.0, + "entropy": 1.1665974482893944, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2490653246641159, + "kl": 0.025841041060630232, + "learning_rate": 3e-05, + "loss": -0.20422951877117157, + "num_tokens": 328011.0, + "reward": 6.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 2.510730266571045, + "sampling/importance_sampling_ratio/mean": 0.7954420447349548, + "sampling/importance_sampling_ratio/min": 0.020566223189234734, + "sampling/sampling_logp_difference/max": 0.36430132389068604, + "sampling/sampling_logp_difference/mean": 0.026844775304198265, + "step": 35, + "step_time": 21.5843787365593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 549.0, + "completions/max_terminated_length": 549.0, + "completions/mean_length": 492.0, + "completions/mean_terminated_length": 492.0, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3467887043952942, + "epoch": 0.072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15268851816654205, + "kl": 0.023660800594370812, + "learning_rate": 3e-05, + "loss": -0.11188814043998718, + "num_tokens": 337731.0, + "reward": 6.5625, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.6270908117294312, + "sampling/importance_sampling_ratio/mean": 0.614537239074707, + "sampling/importance_sampling_ratio/min": 0.10853960365056992, + "sampling/sampling_logp_difference/max": 0.4023854732513428, + "sampling/sampling_logp_difference/mean": 0.028961554169654846, + "step": 36, + "step_time": 18.843947287183255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.0, + "completions/max_terminated_length": 513.0, + "completions/mean_length": 460.75, + "completions/mean_terminated_length": 460.75, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "entropy": 1.2476315572857857, + "epoch": 0.074, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42548227310180664, + "kl": 0.022181478852871805, + "learning_rate": 3e-05, + "loss": 0.37223517894744873, + "num_tokens": 346815.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.6624510288238525, + "sampling/importance_sampling_ratio/mean": 0.7942180633544922, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4037543535232544, + "sampling/sampling_logp_difference/mean": 0.028780322521924973, + "step": 37, + "step_time": 42.04662919091061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 452.0, + "completions/mean_terminated_length": 452.0, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "entropy": 1.1745503433048725, + "epoch": 0.076, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16980381309986115, + "kl": 0.017483733594417572, + "learning_rate": 3e-05, + "loss": -0.09029137343168259, + "num_tokens": 355711.0, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "sampling/importance_sampling_ratio/max": 2.3201518058776855, + "sampling/importance_sampling_ratio/mean": 0.721072793006897, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.30433225631713867, + "sampling/sampling_logp_difference/mean": 0.026646045967936516, + "step": 38, + "step_time": 38.63075139513239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 466.8125, + "completions/mean_terminated_length": 466.8125, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2664988860487938, + "epoch": 0.078, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21305795013904572, + "kl": 0.021121050289366394, + "learning_rate": 3e-05, + "loss": -0.03154226019978523, + "num_tokens": 364860.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.0297553539276123, + "sampling/importance_sampling_ratio/mean": 0.674609363079071, + "sampling/importance_sampling_ratio/min": 0.11245065927505493, + "sampling/sampling_logp_difference/max": 0.37443917989730835, + "sampling/sampling_logp_difference/mean": 0.028257746249437332, + "step": 39, + "step_time": 30.028073193039745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 485.1875, + "completions/mean_terminated_length": 485.1875, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.3458357080817223, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12235487997531891, + "kl": 0.018535695446189493, + "learning_rate": 3e-05, + "loss": -0.035816628485918045, + "num_tokens": 374607.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.242042303085327, + "sampling/importance_sampling_ratio/mean": 0.5344723463058472, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36985254287719727, + "sampling/sampling_logp_difference/mean": 0.029600802809000015, + "step": 40, + "step_time": 15.446288945619017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 498.875, + "completions/mean_terminated_length": 498.875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3402792811393738, + "epoch": 0.082, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15642686188220978, + "kl": 0.013967045990284532, + "learning_rate": 3e-05, + "loss": 0.0011727213859558105, + "num_tokens": 384317.0, + "reward": 6.375, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5691092014312744, + "sampling/importance_sampling_ratio/mean": 0.3696203827857971, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6487679481506348, + "sampling/sampling_logp_difference/mean": 0.03018251620233059, + "step": 41, + "step_time": 18.15720977121964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.3317564576864243, + "epoch": 0.084, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3236794173717499, + "kl": 0.01707366103073582, + "learning_rate": 3e-05, + "loss": 0.10363321751356125, + "num_tokens": 393934.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.0568668842315674, + "sampling/importance_sampling_ratio/mean": 0.6415404677391052, + "sampling/importance_sampling_ratio/min": 0.07682990282773972, + "sampling/sampling_logp_difference/max": 0.7769032716751099, + "sampling/sampling_logp_difference/mean": 0.02936164103448391, + "step": 42, + "step_time": 15.066733688581735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 510.9375, + "completions/mean_terminated_length": 510.9375, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2099780030548573, + "epoch": 0.086, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1771102100610733, + "kl": 0.01784718461567536, + "learning_rate": 3e-05, + "loss": -0.027566466480493546, + "num_tokens": 403893.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.3430161476135254, + "sampling/importance_sampling_ratio/mean": 0.7769261598587036, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36153650283813477, + "sampling/sampling_logp_difference/mean": 0.028799494728446007, + "step": 43, + "step_time": 16.90863296529278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 494.1875, + "completions/mean_terminated_length": 494.1875, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "entropy": 1.2924985438585281, + "epoch": 0.088, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3432070314884186, + "kl": 0.014529847539961338, + "learning_rate": 3e-05, + "loss": -0.04157561436295509, + "num_tokens": 413312.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.726816177368164, + "sampling/importance_sampling_ratio/mean": 0.8989821672439575, + "sampling/importance_sampling_ratio/min": 0.07410597801208496, + "sampling/sampling_logp_difference/max": 0.31444358825683594, + "sampling/sampling_logp_difference/mean": 0.028122060000896454, + "step": 44, + "step_time": 16.7880685236305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 543.5, + "completions/mean_terminated_length": 543.5, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.206408604979515, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36233776807785034, + "kl": 0.015796773659531027, + "learning_rate": 3e-05, + "loss": 0.029176680371165276, + "num_tokens": 423624.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.677195429801941, + "sampling/importance_sampling_ratio/mean": 0.6537867188453674, + "sampling/importance_sampling_ratio/min": 0.09822077304124832, + "sampling/sampling_logp_difference/max": 0.31381869316101074, + "sampling/sampling_logp_difference/mean": 0.02717999368906021, + "step": 45, + "step_time": 23.35960763087496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 643.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 534.5625, + "completions/mean_terminated_length": 534.5625, + "completions/min_length": 453.0, + "completions/min_terminated_length": 453.0, + "entropy": 1.2577891275286674, + "epoch": 0.092, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20219561457633972, + "kl": 0.014481160265859216, + "learning_rate": 3e-05, + "loss": 0.18850615620613098, + "num_tokens": 433817.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.203894853591919, + "sampling/importance_sampling_ratio/mean": 0.697495698928833, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35237741470336914, + "sampling/sampling_logp_difference/mean": 0.02723248302936554, + "step": 46, + "step_time": 19.56032704282552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 538.25, + "completions/mean_terminated_length": 538.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.3271892964839935, + "epoch": 0.094, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22262753546237946, + "kl": 0.012554377142805606, + "learning_rate": 3e-05, + "loss": 0.06984467804431915, + "num_tokens": 444045.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 2.2261502742767334, + "sampling/importance_sampling_ratio/mean": 0.6712950468063354, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4496123790740967, + "sampling/sampling_logp_difference/mean": 0.028838323429226875, + "step": 47, + "step_time": 21.226045075803995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 514.25, + "completions/mean_terminated_length": 514.25, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "entropy": 1.1054812744259834, + "epoch": 0.096, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3244606554508209, + "kl": 0.0157591889728792, + "learning_rate": 3e-05, + "loss": 0.09024985134601593, + "num_tokens": 453945.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.6745388507843018, + "sampling/importance_sampling_ratio/mean": 0.7607913017272949, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650559186935425, + "sampling/sampling_logp_difference/mean": 0.026116060093045235, + "step": 48, + "step_time": 17.565261672716588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 585.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 502.4375, + "completions/mean_terminated_length": 502.4375, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.2281213253736496, + "epoch": 0.098, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4033137857913971, + "kl": 0.015613102470524609, + "learning_rate": 3e-05, + "loss": -0.2898017466068268, + "num_tokens": 463576.0, + "reward": 6.4375, + "reward_std": 1.2632629871368408, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.2632629871368408, + "sampling/importance_sampling_ratio/max": 2.6974196434020996, + "sampling/importance_sampling_ratio/mean": 0.7124314308166504, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3675417900085449, + "sampling/sampling_logp_difference/mean": 0.028341906145215034, + "step": 49, + "step_time": 29.838082558009773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 530.1875, + "completions/mean_terminated_length": 530.1875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.1955150067806244, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23144562542438507, + "kl": 0.01374751579714939, + "learning_rate": 3e-05, + "loss": -0.19065965712070465, + "num_tokens": 473915.0, + "reward": 6.3125, + "reward_std": 1.1954776048660278, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.1954776048660278, + "sampling/importance_sampling_ratio/max": 2.54063081741333, + "sampling/importance_sampling_ratio/mean": 0.8162041902542114, + "sampling/importance_sampling_ratio/min": 0.1628064066171646, + "sampling/sampling_logp_difference/max": 0.2777421474456787, + "sampling/sampling_logp_difference/mean": 0.02853373810648918, + "step": 50, + "step_time": 16.047010839451104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 549.875, + "completions/mean_terminated_length": 549.875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1870752647519112, + "epoch": 0.102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4241364896297455, + "kl": 0.013923745544161648, + "learning_rate": 3e-05, + "loss": -0.1761355698108673, + "num_tokens": 484577.0, + "reward": 6.5625, + "reward_std": 1.0935417413711548, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 1.0935417413711548, + "sampling/importance_sampling_ratio/max": 2.929507255554199, + "sampling/importance_sampling_ratio/mean": 0.6905896663665771, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5649824142456055, + "sampling/sampling_logp_difference/mean": 0.028025619685649872, + "step": 51, + "step_time": 44.488836522214115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 552.375, + "completions/mean_terminated_length": 552.375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.098166175186634, + "epoch": 0.104, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.32851356267929077, + "kl": 0.01297539210645482, + "learning_rate": 3e-05, + "loss": -0.06503897905349731, + "num_tokens": 495015.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.9652340412139893, + "sampling/importance_sampling_ratio/mean": 0.9612224102020264, + "sampling/importance_sampling_ratio/min": 0.040177375078201294, + "sampling/sampling_logp_difference/max": 0.3454720973968506, + "sampling/sampling_logp_difference/mean": 0.02687419019639492, + "step": 52, + "step_time": 20.21497478755191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 578.375, + "completions/mean_terminated_length": 578.375, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2311968132853508, + "epoch": 0.106, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21921825408935547, + "kl": 0.017025968758389354, + "learning_rate": 3e-05, + "loss": -0.18975484371185303, + "num_tokens": 505909.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.0094237327575684, + "sampling/importance_sampling_ratio/mean": 0.5587533116340637, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.39138758182525635, + "sampling/sampling_logp_difference/mean": 0.028095029294490814, + "step": 53, + "step_time": 31.140278964303434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 473.25, + "completions/mean_terminated_length": 473.25, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 1.2682743221521378, + "epoch": 0.108, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27410373091697693, + "kl": 0.018500705540645868, + "learning_rate": 3e-05, + "loss": 0.14847250282764435, + "num_tokens": 515073.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.875750780105591, + "sampling/importance_sampling_ratio/mean": 0.7429271936416626, + "sampling/importance_sampling_ratio/min": 0.09779425710439682, + "sampling/sampling_logp_difference/max": 0.5433444976806641, + "sampling/sampling_logp_difference/mean": 0.02810005284845829, + "step": 54, + "step_time": 18.365382733754814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 549.0, + "completions/mean_terminated_length": 549.0, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.3073157891631126, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29323700070381165, + "kl": 0.016703857632819563, + "learning_rate": 3e-05, + "loss": 0.05967015400528908, + "num_tokens": 525377.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 1.4832638502120972, + "sampling/importance_sampling_ratio/mean": 0.6094669103622437, + "sampling/importance_sampling_ratio/min": 0.08072065562009811, + "sampling/sampling_logp_difference/max": 0.39328718185424805, + "sampling/sampling_logp_difference/mean": 0.02906947024166584, + "step": 55, + "step_time": 30.47015379369259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 576.625, + "completions/mean_terminated_length": 576.625, + "completions/min_length": 500.0, + "completions/min_terminated_length": 500.0, + "entropy": 1.2820357605814934, + "epoch": 0.112, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.200644388794899, + "kl": 0.018819268501829356, + "learning_rate": 3e-05, + "loss": -0.04828515648841858, + "num_tokens": 536163.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.3941831588745117, + "sampling/importance_sampling_ratio/mean": 0.5633801221847534, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3664684295654297, + "sampling/sampling_logp_difference/mean": 0.02962428703904152, + "step": 56, + "step_time": 16.172011949121952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 640.25, + "completions/mean_terminated_length": 640.25, + "completions/min_length": 556.0, + "completions/min_terminated_length": 556.0, + "entropy": 1.4191219061613083, + "epoch": 0.114, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19633841514587402, + "kl": 0.02060725452611223, + "learning_rate": 3e-05, + "loss": -0.12029920518398285, + "num_tokens": 548143.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.3877830505371094, + "sampling/importance_sampling_ratio/mean": 0.6956661343574524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33841991424560547, + "sampling/sampling_logp_difference/mean": 0.028747636824846268, + "step": 57, + "step_time": 26.892430102452636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 646.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 560.4375, + "completions/mean_terminated_length": 560.4375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.3337246850132942, + "epoch": 0.116, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24178451299667358, + "kl": 0.023009511292912066, + "learning_rate": 3e-05, + "loss": -0.02738652005791664, + "num_tokens": 558710.0, + "reward": 6.0625, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.4787031412124634, + "sampling/importance_sampling_ratio/mean": 0.4813012480735779, + "sampling/importance_sampling_ratio/min": 0.05691095441579819, + "sampling/sampling_logp_difference/max": 0.3029825687408447, + "sampling/sampling_logp_difference/mean": 0.029777564108371735, + "step": 58, + "step_time": 17.878377372398973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 697.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 595.25, + "completions/mean_terminated_length": 595.25, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.3107040077447891, + "epoch": 0.118, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1262614130973816, + "kl": 0.022026430582627654, + "learning_rate": 3e-05, + "loss": 0.04775794595479965, + "num_tokens": 569826.0, + "reward": 6.4375, + "reward_std": 1.0307763814926147, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.0307763814926147, + "sampling/importance_sampling_ratio/max": 2.1841602325439453, + "sampling/importance_sampling_ratio/mean": 0.5139275193214417, + "sampling/importance_sampling_ratio/min": 0.026369251310825348, + "sampling/sampling_logp_difference/max": 0.42272377014160156, + "sampling/sampling_logp_difference/mean": 0.028494788333773613, + "step": 59, + "step_time": 24.42572767334059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 549.1875, + "completions/mean_terminated_length": 549.1875, + "completions/min_length": 489.0, + "completions/min_terminated_length": 489.0, + "entropy": 1.1738965958356857, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.588756799697876, + "kl": 0.025482238037511706, + "learning_rate": 3e-05, + "loss": 0.2925710678100586, + "num_tokens": 580229.0, + "reward": 6.0625, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 2.8465583324432373, + "sampling/importance_sampling_ratio/mean": 1.1227651834487915, + "sampling/importance_sampling_ratio/min": 0.1096419170498848, + "sampling/sampling_logp_difference/max": 0.4628920555114746, + "sampling/sampling_logp_difference/mean": 0.02885228767991066, + "step": 60, + "step_time": 21.57787229306996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 692.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 577.3125, + "completions/mean_terminated_length": 577.3125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.1913195550441742, + "epoch": 0.122, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2779391407966614, + "kl": 0.02629381464794278, + "learning_rate": 3e-05, + "loss": 0.12304374575614929, + "num_tokens": 591178.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.1122686862945557, + "sampling/importance_sampling_ratio/mean": 0.650765061378479, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.47726941108703613, + "sampling/sampling_logp_difference/mean": 0.027112768962979317, + "step": 61, + "step_time": 23.71764762001112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 628.3125, + "completions/mean_terminated_length": 628.3125, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3224291801452637, + "epoch": 0.124, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1645125448703766, + "kl": 0.025764177553355694, + "learning_rate": 3e-05, + "loss": 0.17419062554836273, + "num_tokens": 603055.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1662254333496094, + "sampling/importance_sampling_ratio/mean": 0.5809424519538879, + "sampling/importance_sampling_ratio/min": 0.03490918502211571, + "sampling/sampling_logp_difference/max": 0.4525270462036133, + "sampling/sampling_logp_difference/mean": 0.028948483988642693, + "step": 62, + "step_time": 35.74759274255484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 597.375, + "completions/mean_terminated_length": 597.375, + "completions/min_length": 478.0, + "completions/min_terminated_length": 478.0, + "entropy": 1.1552416533231735, + "epoch": 0.126, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20707836747169495, + "kl": 0.026473846402950585, + "learning_rate": 3e-05, + "loss": -0.019145065918564796, + "num_tokens": 614341.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.0268709659576416, + "sampling/importance_sampling_ratio/mean": 0.7066210508346558, + "sampling/importance_sampling_ratio/min": 0.1595209240913391, + "sampling/sampling_logp_difference/max": 0.42907285690307617, + "sampling/sampling_logp_difference/mean": 0.028000790625810623, + "step": 63, + "step_time": 39.37250621803105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 568.5, + "completions/mean_terminated_length": 568.5, + "completions/min_length": 509.0, + "completions/min_terminated_length": 509.0, + "entropy": 1.2810933291912079, + "epoch": 0.128, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21552008390426636, + "kl": 0.029041914734989405, + "learning_rate": 3e-05, + "loss": 0.037037670612335205, + "num_tokens": 625165.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.1896748542785645, + "sampling/importance_sampling_ratio/mean": 0.6408567428588867, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.374176025390625, + "sampling/sampling_logp_difference/mean": 0.02858484350144863, + "step": 64, + "step_time": 18.69576471252367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 622.75, + "completions/mean_terminated_length": 622.75, + "completions/min_length": 545.0, + "completions/min_terminated_length": 545.0, + "entropy": 1.32467532902956, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2287176102399826, + "kl": 0.023987084976397455, + "learning_rate": 3e-05, + "loss": 0.0731797143816948, + "num_tokens": 636633.0, + "reward": 6.375, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 1.4288272857666016, + "sampling/importance_sampling_ratio/mean": 0.5977773666381836, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4503474235534668, + "sampling/sampling_logp_difference/mean": 0.02755960263311863, + "step": 65, + "step_time": 27.502957582939416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 767.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 639.25, + "completions/mean_terminated_length": 639.25, + "completions/min_length": 554.0, + "completions/min_terminated_length": 554.0, + "entropy": 1.400998167693615, + "epoch": 0.132, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27425676584243774, + "kl": 0.028104660217650235, + "learning_rate": 3e-05, + "loss": 0.10324293375015259, + "num_tokens": 648597.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.971261501312256, + "sampling/importance_sampling_ratio/mean": 0.6433250904083252, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4819211959838867, + "sampling/sampling_logp_difference/mean": 0.029852069914340973, + "step": 66, + "step_time": 26.79405551031232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 541.375, + "completions/mean_terminated_length": 541.375, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.2635268718004227, + "epoch": 0.134, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13181555271148682, + "kl": 0.03373931790702045, + "learning_rate": 3e-05, + "loss": -0.11447598040103912, + "num_tokens": 658875.0, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 1.1883972883224487, + "sampling/importance_sampling_ratio/mean": 0.4192371368408203, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.44372403621673584, + "sampling/sampling_logp_difference/mean": 0.02911720983684063, + "step": 67, + "step_time": 27.6137525443919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 842.0, + "completions/max_terminated_length": 842.0, + "completions/mean_length": 689.4375, + "completions/mean_terminated_length": 689.4375, + "completions/min_length": 544.0, + "completions/min_terminated_length": 544.0, + "entropy": 1.2496536895632744, + "epoch": 0.136, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20560002326965332, + "kl": 0.026702777307946235, + "learning_rate": 3e-05, + "loss": -0.10130438208580017, + "num_tokens": 671690.0, + "reward": 5.875, + "reward_std": 2.0289571285247803, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 2.0289571285247803, + "sampling/importance_sampling_ratio/max": 2.8383262157440186, + "sampling/importance_sampling_ratio/mean": 0.9476768374443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35082435607910156, + "sampling/sampling_logp_difference/mean": 0.028364315629005432, + "step": 68, + "step_time": 29.35345455724746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 701.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 614.25, + "completions/mean_terminated_length": 614.25, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.1948458775877953, + "epoch": 0.138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20431844890117645, + "kl": 0.03377161466050893, + "learning_rate": 3e-05, + "loss": -0.0560678169131279, + "num_tokens": 683046.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.162766933441162, + "sampling/importance_sampling_ratio/mean": 0.5678162574768066, + "sampling/importance_sampling_ratio/min": 0.05678822472691536, + "sampling/sampling_logp_difference/max": 0.5758893489837646, + "sampling/sampling_logp_difference/mean": 0.028125843033194542, + "step": 69, + "step_time": 27.574916049838066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 588.3125, + "completions/mean_terminated_length": 588.3125, + "completions/min_length": 532.0, + "completions/min_terminated_length": 532.0, + "entropy": 1.2782762721180916, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26451998949050903, + "kl": 0.03907236340455711, + "learning_rate": 3e-05, + "loss": 0.17035090923309326, + "num_tokens": 694323.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.950176477432251, + "sampling/importance_sampling_ratio/mean": 0.45171743631362915, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46894216537475586, + "sampling/sampling_logp_difference/mean": 0.02863166108727455, + "step": 70, + "step_time": 19.817490340210497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 597.125, + "completions/mean_terminated_length": 597.125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2051330730319023, + "epoch": 0.142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5549097657203674, + "kl": 0.03670362115371972, + "learning_rate": 3e-05, + "loss": 0.4998631179332733, + "num_tokens": 705773.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.9404170513153076, + "sampling/importance_sampling_ratio/mean": 0.7522475123405457, + "sampling/importance_sampling_ratio/min": 0.05456363409757614, + "sampling/sampling_logp_difference/max": 0.4324514865875244, + "sampling/sampling_logp_difference/mean": 0.028340937569737434, + "step": 71, + "step_time": 41.66373607888818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 691.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 604.125, + "completions/mean_terminated_length": 604.125, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.18794547021389, + "epoch": 0.144, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10015435516834259, + "kl": 0.03911226138006896, + "learning_rate": 3e-05, + "loss": -0.02419177070260048, + "num_tokens": 717159.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1231037378311157, + "sampling/importance_sampling_ratio/mean": 0.4037884473800659, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6400742530822754, + "sampling/sampling_logp_difference/mean": 0.028367817401885986, + "step": 72, + "step_time": 23.86539705330506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 721.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 597.1875, + "completions/mean_terminated_length": 597.1875, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.2641174346208572, + "epoch": 0.146, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07918722927570343, + "kl": 0.03177848691120744, + "learning_rate": 3e-05, + "loss": -0.027635926380753517, + "num_tokens": 728402.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.5195796489715576, + "sampling/importance_sampling_ratio/mean": 0.4324396848678589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6819734573364258, + "sampling/sampling_logp_difference/mean": 0.030029678717255592, + "step": 73, + "step_time": 29.29490938829258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 640.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 574.375, + "completions/mean_terminated_length": 574.375, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3364054411649704, + "epoch": 0.148, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3853444755077362, + "kl": 0.03433372196741402, + "learning_rate": 3e-05, + "loss": -0.031142480671405792, + "num_tokens": 739632.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6853926181793213, + "sampling/importance_sampling_ratio/mean": 0.9200767874717712, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42021608352661133, + "sampling/sampling_logp_difference/mean": 0.030795859172940254, + "step": 74, + "step_time": 34.52008486771956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 618.25, + "completions/mean_terminated_length": 618.25, + "completions/min_length": 539.0, + "completions/min_terminated_length": 539.0, + "entropy": 1.365300178527832, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18101376295089722, + "kl": 0.02779634529724717, + "learning_rate": 3e-05, + "loss": -0.2718795835971832, + "num_tokens": 751164.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.8397568464279175, + "sampling/importance_sampling_ratio/mean": 0.5582400560379028, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42606019973754883, + "sampling/sampling_logp_difference/mean": 0.028895940631628036, + "step": 75, + "step_time": 18.76476171752438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 799.0, + "completions/max_terminated_length": 799.0, + "completions/mean_length": 603.375, + "completions/mean_terminated_length": 603.375, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2181015871465206, + "epoch": 0.152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1852622777223587, + "kl": 0.03176840906962752, + "learning_rate": 3e-05, + "loss": -0.10660596191883087, + "num_tokens": 762370.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.317336082458496, + "sampling/importance_sampling_ratio/mean": 0.550554633140564, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.481325626373291, + "sampling/sampling_logp_difference/mean": 0.028557566925883293, + "step": 76, + "step_time": 27.090129756834358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 572.9375, + "completions/mean_terminated_length": 572.9375, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2986655682325363, + "epoch": 0.154, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1531844586133957, + "kl": 0.03523328877054155, + "learning_rate": 3e-05, + "loss": -0.20856647193431854, + "num_tokens": 773169.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 2.855010986328125, + "sampling/importance_sampling_ratio/mean": 0.8239375352859497, + "sampling/importance_sampling_ratio/min": 0.1521405428647995, + "sampling/sampling_logp_difference/max": 0.4692528247833252, + "sampling/sampling_logp_difference/mean": 0.029906686395406723, + "step": 77, + "step_time": 25.004970545414835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 607.1875, + "completions/mean_terminated_length": 607.1875, + "completions/min_length": 540.0, + "completions/min_terminated_length": 540.0, + "entropy": 1.1003647185862064, + "epoch": 0.156, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14113759994506836, + "kl": 0.025135049945674837, + "learning_rate": 3e-05, + "loss": -0.10802068561315536, + "num_tokens": 784724.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.0096027851104736, + "sampling/importance_sampling_ratio/mean": 0.613497257232666, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4082927703857422, + "sampling/sampling_logp_difference/mean": 0.027884263545274734, + "step": 78, + "step_time": 29.614154959097505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 568.0625, + "completions/mean_terminated_length": 568.0625, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.266264721751213, + "epoch": 0.158, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18073000013828278, + "kl": 0.028119354974478483, + "learning_rate": 3e-05, + "loss": -0.0687609314918518, + "num_tokens": 795517.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.277299404144287, + "sampling/importance_sampling_ratio/mean": 0.3485238552093506, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4658241271972656, + "sampling/sampling_logp_difference/mean": 0.029626762494444847, + "step": 79, + "step_time": 23.5287338164635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 565.8125, + "completions/mean_terminated_length": 565.8125, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2773499339818954, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.31362995505332947, + "kl": 0.028471783734858036, + "learning_rate": 3e-05, + "loss": 0.059164684265851974, + "num_tokens": 806258.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.5347814559936523, + "sampling/importance_sampling_ratio/mean": 0.7027873396873474, + "sampling/importance_sampling_ratio/min": 0.06356429308652878, + "sampling/sampling_logp_difference/max": 0.4123659133911133, + "sampling/sampling_logp_difference/mean": 0.02946357987821102, + "step": 80, + "step_time": 24.251087154261768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 600.0, + "completions/max_terminated_length": 600.0, + "completions/mean_length": 536.1875, + "completions/mean_terminated_length": 536.1875, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.211024284362793, + "epoch": 0.162, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1959177404642105, + "kl": 0.02152467134874314, + "learning_rate": 3e-05, + "loss": 0.14755703508853912, + "num_tokens": 816717.0, + "reward": 6.0, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.764387369155884, + "sampling/importance_sampling_ratio/mean": 0.8167816400527954, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.28648805618286133, + "sampling/sampling_logp_difference/mean": 0.02814806066453457, + "step": 81, + "step_time": 22.752198832575232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 568.625, + "completions/mean_terminated_length": 568.625, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "entropy": 1.2584010139107704, + "epoch": 0.164, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16224367916584015, + "kl": 0.02812566957436502, + "learning_rate": 3e-05, + "loss": -0.1586945354938507, + "num_tokens": 827591.0, + "reward": 6.5, + "reward_std": 1.26491117477417, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.26491117477417, + "sampling/importance_sampling_ratio/max": 2.4172537326812744, + "sampling/importance_sampling_ratio/mean": 0.7026975154876709, + "sampling/importance_sampling_ratio/min": 0.1125984862446785, + "sampling/sampling_logp_difference/max": 0.3966444730758667, + "sampling/sampling_logp_difference/mean": 0.02937215007841587, + "step": 82, + "step_time": 22.57465209439397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 783.0, + "completions/max_terminated_length": 783.0, + "completions/mean_length": 583.5625, + "completions/mean_terminated_length": 583.5625, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2498536258935928, + "epoch": 0.166, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28551891446113586, + "kl": 0.023335880250670016, + "learning_rate": 3e-05, + "loss": 0.09868354350328445, + "num_tokens": 838760.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 1.9430233240127563, + "sampling/importance_sampling_ratio/mean": 0.7391272783279419, + "sampling/importance_sampling_ratio/min": 0.2032935619354248, + "sampling/sampling_logp_difference/max": 0.3390723466873169, + "sampling/sampling_logp_difference/mean": 0.02833949774503708, + "step": 83, + "step_time": 24.9633763525635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 581.0625, + "completions/mean_terminated_length": 581.0625, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.1582137942314148, + "epoch": 0.168, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1089889332652092, + "kl": 0.02546319324756041, + "learning_rate": 3e-05, + "loss": -0.04368923604488373, + "num_tokens": 849945.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.6800583600997925, + "sampling/importance_sampling_ratio/mean": 0.4864083528518677, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48818397521972656, + "sampling/sampling_logp_difference/mean": 0.02942320704460144, + "step": 84, + "step_time": 22.7196712391451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 553.3125, + "completions/mean_terminated_length": 553.3125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.1762890554964542, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18219847977161407, + "kl": 0.023275951389223337, + "learning_rate": 3e-05, + "loss": 0.055040232837200165, + "num_tokens": 860734.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.2940757274627686, + "sampling/importance_sampling_ratio/mean": 0.6381184458732605, + "sampling/importance_sampling_ratio/min": 0.08803392946720123, + "sampling/sampling_logp_difference/max": 0.3728243112564087, + "sampling/sampling_logp_difference/mean": 0.028103860095143318, + "step": 85, + "step_time": 28.569310082122684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 538.5, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.1561524420976639, + "epoch": 0.172, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2631295323371887, + "kl": 0.027657793601974845, + "learning_rate": 3e-05, + "loss": 0.019699495285749435, + "num_tokens": 871182.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.950981378555298, + "sampling/importance_sampling_ratio/mean": 0.5496660470962524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.358644962310791, + "sampling/sampling_logp_difference/mean": 0.02922222763299942, + "step": 86, + "step_time": 19.95468496438116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 688.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 528.625, + "completions/mean_terminated_length": 528.625, + "completions/min_length": 418.0, + "completions/min_terminated_length": 418.0, + "entropy": 1.382395550608635, + "epoch": 0.174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25233277678489685, + "kl": 0.025461523793637753, + "learning_rate": 3e-05, + "loss": -0.0012568621896207333, + "num_tokens": 881272.0, + "reward": 6.3125, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 1.940340280532837, + "sampling/importance_sampling_ratio/mean": 0.44232380390167236, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3433331251144409, + "sampling/sampling_logp_difference/mean": 0.030555889010429382, + "step": 87, + "step_time": 29.044239778537303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 537.0, + "completions/mean_terminated_length": 537.0, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.1740282103419304, + "epoch": 0.176, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1609290987253189, + "kl": 0.023582924506627023, + "learning_rate": 3e-05, + "loss": -0.0040507810190320015, + "num_tokens": 891608.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.8592076301574707, + "sampling/importance_sampling_ratio/mean": 0.4413543939590454, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3994784355163574, + "sampling/sampling_logp_difference/mean": 0.028627343475818634, + "step": 88, + "step_time": 24.642031190916896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 562.0, + "completions/mean_terminated_length": 562.0, + "completions/min_length": 490.0, + "completions/min_terminated_length": 490.0, + "entropy": 1.3354259580373764, + "epoch": 0.178, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25136521458625793, + "kl": 0.03104234568309039, + "learning_rate": 3e-05, + "loss": -0.10014888644218445, + "num_tokens": 902472.0, + "reward": 6.5, + "reward_std": 1.154700517654419, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.154700517654419, + "sampling/importance_sampling_ratio/max": 2.0172529220581055, + "sampling/importance_sampling_ratio/mean": 0.5528109073638916, + "sampling/importance_sampling_ratio/min": 0.031755149364471436, + "sampling/sampling_logp_difference/max": 0.48168420791625977, + "sampling/sampling_logp_difference/mean": 0.029525987803936005, + "step": 89, + "step_time": 23.934129936154932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 543.6875, + "completions/mean_terminated_length": 543.6875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.3072879239916801, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2416294664144516, + "kl": 0.02474795945454389, + "learning_rate": 3e-05, + "loss": 0.02994484454393387, + "num_tokens": 913003.0, + "reward": 6.125, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4122473001480103, + "sampling/importance_sampling_ratio/mean": 0.5672672390937805, + "sampling/importance_sampling_ratio/min": 0.1312582641839981, + "sampling/sampling_logp_difference/max": 0.36547136306762695, + "sampling/sampling_logp_difference/mean": 0.030115650966763496, + "step": 90, + "step_time": 16.719651044346392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 587.5, + "completions/mean_terminated_length": 587.5, + "completions/min_length": 491.0, + "completions/min_terminated_length": 491.0, + "entropy": 1.2642723061144352, + "epoch": 0.182, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11401186883449554, + "kl": 0.018764875654596835, + "learning_rate": 3e-05, + "loss": 0.17740829288959503, + "num_tokens": 923971.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.223915934562683, + "sampling/importance_sampling_ratio/mean": 0.4373893141746521, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.43628501892089844, + "sampling/sampling_logp_difference/mean": 0.027218280360102654, + "step": 91, + "step_time": 21.256958127953112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 519.3125, + "completions/mean_terminated_length": 519.3125, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.2509336844086647, + "epoch": 0.184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14000695943832397, + "kl": 0.017409664229489863, + "learning_rate": 3e-05, + "loss": -0.1092228814959526, + "num_tokens": 933880.0, + "reward": 7.125, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 2.4079525470733643, + "sampling/importance_sampling_ratio/mean": 0.6406391263008118, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3843420743942261, + "sampling/sampling_logp_difference/mean": 0.02841360680758953, + "step": 92, + "step_time": 20.99564675288275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 681.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 587.8125, + "completions/mean_terminated_length": 587.8125, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.1215453520417213, + "epoch": 0.186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16520461440086365, + "kl": 0.028165725176222622, + "learning_rate": 3e-05, + "loss": -0.15029624104499817, + "num_tokens": 945269.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.7825064659118652, + "sampling/importance_sampling_ratio/mean": 0.5902000069618225, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3571600914001465, + "sampling/sampling_logp_difference/mean": 0.02762974239885807, + "step": 93, + "step_time": 17.922352592926472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 566.0, + "completions/mean_terminated_length": 566.0, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.2864234894514084, + "epoch": 0.188, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24750091135501862, + "kl": 0.02070689742686227, + "learning_rate": 3e-05, + "loss": 0.2850193381309509, + "num_tokens": 956021.0, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "sampling/importance_sampling_ratio/max": 1.9146449565887451, + "sampling/importance_sampling_ratio/mean": 0.6849822402000427, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4927506446838379, + "sampling/sampling_logp_difference/mean": 0.029227914288640022, + "step": 94, + "step_time": 23.034203104209155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 516.6875, + "completions/mean_terminated_length": 516.6875, + "completions/min_length": 486.0, + "completions/min_terminated_length": 486.0, + "entropy": 1.2404684573411942, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11928244680166245, + "kl": 0.023086783126927912, + "learning_rate": 3e-05, + "loss": -0.032361116260290146, + "num_tokens": 965920.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.216193437576294, + "sampling/importance_sampling_ratio/mean": 0.32463955879211426, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37001991271972656, + "sampling/sampling_logp_difference/mean": 0.02843114361166954, + "step": 95, + "step_time": 15.103232022374868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 650.0, + "completions/max_terminated_length": 650.0, + "completions/mean_length": 548.4375, + "completions/mean_terminated_length": 548.4375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.2861761301755905, + "epoch": 0.192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2920665442943573, + "kl": 0.017841250344645232, + "learning_rate": 3e-05, + "loss": 0.1640928089618683, + "num_tokens": 976695.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.8954812288284302, + "sampling/importance_sampling_ratio/mean": 0.754618763923645, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31093156337738037, + "sampling/sampling_logp_difference/mean": 0.02842729538679123, + "step": 96, + "step_time": 40.74571412149817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 553.5625, + "completions/mean_terminated_length": 553.5625, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.1928813084959984, + "epoch": 0.194, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29983460903167725, + "kl": 0.020173200638964772, + "learning_rate": 3e-05, + "loss": 0.05926981568336487, + "num_tokens": 987120.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.0362496376037598, + "sampling/importance_sampling_ratio/mean": 0.6645779609680176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40996313095092773, + "sampling/sampling_logp_difference/mean": 0.02854262851178646, + "step": 97, + "step_time": 17.38945102225989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 546.1875, + "completions/mean_terminated_length": 546.1875, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "entropy": 1.365786962211132, + "epoch": 0.196, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12385546416044235, + "kl": 0.02262102661188692, + "learning_rate": 3e-05, + "loss": -0.09927551448345184, + "num_tokens": 997395.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2717533111572266, + "sampling/importance_sampling_ratio/mean": 0.5988417267799377, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35561084747314453, + "sampling/sampling_logp_difference/mean": 0.029298899695277214, + "step": 98, + "step_time": 23.35145698580891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 593.4375, + "completions/mean_terminated_length": 593.4375, + "completions/min_length": 508.0, + "completions/min_terminated_length": 508.0, + "entropy": 1.1754724085330963, + "epoch": 0.198, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2662176787853241, + "kl": 0.02589411090593785, + "learning_rate": 3e-05, + "loss": 0.2574020326137543, + "num_tokens": 1008458.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1308085918426514, + "sampling/importance_sampling_ratio/mean": 0.6420408487319946, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42550981044769287, + "sampling/sampling_logp_difference/mean": 0.02820964716374874, + "step": 99, + "step_time": 21.02622760878876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 538.4375, + "completions/mean_terminated_length": 538.4375, + "completions/min_length": 483.0, + "completions/min_terminated_length": 483.0, + "entropy": 1.3136962801218033, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2549664378166199, + "kl": 0.024644543533213437, + "learning_rate": 3e-05, + "loss": 0.06747792661190033, + "num_tokens": 1018793.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.3327062129974365, + "sampling/importance_sampling_ratio/mean": 0.7165549993515015, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4304838180541992, + "sampling/sampling_logp_difference/mean": 0.0299112256616354, + "step": 100, + "step_time": 16.768271412234753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 567.875, + "completions/mean_terminated_length": 567.875, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.15415108948946, + "epoch": 0.202, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34537598490715027, + "kl": 0.025688456720672548, + "learning_rate": 3e-05, + "loss": -0.21041882038116455, + "num_tokens": 1029751.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.282634973526001, + "sampling/importance_sampling_ratio/mean": 0.5392330288887024, + "sampling/importance_sampling_ratio/min": 0.026465320959687233, + "sampling/sampling_logp_difference/max": 0.3903813362121582, + "sampling/sampling_logp_difference/mean": 0.02962569333612919, + "step": 101, + "step_time": 16.715499105863273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 574.125, + "completions/mean_terminated_length": 574.125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.3766441270709038, + "epoch": 0.204, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27241969108581543, + "kl": 0.025680337683297694, + "learning_rate": 3e-05, + "loss": 0.24403473734855652, + "num_tokens": 1040601.0, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 2.3365371227264404, + "sampling/importance_sampling_ratio/mean": 0.6375491619110107, + "sampling/importance_sampling_ratio/min": 0.07846305519342422, + "sampling/sampling_logp_difference/max": 0.4158613681793213, + "sampling/sampling_logp_difference/mean": 0.030232973396778107, + "step": 102, + "step_time": 19.942134194541723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 557.75, + "completions/mean_terminated_length": 557.75, + "completions/min_length": 510.0, + "completions/min_terminated_length": 510.0, + "entropy": 1.3887510225176811, + "epoch": 0.206, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2711203992366791, + "kl": 0.022622586810030043, + "learning_rate": 3e-05, + "loss": -0.07210355252027512, + "num_tokens": 1051229.0, + "reward": 6.3125, + "reward_std": 1.0144785642623901, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.0144785642623901, + "sampling/importance_sampling_ratio/max": 1.8279200792312622, + "sampling/importance_sampling_ratio/mean": 0.5626887083053589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3629894256591797, + "sampling/sampling_logp_difference/mean": 0.030201904475688934, + "step": 103, + "step_time": 30.44108156999573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 544.75, + "completions/mean_terminated_length": 544.75, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.1656717136502266, + "epoch": 0.208, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27351662516593933, + "kl": 0.02198210428468883, + "learning_rate": 3e-05, + "loss": 0.06065649166703224, + "num_tokens": 1061745.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.500911235809326, + "sampling/importance_sampling_ratio/mean": 0.8908482789993286, + "sampling/importance_sampling_ratio/min": 0.05167346075177193, + "sampling/sampling_logp_difference/max": 0.3885481357574463, + "sampling/sampling_logp_difference/mean": 0.027608510106801987, + "step": 104, + "step_time": 16.46229960815981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 616.0, + "completions/max_terminated_length": 616.0, + "completions/mean_length": 562.875, + "completions/mean_terminated_length": 562.875, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.3012276738882065, + "epoch": 0.21, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21975310146808624, + "kl": 0.023721053614281118, + "learning_rate": 3e-05, + "loss": 0.06463687866926193, + "num_tokens": 1072231.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 1.7785983085632324, + "sampling/importance_sampling_ratio/mean": 0.5429776906967163, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.9359657764434814, + "sampling/sampling_logp_difference/mean": 0.030092012137174606, + "step": 105, + "step_time": 24.80119998846203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 580.0625, + "completions/mean_terminated_length": 580.0625, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.3428374752402306, + "epoch": 0.212, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23242872953414917, + "kl": 0.025716557982377708, + "learning_rate": 3e-05, + "loss": -0.004262822680175304, + "num_tokens": 1083184.0, + "reward": 6.375, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.0552361011505127, + "sampling/importance_sampling_ratio/mean": 0.6959555745124817, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7246572971343994, + "sampling/sampling_logp_difference/mean": 0.02952728420495987, + "step": 106, + "step_time": 17.27699494967237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 655.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 589.75, + "completions/mean_terminated_length": 589.75, + "completions/min_length": 547.0, + "completions/min_terminated_length": 547.0, + "entropy": 1.471379242837429, + "epoch": 0.214, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23287689685821533, + "kl": 0.025674917036667466, + "learning_rate": 3e-05, + "loss": 0.08491670340299606, + "num_tokens": 1094204.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.1368408203125, + "sampling/importance_sampling_ratio/mean": 0.5767678022384644, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36995506286621094, + "sampling/sampling_logp_difference/mean": 0.02880111336708069, + "step": 107, + "step_time": 40.17427978478372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 595.6875, + "completions/mean_terminated_length": 595.6875, + "completions/min_length": 524.0, + "completions/min_terminated_length": 524.0, + "entropy": 1.2550728917121887, + "epoch": 0.216, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09948146343231201, + "kl": 0.022876911563798785, + "learning_rate": 3e-05, + "loss": 0.04861483350396156, + "num_tokens": 1105303.0, + "reward": 6.125, + "reward_std": 1.0878112316131592, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0878112316131592, + "sampling/importance_sampling_ratio/max": 1.1923820972442627, + "sampling/importance_sampling_ratio/mean": 0.3343955874443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3477153778076172, + "sampling/sampling_logp_difference/mean": 0.029913678765296936, + "step": 108, + "step_time": 18.59066634438932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 583.5, + "completions/mean_terminated_length": 583.5, + "completions/min_length": 467.0, + "completions/min_terminated_length": 467.0, + "entropy": 1.2097205966711044, + "epoch": 0.218, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1392563134431839, + "kl": 0.0263087993953377, + "learning_rate": 3e-05, + "loss": 0.10488568246364594, + "num_tokens": 1116591.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.0463244915008545, + "sampling/importance_sampling_ratio/mean": 0.4996475875377655, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31003570556640625, + "sampling/sampling_logp_difference/mean": 0.0288787130266428, + "step": 109, + "step_time": 17.845282280817628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 571.375, + "completions/mean_terminated_length": 571.375, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.28146480768919, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5394540429115295, + "kl": 0.024339908617548645, + "learning_rate": 3e-05, + "loss": -0.23892748355865479, + "num_tokens": 1127493.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.671478509902954, + "sampling/importance_sampling_ratio/mean": 1.223832130432129, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650541305541992, + "sampling/sampling_logp_difference/mean": 0.028643080964684486, + "step": 110, + "step_time": 22.93386760680005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 493.875, + "completions/mean_terminated_length": 493.875, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 1.12203798443079, + "epoch": 0.222, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4297163188457489, + "kl": 0.024493444827385247, + "learning_rate": 3e-05, + "loss": -0.21332937479019165, + "num_tokens": 1136979.0, + "reward": 6.9375, + "reward_std": 0.6800735592842102, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.6800735592842102, + "sampling/importance_sampling_ratio/max": 2.889394760131836, + "sampling/importance_sampling_ratio/mean": 0.7321407794952393, + "sampling/importance_sampling_ratio/min": 0.02116413414478302, + "sampling/sampling_logp_difference/max": 0.49600934982299805, + "sampling/sampling_logp_difference/mean": 0.028577474877238274, + "step": 111, + "step_time": 21.056686570402235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 672.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 583.8125, + "completions/mean_terminated_length": 583.8125, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.4159239530563354, + "epoch": 0.224, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.008436380885541439, + "kl": 0.024869016953743994, + "learning_rate": 3e-05, + "loss": 0.0004943995736539364, + "num_tokens": 1148176.0, + "reward": 7.0, + "reward_std": 0.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.642366886138916, + "sampling/importance_sampling_ratio/mean": 0.7060814499855042, + "sampling/importance_sampling_ratio/min": 0.006825839169323444, + "sampling/sampling_logp_difference/max": 0.572547435760498, + "sampling/sampling_logp_difference/mean": 0.03075096383690834, + "step": 112, + "step_time": 20.1555822850205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 638.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 582.75, + "completions/mean_terminated_length": 582.75, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2932439520955086, + "epoch": 0.226, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2439681738615036, + "kl": 0.025248280493542552, + "learning_rate": 3e-05, + "loss": -0.22745110094547272, + "num_tokens": 1159380.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.5284109115600586, + "sampling/importance_sampling_ratio/mean": 1.0150926113128662, + "sampling/importance_sampling_ratio/min": 0.021104907616972923, + "sampling/sampling_logp_difference/max": 0.2513730525970459, + "sampling/sampling_logp_difference/mean": 0.02884429693222046, + "step": 113, + "step_time": 18.09852197067812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 580.0, + "completions/mean_terminated_length": 580.0, + "completions/min_length": 535.0, + "completions/min_terminated_length": 535.0, + "entropy": 1.246352232992649, + "epoch": 0.228, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23755788803100586, + "kl": 0.02679906424600631, + "learning_rate": 3e-05, + "loss": 0.3550976812839508, + "num_tokens": 1170628.0, + "reward": 7.25, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.750225305557251, + "sampling/importance_sampling_ratio/mean": 1.0749173164367676, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5576918125152588, + "sampling/sampling_logp_difference/mean": 0.03159492090344429, + "step": 114, + "step_time": 24.629896679893136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 657.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 562.6875, + "completions/mean_terminated_length": 562.6875, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.253239594399929, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3102787733078003, + "kl": 0.027133187628351152, + "learning_rate": 3e-05, + "loss": -0.05118201673030853, + "num_tokens": 1181295.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.634671926498413, + "sampling/importance_sampling_ratio/mean": 0.8949281573295593, + "sampling/importance_sampling_ratio/min": 0.09920533001422882, + "sampling/sampling_logp_difference/max": 0.6224374771118164, + "sampling/sampling_logp_difference/mean": 0.030200565233826637, + "step": 115, + "step_time": 38.715506959706545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 610.9375, + "completions/mean_terminated_length": 610.9375, + "completions/min_length": 538.0, + "completions/min_terminated_length": 538.0, + "entropy": 1.2940760999917984, + "epoch": 0.232, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14933602511882782, + "kl": 0.031228074803948402, + "learning_rate": 3e-05, + "loss": -0.05550215765833855, + "num_tokens": 1192750.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6116093397140503, + "sampling/importance_sampling_ratio/mean": 0.41488125920295715, + "sampling/importance_sampling_ratio/min": 0.009796842001378536, + "sampling/sampling_logp_difference/max": 0.5450258255004883, + "sampling/sampling_logp_difference/mean": 0.03152918070554733, + "step": 116, + "step_time": 17.821606623008847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 560.8125, + "completions/mean_terminated_length": 560.8125, + "completions/min_length": 501.0, + "completions/min_terminated_length": 501.0, + "entropy": 1.21239273250103, + "epoch": 0.234, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23778750002384186, + "kl": 0.03231424337718636, + "learning_rate": 3e-05, + "loss": -0.09421571344137192, + "num_tokens": 1203219.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.1054162979125977, + "sampling/importance_sampling_ratio/mean": 0.7292319536209106, + "sampling/importance_sampling_ratio/min": 0.010288448072969913, + "sampling/sampling_logp_difference/max": 0.8687701225280762, + "sampling/sampling_logp_difference/mean": 0.030015992000699043, + "step": 117, + "step_time": 16.80020274501294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 583.4375, + "completions/mean_terminated_length": 583.4375, + "completions/min_length": 503.0, + "completions/min_terminated_length": 503.0, + "entropy": 1.0914121232926846, + "epoch": 0.236, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.177206888794899, + "kl": 0.027808396029286087, + "learning_rate": 3e-05, + "loss": 0.010135526768863201, + "num_tokens": 1214562.0, + "reward": 6.375, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.6413226127624512, + "sampling/importance_sampling_ratio/mean": 0.5455202460289001, + "sampling/importance_sampling_ratio/min": 0.14393718540668488, + "sampling/sampling_logp_difference/max": 0.37839651107788086, + "sampling/sampling_logp_difference/mean": 0.02755727432668209, + "step": 118, + "step_time": 32.04508572584018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 802.0, + "completions/max_terminated_length": 802.0, + "completions/mean_length": 623.875, + "completions/mean_terminated_length": 623.875, + "completions/min_length": 555.0, + "completions/min_terminated_length": 555.0, + "entropy": 1.2099597677588463, + "epoch": 0.238, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10887355357408524, + "kl": 0.029803494922816753, + "learning_rate": 3e-05, + "loss": -0.05460066720843315, + "num_tokens": 1226136.0, + "reward": 6.25, + "reward_std": 1.983263373374939, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.983263373374939, + "sampling/importance_sampling_ratio/max": 1.5456031560897827, + "sampling/importance_sampling_ratio/mean": 0.4625241756439209, + "sampling/importance_sampling_ratio/min": 0.06713607162237167, + "sampling/sampling_logp_difference/max": 0.5650186538696289, + "sampling/sampling_logp_difference/mean": 0.03079008124768734, + "step": 119, + "step_time": 20.80143166380003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 621.0, + "completions/mean_terminated_length": 621.0, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.3042216151952744, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1758948117494583, + "kl": 0.03042414935771376, + "learning_rate": 3e-05, + "loss": -0.1489834189414978, + "num_tokens": 1237856.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.4328413009643555, + "sampling/importance_sampling_ratio/mean": 0.7058912515640259, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40829265117645264, + "sampling/sampling_logp_difference/mean": 0.029741039499640465, + "step": 120, + "step_time": 17.90572352707386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 593.8125, + "completions/mean_terminated_length": 593.8125, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 1.132676463574171, + "epoch": 0.242, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42553070187568665, + "kl": 0.04428348131477833, + "learning_rate": 3e-05, + "loss": 0.22666211426258087, + "num_tokens": 1249173.0, + "reward": 5.625, + "reward_std": 1.8574175834655762, + "rewards/quality_reward/mean": 5.625, + "rewards/quality_reward/std": 1.8574175834655762, + "sampling/importance_sampling_ratio/max": 2.5172836780548096, + "sampling/importance_sampling_ratio/mean": 0.6919515132904053, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6260476112365723, + "sampling/sampling_logp_difference/mean": 0.030399736016988754, + "step": 121, + "step_time": 38.02521731564775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 587.625, + "completions/mean_terminated_length": 587.625, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.2395975515246391, + "epoch": 0.244, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.248436838388443, + "kl": 0.03361149993725121, + "learning_rate": 3e-05, + "loss": 0.024570390582084656, + "num_tokens": 1260463.0, + "reward": 6.0625, + "reward_std": 1.236594796180725, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.236594796180725, + "sampling/importance_sampling_ratio/max": 2.833437442779541, + "sampling/importance_sampling_ratio/mean": 0.8825340867042542, + "sampling/importance_sampling_ratio/min": 0.06326956301927567, + "sampling/sampling_logp_difference/max": 0.540553092956543, + "sampling/sampling_logp_difference/mean": 0.030399201437830925, + "step": 122, + "step_time": 17.796182619407773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 585.3125, + "completions/mean_terminated_length": 585.3125, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.375852882862091, + "epoch": 0.246, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16029156744480133, + "kl": 0.03495740657672286, + "learning_rate": 3e-05, + "loss": -0.053390923887491226, + "num_tokens": 1271644.0, + "reward": 6.4375, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.6618704795837402, + "sampling/importance_sampling_ratio/mean": 0.5607253909111023, + "sampling/importance_sampling_ratio/min": 0.08802779763936996, + "sampling/sampling_logp_difference/max": 0.7028732299804688, + "sampling/sampling_logp_difference/mean": 0.031593482941389084, + "step": 123, + "step_time": 14.86260191956535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 630.1875, + "completions/mean_terminated_length": 630.1875, + "completions/min_length": 571.0, + "completions/min_terminated_length": 571.0, + "entropy": 1.3353190049529076, + "epoch": 0.248, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19288285076618195, + "kl": 0.033586084260605276, + "learning_rate": 3e-05, + "loss": -0.010514559224247932, + "num_tokens": 1283351.0, + "reward": 6.625, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4858638048171997, + "sampling/importance_sampling_ratio/mean": 0.5407211184501648, + "sampling/importance_sampling_ratio/min": 0.007580960635095835, + "sampling/sampling_logp_difference/max": 0.6886825561523438, + "sampling/sampling_logp_difference/mean": 0.030591927468776703, + "step": 124, + "step_time": 19.809663326013833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 599.875, + "completions/mean_terminated_length": 599.875, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.168940719217062, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15600983798503876, + "kl": 0.03213575447443873, + "learning_rate": 3e-05, + "loss": 0.10287950932979584, + "num_tokens": 1294997.0, + "reward": 7.0, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 1.832617163658142, + "sampling/importance_sampling_ratio/mean": 0.6814589500427246, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5988303422927856, + "sampling/sampling_logp_difference/mean": 0.030487919226288795, + "step": 125, + "step_time": 36.94939920771867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 634.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 582.25, + "completions/mean_terminated_length": 582.25, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.0670793130993843, + "epoch": 0.252, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3560245931148529, + "kl": 0.025753034162335098, + "learning_rate": 3e-05, + "loss": 0.16848862171173096, + "num_tokens": 1305993.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.17899489402771, + "sampling/importance_sampling_ratio/mean": 0.7458471059799194, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7731144428253174, + "sampling/sampling_logp_difference/mean": 0.029648227617144585, + "step": 126, + "step_time": 16.284966757521033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 717.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 650.8125, + "completions/mean_terminated_length": 650.8125, + "completions/min_length": 571.0, + "completions/min_terminated_length": 571.0, + "entropy": 1.2732752412557602, + "epoch": 0.254, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18122251331806183, + "kl": 0.03214431612286717, + "learning_rate": 3e-05, + "loss": 0.16780534386634827, + "num_tokens": 1318054.0, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 1.937699556350708, + "sampling/importance_sampling_ratio/mean": 0.5485143065452576, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.0368115901947021, + "sampling/sampling_logp_difference/mean": 0.03218457102775574, + "step": 127, + "step_time": 19.58785921242088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 743.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 617.375, + "completions/mean_terminated_length": 617.375, + "completions/min_length": 530.0, + "completions/min_terminated_length": 530.0, + "entropy": 1.306506209075451, + "epoch": 0.256, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1356409788131714, + "kl": 0.026579287368804216, + "learning_rate": 3e-05, + "loss": 0.06286599487066269, + "num_tokens": 1329500.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.106013298034668, + "sampling/importance_sampling_ratio/mean": 0.4681059420108795, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.676828145980835, + "sampling/sampling_logp_difference/mean": 0.032271042466163635, + "step": 128, + "step_time": 17.268729500938207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 599.0, + "completions/mean_terminated_length": 599.0, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.3493447452783585, + "epoch": 0.258, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2950517237186432, + "kl": 0.02614310395438224, + "learning_rate": 3e-05, + "loss": -0.059055861085653305, + "num_tokens": 1341020.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.7381844520568848, + "sampling/importance_sampling_ratio/mean": 0.6402126550674438, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5858409404754639, + "sampling/sampling_logp_difference/mean": 0.031067587435245514, + "step": 129, + "step_time": 28.869210730306804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 710.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 585.75, + "completions/mean_terminated_length": 585.75, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2956265732645988, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42696425318717957, + "kl": 0.02683100081048906, + "learning_rate": 3e-05, + "loss": 0.003650778206065297, + "num_tokens": 1351928.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.9060652256011963, + "sampling/importance_sampling_ratio/mean": 0.8535536527633667, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4930081367492676, + "sampling/sampling_logp_difference/mean": 0.031333789229393005, + "step": 130, + "step_time": 18.11609491892159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 568.5625, + "completions/mean_terminated_length": 568.5625, + "completions/min_length": 470.0, + "completions/min_terminated_length": 470.0, + "entropy": 1.2097233161330223, + "epoch": 0.262, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20076203346252441, + "kl": 0.024291134322993457, + "learning_rate": 3e-05, + "loss": -0.01662549562752247, + "num_tokens": 1362825.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.6266331672668457, + "sampling/importance_sampling_ratio/mean": 0.6302506327629089, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.963031530380249, + "sampling/sampling_logp_difference/mean": 0.0319429412484169, + "step": 131, + "step_time": 24.060474771540612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 741.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 605.9375, + "completions/mean_terminated_length": 605.9375, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.2823583781719208, + "epoch": 0.264, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.184434711933136, + "kl": 0.02043789450544864, + "learning_rate": 3e-05, + "loss": -0.0666906088590622, + "num_tokens": 1374296.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.831458568572998, + "sampling/importance_sampling_ratio/mean": 0.5425443649291992, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.671515941619873, + "sampling/sampling_logp_difference/mean": 0.030500290915369987, + "step": 132, + "step_time": 17.10482985060662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 665.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 591.375, + "completions/mean_terminated_length": 591.375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.1699804589152336, + "epoch": 0.266, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4572683274745941, + "kl": 0.025262096198275685, + "learning_rate": 3e-05, + "loss": 0.6353421211242676, + "num_tokens": 1385318.0, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.3584084510803223, + "sampling/importance_sampling_ratio/mean": 0.7995439767837524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3869748115539551, + "sampling/sampling_logp_difference/mean": 0.028536029160022736, + "step": 133, + "step_time": 36.004622367210686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 652.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 554.25, + "completions/mean_terminated_length": 554.25, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1336797252297401, + "epoch": 0.268, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.256655752658844, + "kl": 0.022568197746295482, + "learning_rate": 3e-05, + "loss": 0.026529084891080856, + "num_tokens": 1396234.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 1.9517148733139038, + "sampling/importance_sampling_ratio/mean": 0.8202446699142456, + "sampling/importance_sampling_ratio/min": 0.022647401317954063, + "sampling/sampling_logp_difference/max": 1.145315170288086, + "sampling/sampling_logp_difference/mean": 0.02867746911942959, + "step": 134, + "step_time": 20.057327402289957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 641.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 573.5, + "completions/mean_terminated_length": 573.5, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.1878332123160362, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30302342772483826, + "kl": 0.02298387698829174, + "learning_rate": 3e-05, + "loss": 0.2014756053686142, + "num_tokens": 1407322.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.9701545238494873, + "sampling/importance_sampling_ratio/mean": 0.6178270578384399, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4479391574859619, + "sampling/sampling_logp_difference/mean": 0.03068450093269348, + "step": 135, + "step_time": 36.01238542608917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 534.0, + "completions/mean_terminated_length": 534.0, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2640416622161865, + "epoch": 0.272, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39115583896636963, + "kl": 0.020954113570041955, + "learning_rate": 3e-05, + "loss": 0.3642374873161316, + "num_tokens": 1418010.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.0828306674957275, + "sampling/importance_sampling_ratio/mean": 0.7149391770362854, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5613884925842285, + "sampling/sampling_logp_difference/mean": 0.0298798568546772, + "step": 136, + "step_time": 20.20259432308376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 538.8125, + "completions/mean_terminated_length": 538.8125, + "completions/min_length": 451.0, + "completions/min_terminated_length": 451.0, + "entropy": 1.2438515722751617, + "epoch": 0.274, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4104433059692383, + "kl": 0.02069689182098955, + "learning_rate": 3e-05, + "loss": -0.15938539803028107, + "num_tokens": 1428335.0, + "reward": 6.875, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.013305425643921, + "sampling/importance_sampling_ratio/mean": 0.5245123505592346, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46263813972473145, + "sampling/sampling_logp_difference/mean": 0.030781995505094528, + "step": 137, + "step_time": 18.362532567232847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 542.4375, + "completions/mean_terminated_length": 542.4375, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2741251289844513, + "epoch": 0.276, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41347458958625793, + "kl": 0.020388772361911833, + "learning_rate": 3e-05, + "loss": 0.3297041654586792, + "num_tokens": 1438734.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.6671712398529053, + "sampling/importance_sampling_ratio/mean": 0.822363555431366, + "sampling/importance_sampling_ratio/min": 0.016625043004751205, + "sampling/sampling_logp_difference/max": 0.40987062454223633, + "sampling/sampling_logp_difference/mean": 0.02844768762588501, + "step": 138, + "step_time": 34.91616539563984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 516.3125, + "completions/mean_terminated_length": 516.3125, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.2030403539538383, + "epoch": 0.278, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3896540701389313, + "kl": 0.022598200594075024, + "learning_rate": 3e-05, + "loss": -0.25778308510780334, + "num_tokens": 1448611.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.9001679420471191, + "sampling/importance_sampling_ratio/mean": 0.6815162897109985, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.2820701599121094, + "sampling/sampling_logp_difference/mean": 0.027655858546495438, + "step": 139, + "step_time": 20.81112790806219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 512.8125, + "completions/mean_terminated_length": 512.8125, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "entropy": 1.124063789844513, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2338583916425705, + "kl": 0.022081921808421612, + "learning_rate": 3e-05, + "loss": 0.09288515895605087, + "num_tokens": 1458456.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.7245709896087646, + "sampling/importance_sampling_ratio/mean": 0.7086957693099976, + "sampling/importance_sampling_ratio/min": 0.13776090741157532, + "sampling/sampling_logp_difference/max": 0.4399615526199341, + "sampling/sampling_logp_difference/mean": 0.02669401653110981, + "step": 140, + "step_time": 22.541061893571168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 521.9375, + "completions/mean_terminated_length": 521.9375, + "completions/min_length": 460.0, + "completions/min_terminated_length": 460.0, + "entropy": 1.3581550270318985, + "epoch": 0.282, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1792808622121811, + "kl": 0.02498150523751974, + "learning_rate": 3e-05, + "loss": 0.08992868661880493, + "num_tokens": 1468623.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.6533762216567993, + "sampling/importance_sampling_ratio/mean": 0.5165826082229614, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42975759506225586, + "sampling/sampling_logp_difference/mean": 0.028398238122463226, + "step": 141, + "step_time": 20.935550182592124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 454.625, + "completions/mean_terminated_length": 454.625, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 1.2165675312280655, + "epoch": 0.284, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2593871057033539, + "kl": 0.024267837987281382, + "learning_rate": 3e-05, + "loss": 0.24750135838985443, + "num_tokens": 1477449.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 1.5698224306106567, + "sampling/importance_sampling_ratio/mean": 0.6540807485580444, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3056577444076538, + "sampling/sampling_logp_difference/mean": 0.029166901484131813, + "step": 142, + "step_time": 17.03540184069425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 490.625, + "completions/mean_terminated_length": 490.625, + "completions/min_length": 446.0, + "completions/min_terminated_length": 446.0, + "entropy": 1.1259984448552132, + "epoch": 0.286, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37285444140434265, + "kl": 0.019997276307549328, + "learning_rate": 3e-05, + "loss": 0.330167293548584, + "num_tokens": 1486931.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.156266927719116, + "sampling/importance_sampling_ratio/mean": 0.7264441251754761, + "sampling/importance_sampling_ratio/min": 0.07088703662157059, + "sampling/sampling_logp_difference/max": 0.42168426513671875, + "sampling/sampling_logp_difference/mean": 0.02719968557357788, + "step": 143, + "step_time": 15.82226228993386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 529.0625, + "completions/mean_terminated_length": 496.0666809082031, + "completions/min_length": 421.0, + "completions/min_terminated_length": 421.0, + "entropy": 1.0426596216857433, + "epoch": 0.288, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11882677674293518, + "kl": 0.027060870255809277, + "learning_rate": 3e-05, + "loss": 0.07405021786689758, + "num_tokens": 1496916.0, + "reward": 5.75, + "reward_std": 1.732050895690918, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.732050895690918, + "sampling/importance_sampling_ratio/max": 0.8768613934516907, + "sampling/importance_sampling_ratio/mean": 0.39511895179748535, + "sampling/importance_sampling_ratio/min": 0.11731626838445663, + "sampling/sampling_logp_difference/max": 0.6632819175720215, + "sampling/sampling_logp_difference/mean": 0.02569635957479477, + "step": 144, + "step_time": 21.100794151891023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 502.5625, + "completions/mean_terminated_length": 502.5625, + "completions/min_length": 452.0, + "completions/min_terminated_length": 452.0, + "entropy": 1.2609772458672523, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25876912474632263, + "kl": 0.028287828317843378, + "learning_rate": 3e-05, + "loss": -0.18078479170799255, + "num_tokens": 1507157.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.90565824508667, + "sampling/importance_sampling_ratio/mean": 0.7513852119445801, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3683091402053833, + "sampling/sampling_logp_difference/mean": 0.02768835797905922, + "step": 145, + "step_time": 19.17377450503409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 586.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 478.25, + "completions/mean_terminated_length": 478.25, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "entropy": 1.2076954543590546, + "epoch": 0.292, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4150021970272064, + "kl": 0.027647150680422783, + "learning_rate": 3e-05, + "loss": -0.07925756275653839, + "num_tokens": 1516833.0, + "reward": 6.5, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.0522961616516113, + "sampling/importance_sampling_ratio/mean": 0.6499220132827759, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40656137466430664, + "sampling/sampling_logp_difference/mean": 0.028425993397831917, + "step": 146, + "step_time": 19.426104408223182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 467.1875, + "completions/mean_terminated_length": 467.1875, + "completions/min_length": 396.0, + "completions/min_terminated_length": 396.0, + "entropy": 1.1012553870677948, + "epoch": 0.294, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33721745014190674, + "kl": 0.02999569766689092, + "learning_rate": 3e-05, + "loss": 0.04165569692850113, + "num_tokens": 1526028.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.507693290710449, + "sampling/importance_sampling_ratio/mean": 0.8091086149215698, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4148428440093994, + "sampling/sampling_logp_difference/mean": 0.028098180890083313, + "step": 147, + "step_time": 32.705999514088035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 467.0, + "completions/mean_terminated_length": 467.0, + "completions/min_length": 385.0, + "completions/min_terminated_length": 385.0, + "entropy": 1.2554677203297615, + "epoch": 0.296, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26061347126960754, + "kl": 0.0399768726201728, + "learning_rate": 3e-05, + "loss": -0.018139198422431946, + "num_tokens": 1535348.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.5010173320770264, + "sampling/importance_sampling_ratio/mean": 0.6952720880508423, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35097575187683105, + "sampling/sampling_logp_difference/mean": 0.028018539771437645, + "step": 148, + "step_time": 34.92355508869514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 486.75, + "completions/mean_terminated_length": 486.75, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "entropy": 1.2563373371958733, + "epoch": 0.298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2081325203180313, + "kl": 0.03278218396008015, + "learning_rate": 3e-05, + "loss": -0.01242863480001688, + "num_tokens": 1544912.0, + "reward": 5.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.0925099849700928, + "sampling/importance_sampling_ratio/mean": 0.4945816695690155, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45126640796661377, + "sampling/sampling_logp_difference/mean": 0.030079778283834457, + "step": 149, + "step_time": 26.784944237209857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 571.0, + "completions/max_terminated_length": 571.0, + "completions/mean_length": 483.0625, + "completions/mean_terminated_length": 483.0625, + "completions/min_length": 366.0, + "completions/min_terminated_length": 366.0, + "entropy": 1.1289120316505432, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40016695857048035, + "kl": 0.032314015785232186, + "learning_rate": 3e-05, + "loss": -0.1471974402666092, + "num_tokens": 1554417.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.876359701156616, + "sampling/importance_sampling_ratio/mean": 0.8288668394088745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.438828706741333, + "sampling/sampling_logp_difference/mean": 0.027187082916498184, + "step": 150, + "step_time": 25.687996607273817 + } + ], + "logging_steps": 1, + "max_steps": 300, + "num_input_tokens_seen": 1554417, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/outputs/E0-baseline/checkpoint-150/training_args.bin b/outputs/E0-baseline/checkpoint-150/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c3a2693682d14c331dbd12e8872552fc28d54b49 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-150/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6 +size 7697 diff --git a/outputs/E0-baseline/checkpoint-200/README.md b/outputs/E0-baseline/checkpoint-200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88990785a4695219513364eafae195070ddd3182 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-200/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507 +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.20.0 \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-200/adapter_config.json b/outputs/E0-baseline/checkpoint-200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3b383bef044f277e3ff79f0fc5addabd3c441593 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-200/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.20.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-200/adapter_model.safetensors b/outputs/E0-baseline/checkpoint-200/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..599a10c021fe20503769119161deaae40f950ec0 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-200/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:794e31718cff267ee61edd0a5fa827f87f1677b8f7de4a21a2058d2f3a5a21fd +size 264308896 diff --git a/outputs/E0-baseline/checkpoint-200/chat_template.jinja b/outputs/E0-baseline/checkpoint-200/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-200/chat_template.jinja @@ -0,0 +1,61 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-200/tokenizer.json b/outputs/E0-baseline/checkpoint-200/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-200/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/outputs/E0-baseline/checkpoint-200/tokenizer_config.json b/outputs/E0-baseline/checkpoint-200/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..86d67326516569b4f1cec3ac7b51b8903062c648 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-200/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/outputs/E0-baseline/checkpoint-200/trainer_state.json b/outputs/E0-baseline/checkpoint-200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c3c57efaeef37d99918df2dfbd1c13389499add8 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-200/trainer_state.json @@ -0,0 +1,6634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4, + "eval_steps": 500, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 529.0, + "completions/mean_terminated_length": 529.0, + "completions/min_length": 482.0, + "completions/min_terminated_length": 482.0, + "entropy": 1.2025159299373627, + "epoch": 0.002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22668755054473877, + "kl": 0.0, + "learning_rate": 0.0, + "loss": 0.2398601919412613, + "num_tokens": 10400.0, + "reward": 6.375, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6477458477020264, + "sampling/importance_sampling_ratio/mean": 0.5064857602119446, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40894174575805664, + "sampling/sampling_logp_difference/mean": 0.026567919179797173, + "step": 1, + "step_time": 12.760562099982053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 644.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 562.25, + "completions/mean_terminated_length": 562.25, + "completions/min_length": 497.0, + "completions/min_terminated_length": 497.0, + "entropy": 1.1930748969316483, + "epoch": 0.004, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12967805564403534, + "kl": 0.0, + "learning_rate": 3e-06, + "loss": -0.08571265637874603, + "num_tokens": 20924.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 1.0037232637405396, + "sampling/importance_sampling_ratio/mean": 0.41275694966316223, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45699238777160645, + "sampling/sampling_logp_difference/mean": 0.025086138397455215, + "step": 2, + "step_time": 11.406366533134133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 483.625, + "completions/mean_terminated_length": 483.625, + "completions/min_length": 407.0, + "completions/min_terminated_length": 407.0, + "entropy": 1.1096689626574516, + "epoch": 0.006, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22966289520263672, + "kl": 0.0008355328354809899, + "learning_rate": 6e-06, + "loss": 0.020913563668727875, + "num_tokens": 30222.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.927120566368103, + "sampling/importance_sampling_ratio/mean": 0.6114993095397949, + "sampling/importance_sampling_ratio/min": 0.11067161709070206, + "sampling/sampling_logp_difference/max": 0.4620504379272461, + "sampling/sampling_logp_difference/mean": 0.024864500388503075, + "step": 3, + "step_time": 26.480680393986404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 608.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 524.75, + "completions/mean_terminated_length": 524.75, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.211122527718544, + "epoch": 0.008, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30378466844558716, + "kl": 0.0008403196770814247, + "learning_rate": 9e-06, + "loss": -0.12959149479866028, + "num_tokens": 40410.0, + "reward": 6.25, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.406888961791992, + "sampling/importance_sampling_ratio/mean": 0.5457419753074646, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3680229187011719, + "sampling/sampling_logp_difference/mean": 0.02656388282775879, + "step": 4, + "step_time": 22.95301443943754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 487.5625, + "completions/mean_terminated_length": 487.5625, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "entropy": 1.247180238366127, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5199307799339294, + "kl": 0.0008723233368073124, + "learning_rate": 1.2e-05, + "loss": 0.11524428427219391, + "num_tokens": 49915.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.1544158458709717, + "sampling/importance_sampling_ratio/mean": 0.963020920753479, + "sampling/importance_sampling_ratio/min": 0.04948288947343826, + "sampling/sampling_logp_difference/max": 0.4774587154388428, + "sampling/sampling_logp_difference/mean": 0.02625642716884613, + "step": 5, + "step_time": 22.37928077671677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 533.0, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 394.0, + "completions/min_terminated_length": 394.0, + "entropy": 1.1693861335515976, + "epoch": 0.012, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27643197774887085, + "kl": 0.0009261858467652928, + "learning_rate": 1.5e-05, + "loss": 0.15093231201171875, + "num_tokens": 60219.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 2.125091791152954, + "sampling/importance_sampling_ratio/mean": 0.7733402252197266, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7049552202224731, + "sampling/sampling_logp_difference/mean": 0.025986071676015854, + "step": 6, + "step_time": 21.00841654697433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 505.6875, + "completions/mean_terminated_length": 505.6875, + "completions/min_length": 425.0, + "completions/min_terminated_length": 425.0, + "entropy": 1.2473183795809746, + "epoch": 0.014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2166663259267807, + "kl": 0.0009701631606731098, + "learning_rate": 1.8e-05, + "loss": -0.08582288026809692, + "num_tokens": 69902.0, + "reward": 7.1875, + "reward_std": 0.75, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.8287386894226074, + "sampling/importance_sampling_ratio/mean": 0.5286832451820374, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.38030898571014404, + "sampling/sampling_logp_difference/mean": 0.0264718160033226, + "step": 7, + "step_time": 46.596127359196544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.1040107272565365, + "epoch": 0.016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18874908983707428, + "kl": 0.0010721117541834246, + "learning_rate": 2.1e-05, + "loss": -0.1946130096912384, + "num_tokens": 79501.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.3561792373657227, + "sampling/importance_sampling_ratio/mean": 0.6918502449989319, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.26114892959594727, + "sampling/sampling_logp_difference/mean": 0.02554757334291935, + "step": 8, + "step_time": 25.433595282491297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 554.0, + "completions/max_terminated_length": 554.0, + "completions/mean_length": 490.1875, + "completions/mean_terminated_length": 490.1875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1377170644700527, + "epoch": 0.018, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.204215869307518, + "kl": 0.002002388624532614, + "learning_rate": 2.4e-05, + "loss": -0.08130021393299103, + "num_tokens": 88976.0, + "reward": 6.5, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.0982444286346436, + "sampling/importance_sampling_ratio/mean": 0.5873216986656189, + "sampling/importance_sampling_ratio/min": 0.04890051856637001, + "sampling/sampling_logp_difference/max": 0.8512144088745117, + "sampling/sampling_logp_difference/mean": 0.02638406865298748, + "step": 9, + "step_time": 18.427699581719935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 500.1875, + "completions/mean_terminated_length": 500.1875, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.1230391450226307, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1985069215297699, + "kl": 0.0026735300780273974, + "learning_rate": 2.7000000000000002e-05, + "loss": -0.12387816607952118, + "num_tokens": 98603.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.983199954032898, + "sampling/importance_sampling_ratio/mean": 0.5639468431472778, + "sampling/importance_sampling_ratio/min": 0.012905921787023544, + "sampling/sampling_logp_difference/max": 0.3653905391693115, + "sampling/sampling_logp_difference/mean": 0.025383003056049347, + "step": 10, + "step_time": 14.99981931829825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 459.6875, + "completions/mean_terminated_length": 459.6875, + "completions/min_length": 379.0, + "completions/min_terminated_length": 379.0, + "entropy": 1.213625729084015, + "epoch": 0.022, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3695620596408844, + "kl": 0.00424640768324025, + "learning_rate": 3e-05, + "loss": -0.06913074851036072, + "num_tokens": 107734.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.000136375427246, + "sampling/importance_sampling_ratio/mean": 0.8144867420196533, + "sampling/importance_sampling_ratio/min": 0.06302778422832489, + "sampling/sampling_logp_difference/max": 0.3647327423095703, + "sampling/sampling_logp_difference/mean": 0.026674197986721992, + "step": 11, + "step_time": 30.46549107739702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.191277615725994, + "epoch": 0.024, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35457733273506165, + "kl": 0.007200263120466843, + "learning_rate": 3e-05, + "loss": 0.22097699344158173, + "num_tokens": 117199.0, + "reward": 5.9375, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 1.5660414695739746, + "sampling/importance_sampling_ratio/mean": 0.649204432964325, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3921785354614258, + "sampling/sampling_logp_difference/mean": 0.02726689912378788, + "step": 12, + "step_time": 15.719243939965963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 464.3125, + "completions/mean_terminated_length": 464.3125, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "entropy": 1.24251464381814, + "epoch": 0.026, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30047014355659485, + "kl": 0.0058551667898427695, + "learning_rate": 3e-05, + "loss": -0.07746122777462006, + "num_tokens": 126556.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6028401851654053, + "sampling/importance_sampling_ratio/mean": 0.7561360597610474, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4946432113647461, + "sampling/sampling_logp_difference/mean": 0.02639186754822731, + "step": 13, + "step_time": 18.08984712138772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 444.8125, + "completions/mean_terminated_length": 444.8125, + "completions/min_length": 389.0, + "completions/min_terminated_length": 389.0, + "entropy": 1.1501125395298004, + "epoch": 0.028, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24105942249298096, + "kl": 0.012796793889719993, + "learning_rate": 3e-05, + "loss": 0.10855278372764587, + "num_tokens": 135417.0, + "reward": 3.1875, + "reward_std": 3.310966730117798, + "rewards/quality_reward/mean": 3.1875, + "rewards/quality_reward/std": 3.310966730117798, + "sampling/importance_sampling_ratio/max": 2.541518211364746, + "sampling/importance_sampling_ratio/mean": 0.5903321504592896, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.8390979766845703, + "sampling/sampling_logp_difference/mean": 0.02838001400232315, + "step": 14, + "step_time": 20.055794408079237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 497.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 442.25, + "completions/mean_terminated_length": 442.25, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 1.1262825280427933, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19040776789188385, + "kl": 0.010701361010433175, + "learning_rate": 3e-05, + "loss": -0.007966680452227592, + "num_tokens": 144205.0, + "reward": 5.875, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.0754293203353882, + "sampling/importance_sampling_ratio/mean": 0.41446638107299805, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3982200622558594, + "sampling/sampling_logp_difference/mean": 0.027210108935832977, + "step": 15, + "step_time": 14.176074750721455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 478.125, + "completions/mean_terminated_length": 478.125, + "completions/min_length": 433.0, + "completions/min_terminated_length": 433.0, + "entropy": 1.2985924184322357, + "epoch": 0.032, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23625652492046356, + "kl": 0.0213887135614641, + "learning_rate": 3e-05, + "loss": -0.21546132862567902, + "num_tokens": 153543.0, + "reward": 6.5, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 2.0074336528778076, + "sampling/importance_sampling_ratio/mean": 0.49076417088508606, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5095968246459961, + "sampling/sampling_logp_difference/mean": 0.028833162039518356, + "step": 16, + "step_time": 15.848205763846636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 469.25, + "completions/mean_terminated_length": 469.25, + "completions/min_length": 423.0, + "completions/min_terminated_length": 423.0, + "entropy": 1.3148910626769066, + "epoch": 0.034, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17765119671821594, + "kl": 0.02128879475640133, + "learning_rate": 3e-05, + "loss": -0.0828079879283905, + "num_tokens": 163043.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 1.5988941192626953, + "sampling/importance_sampling_ratio/mean": 0.5021570324897766, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.591062068939209, + "sampling/sampling_logp_difference/mean": 0.030804751440882683, + "step": 17, + "step_time": 28.313011147081852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 447.0, + "completions/mean_terminated_length": 447.0, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 1.390664003789425, + "epoch": 0.036, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37244123220443726, + "kl": 0.019650122558232397, + "learning_rate": 3e-05, + "loss": -0.01184748113155365, + "num_tokens": 172379.0, + "reward": 6.0, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.675238847732544, + "sampling/importance_sampling_ratio/mean": 0.9581233263015747, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5762512683868408, + "sampling/sampling_logp_difference/mean": 0.03126702085137367, + "step": 18, + "step_time": 30.991567107848823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 498.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 447.75, + "completions/mean_terminated_length": 447.75, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 1.3287223279476166, + "epoch": 0.038, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23734751343727112, + "kl": 0.022738213243428618, + "learning_rate": 3e-05, + "loss": 0.040024783462285995, + "num_tokens": 181239.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.095722198486328, + "sampling/importance_sampling_ratio/mean": 0.6408629417419434, + "sampling/importance_sampling_ratio/min": 0.1203346848487854, + "sampling/sampling_logp_difference/max": 0.4722297191619873, + "sampling/sampling_logp_difference/mean": 0.030378391966223717, + "step": 19, + "step_time": 18.615950418636203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 452.3125, + "completions/mean_terminated_length": 452.3125, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 1.1001618690788746, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34637194871902466, + "kl": 0.015380491153337061, + "learning_rate": 3e-05, + "loss": 0.1691148728132248, + "num_tokens": 190068.0, + "reward": 6.125, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.6566128730773926, + "sampling/importance_sampling_ratio/mean": 0.7767290472984314, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.6531033515930176, + "sampling/sampling_logp_difference/mean": 0.030463142320513725, + "step": 20, + "step_time": 15.741292077116668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 457.5625, + "completions/mean_terminated_length": 457.5625, + "completions/min_length": 390.0, + "completions/min_terminated_length": 390.0, + "entropy": 1.3708399310708046, + "epoch": 0.042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2514922618865967, + "kl": 0.018277494702488184, + "learning_rate": 3e-05, + "loss": -0.13425321877002716, + "num_tokens": 198941.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.5612297058105469, + "sampling/importance_sampling_ratio/mean": 0.601022481918335, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4818992614746094, + "sampling/sampling_logp_difference/mean": 0.0315740592777729, + "step": 21, + "step_time": 17.24192419089377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 421.4375, + "completions/mean_terminated_length": 421.4375, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 1.136269599199295, + "epoch": 0.044, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2268020063638687, + "kl": 0.017973962530959398, + "learning_rate": 3e-05, + "loss": 0.010622119531035423, + "num_tokens": 207300.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.7986358404159546, + "sampling/importance_sampling_ratio/mean": 0.46136727929115295, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5335149765014648, + "sampling/sampling_logp_difference/mean": 0.02804401144385338, + "step": 22, + "step_time": 19.37282825494185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 452.6875, + "completions/mean_terminated_length": 452.6875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3299130946397781, + "epoch": 0.046, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33994877338409424, + "kl": 0.021804221265483648, + "learning_rate": 3e-05, + "loss": -0.24404363334178925, + "num_tokens": 216343.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.452249526977539, + "sampling/importance_sampling_ratio/mean": 0.747193455696106, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6328549385070801, + "sampling/sampling_logp_difference/mean": 0.02918298915028572, + "step": 23, + "step_time": 15.356728344224393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 462.4375, + "completions/mean_terminated_length": 462.4375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.379701942205429, + "epoch": 0.048, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3076202869415283, + "kl": 0.020299295720178634, + "learning_rate": 3e-05, + "loss": -0.09123071283102036, + "num_tokens": 225550.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.5845099687576294, + "sampling/importance_sampling_ratio/mean": 0.6625345945358276, + "sampling/importance_sampling_ratio/min": 0.04495502635836601, + "sampling/sampling_logp_difference/max": 0.36547183990478516, + "sampling/sampling_logp_difference/mean": 0.028946854174137115, + "step": 24, + "step_time": 14.95462113339454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.2471638694405556, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22180576622486115, + "kl": 0.018309170845896006, + "learning_rate": 3e-05, + "loss": -0.1412944793701172, + "num_tokens": 235005.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.6407876014709473, + "sampling/importance_sampling_ratio/mean": 0.7712795734405518, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4363563060760498, + "sampling/sampling_logp_difference/mean": 0.02898716926574707, + "step": 25, + "step_time": 16.46686205593869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 455.75, + "completions/mean_terminated_length": 455.75, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 1.28530602902174, + "epoch": 0.052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37535253167152405, + "kl": 0.020504546992015094, + "learning_rate": 3e-05, + "loss": 0.10839397460222244, + "num_tokens": 243953.0, + "reward": 6.25, + "reward_std": 1.0645813941955566, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.0645813941955566, + "sampling/importance_sampling_ratio/max": 2.0567266941070557, + "sampling/importance_sampling_ratio/mean": 0.5800145864486694, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4523625373840332, + "sampling/sampling_logp_difference/mean": 0.027610119432210922, + "step": 26, + "step_time": 22.121026155073196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.0, + "completions/max_terminated_length": 514.0, + "completions/mean_length": 462.75, + "completions/mean_terminated_length": 462.75, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.2253629937767982, + "epoch": 0.054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2340388149023056, + "kl": 0.020236384589225054, + "learning_rate": 3e-05, + "loss": 0.04823978245258331, + "num_tokens": 253237.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.1560932397842407, + "sampling/importance_sampling_ratio/mean": 0.40753045678138733, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35022830963134766, + "sampling/sampling_logp_difference/mean": 0.028367744758725166, + "step": 27, + "step_time": 15.10967448912561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 538.0, + "completions/max_terminated_length": 538.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "entropy": 1.3779077678918839, + "epoch": 0.056, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5531017184257507, + "kl": 0.01706669357372448, + "learning_rate": 3e-05, + "loss": 0.0933581069111824, + "num_tokens": 262454.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.5711586475372314, + "sampling/importance_sampling_ratio/mean": 0.7730721831321716, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.343827486038208, + "sampling/sampling_logp_difference/mean": 0.028883326798677444, + "step": 28, + "step_time": 38.59647103771567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 477.0, + "completions/mean_terminated_length": 477.0, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.3791070505976677, + "epoch": 0.058, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33771538734436035, + "kl": 0.02141271048458293, + "learning_rate": 3e-05, + "loss": 0.010216176509857178, + "num_tokens": 271918.0, + "reward": 5.75, + "reward_std": 1.2909945249557495, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.2909945249557495, + "sampling/importance_sampling_ratio/max": 2.4999797344207764, + "sampling/importance_sampling_ratio/mean": 1.0250636339187622, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3258817195892334, + "sampling/sampling_logp_difference/mean": 0.029029540717601776, + "step": 29, + "step_time": 23.610272648278624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 471.1875, + "completions/mean_terminated_length": 471.1875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.4652926102280617, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21955685317516327, + "kl": 0.019562674744520336, + "learning_rate": 3e-05, + "loss": -0.014814459718763828, + "num_tokens": 281305.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 2.802098274230957, + "sampling/importance_sampling_ratio/mean": 0.866391658782959, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6354451179504395, + "sampling/sampling_logp_difference/mean": 0.03177585452795029, + "step": 30, + "step_time": 16.961607525125146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 490.5, + "completions/mean_terminated_length": 490.5, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 1.1957123205065727, + "epoch": 0.062, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12176825106143951, + "kl": 0.026934220048133284, + "learning_rate": 3e-05, + "loss": -0.08307373523712158, + "num_tokens": 291409.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.610858678817749, + "sampling/importance_sampling_ratio/mean": 0.6937527656555176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4253416061401367, + "sampling/sampling_logp_difference/mean": 0.02871524728834629, + "step": 31, + "step_time": 15.012207658961415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 462.0, + "completions/mean_terminated_length": 462.0, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3018206283450127, + "epoch": 0.064, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4979296624660492, + "kl": 0.03539742121938616, + "learning_rate": 3e-05, + "loss": 0.0017175457905977964, + "num_tokens": 300649.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8527640104293823, + "sampling/importance_sampling_ratio/mean": 0.7824680209159851, + "sampling/importance_sampling_ratio/min": 0.07447884231805801, + "sampling/sampling_logp_difference/max": 0.5221483707427979, + "sampling/sampling_logp_difference/mean": 0.029107660055160522, + "step": 32, + "step_time": 22.78309725271538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 474.0, + "completions/mean_terminated_length": 474.0, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.321175642311573, + "epoch": 0.066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22016967833042145, + "kl": 0.029592803912237287, + "learning_rate": 3e-05, + "loss": 0.05625719577074051, + "num_tokens": 309889.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.9875097274780273, + "sampling/importance_sampling_ratio/mean": 0.7832435369491577, + "sampling/importance_sampling_ratio/min": 0.18006731569766998, + "sampling/sampling_logp_difference/max": 0.38585615158081055, + "sampling/sampling_logp_difference/mean": 0.027828343212604523, + "step": 33, + "step_time": 44.51360382232815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 459.9375, + "completions/mean_terminated_length": 459.9375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.2222414836287498, + "epoch": 0.068, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2873040437698364, + "kl": 0.02840927499346435, + "learning_rate": 3e-05, + "loss": -0.037432070821523666, + "num_tokens": 318904.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 2.6647069454193115, + "sampling/importance_sampling_ratio/mean": 0.5744763016700745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.41987133026123047, + "sampling/sampling_logp_difference/mean": 0.029470665380358696, + "step": 34, + "step_time": 133.03877451224253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 522.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 462.1875, + "completions/mean_terminated_length": 462.1875, + "completions/min_length": 413.0, + "completions/min_terminated_length": 413.0, + "entropy": 1.1665974482893944, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2490653246641159, + "kl": 0.025841041060630232, + "learning_rate": 3e-05, + "loss": -0.20422951877117157, + "num_tokens": 328011.0, + "reward": 6.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 2.510730266571045, + "sampling/importance_sampling_ratio/mean": 0.7954420447349548, + "sampling/importance_sampling_ratio/min": 0.020566223189234734, + "sampling/sampling_logp_difference/max": 0.36430132389068604, + "sampling/sampling_logp_difference/mean": 0.026844775304198265, + "step": 35, + "step_time": 21.5843787365593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 549.0, + "completions/max_terminated_length": 549.0, + "completions/mean_length": 492.0, + "completions/mean_terminated_length": 492.0, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3467887043952942, + "epoch": 0.072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15268851816654205, + "kl": 0.023660800594370812, + "learning_rate": 3e-05, + "loss": -0.11188814043998718, + "num_tokens": 337731.0, + "reward": 6.5625, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.6270908117294312, + "sampling/importance_sampling_ratio/mean": 0.614537239074707, + "sampling/importance_sampling_ratio/min": 0.10853960365056992, + "sampling/sampling_logp_difference/max": 0.4023854732513428, + "sampling/sampling_logp_difference/mean": 0.028961554169654846, + "step": 36, + "step_time": 18.843947287183255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.0, + "completions/max_terminated_length": 513.0, + "completions/mean_length": 460.75, + "completions/mean_terminated_length": 460.75, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "entropy": 1.2476315572857857, + "epoch": 0.074, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42548227310180664, + "kl": 0.022181478852871805, + "learning_rate": 3e-05, + "loss": 0.37223517894744873, + "num_tokens": 346815.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.6624510288238525, + "sampling/importance_sampling_ratio/mean": 0.7942180633544922, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4037543535232544, + "sampling/sampling_logp_difference/mean": 0.028780322521924973, + "step": 37, + "step_time": 42.04662919091061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 452.0, + "completions/mean_terminated_length": 452.0, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "entropy": 1.1745503433048725, + "epoch": 0.076, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16980381309986115, + "kl": 0.017483733594417572, + "learning_rate": 3e-05, + "loss": -0.09029137343168259, + "num_tokens": 355711.0, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "sampling/importance_sampling_ratio/max": 2.3201518058776855, + "sampling/importance_sampling_ratio/mean": 0.721072793006897, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.30433225631713867, + "sampling/sampling_logp_difference/mean": 0.026646045967936516, + "step": 38, + "step_time": 38.63075139513239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 466.8125, + "completions/mean_terminated_length": 466.8125, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2664988860487938, + "epoch": 0.078, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21305795013904572, + "kl": 0.021121050289366394, + "learning_rate": 3e-05, + "loss": -0.03154226019978523, + "num_tokens": 364860.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.0297553539276123, + "sampling/importance_sampling_ratio/mean": 0.674609363079071, + "sampling/importance_sampling_ratio/min": 0.11245065927505493, + "sampling/sampling_logp_difference/max": 0.37443917989730835, + "sampling/sampling_logp_difference/mean": 0.028257746249437332, + "step": 39, + "step_time": 30.028073193039745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 485.1875, + "completions/mean_terminated_length": 485.1875, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.3458357080817223, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12235487997531891, + "kl": 0.018535695446189493, + "learning_rate": 3e-05, + "loss": -0.035816628485918045, + "num_tokens": 374607.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.242042303085327, + "sampling/importance_sampling_ratio/mean": 0.5344723463058472, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36985254287719727, + "sampling/sampling_logp_difference/mean": 0.029600802809000015, + "step": 40, + "step_time": 15.446288945619017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 498.875, + "completions/mean_terminated_length": 498.875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3402792811393738, + "epoch": 0.082, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15642686188220978, + "kl": 0.013967045990284532, + "learning_rate": 3e-05, + "loss": 0.0011727213859558105, + "num_tokens": 384317.0, + "reward": 6.375, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5691092014312744, + "sampling/importance_sampling_ratio/mean": 0.3696203827857971, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6487679481506348, + "sampling/sampling_logp_difference/mean": 0.03018251620233059, + "step": 41, + "step_time": 18.15720977121964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.3317564576864243, + "epoch": 0.084, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3236794173717499, + "kl": 0.01707366103073582, + "learning_rate": 3e-05, + "loss": 0.10363321751356125, + "num_tokens": 393934.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.0568668842315674, + "sampling/importance_sampling_ratio/mean": 0.6415404677391052, + "sampling/importance_sampling_ratio/min": 0.07682990282773972, + "sampling/sampling_logp_difference/max": 0.7769032716751099, + "sampling/sampling_logp_difference/mean": 0.02936164103448391, + "step": 42, + "step_time": 15.066733688581735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 510.9375, + "completions/mean_terminated_length": 510.9375, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2099780030548573, + "epoch": 0.086, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1771102100610733, + "kl": 0.01784718461567536, + "learning_rate": 3e-05, + "loss": -0.027566466480493546, + "num_tokens": 403893.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.3430161476135254, + "sampling/importance_sampling_ratio/mean": 0.7769261598587036, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36153650283813477, + "sampling/sampling_logp_difference/mean": 0.028799494728446007, + "step": 43, + "step_time": 16.90863296529278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 494.1875, + "completions/mean_terminated_length": 494.1875, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "entropy": 1.2924985438585281, + "epoch": 0.088, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3432070314884186, + "kl": 0.014529847539961338, + "learning_rate": 3e-05, + "loss": -0.04157561436295509, + "num_tokens": 413312.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.726816177368164, + "sampling/importance_sampling_ratio/mean": 0.8989821672439575, + "sampling/importance_sampling_ratio/min": 0.07410597801208496, + "sampling/sampling_logp_difference/max": 0.31444358825683594, + "sampling/sampling_logp_difference/mean": 0.028122060000896454, + "step": 44, + "step_time": 16.7880685236305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 543.5, + "completions/mean_terminated_length": 543.5, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.206408604979515, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36233776807785034, + "kl": 0.015796773659531027, + "learning_rate": 3e-05, + "loss": 0.029176680371165276, + "num_tokens": 423624.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.677195429801941, + "sampling/importance_sampling_ratio/mean": 0.6537867188453674, + "sampling/importance_sampling_ratio/min": 0.09822077304124832, + "sampling/sampling_logp_difference/max": 0.31381869316101074, + "sampling/sampling_logp_difference/mean": 0.02717999368906021, + "step": 45, + "step_time": 23.35960763087496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 643.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 534.5625, + "completions/mean_terminated_length": 534.5625, + "completions/min_length": 453.0, + "completions/min_terminated_length": 453.0, + "entropy": 1.2577891275286674, + "epoch": 0.092, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20219561457633972, + "kl": 0.014481160265859216, + "learning_rate": 3e-05, + "loss": 0.18850615620613098, + "num_tokens": 433817.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.203894853591919, + "sampling/importance_sampling_ratio/mean": 0.697495698928833, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35237741470336914, + "sampling/sampling_logp_difference/mean": 0.02723248302936554, + "step": 46, + "step_time": 19.56032704282552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 538.25, + "completions/mean_terminated_length": 538.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.3271892964839935, + "epoch": 0.094, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22262753546237946, + "kl": 0.012554377142805606, + "learning_rate": 3e-05, + "loss": 0.06984467804431915, + "num_tokens": 444045.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 2.2261502742767334, + "sampling/importance_sampling_ratio/mean": 0.6712950468063354, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4496123790740967, + "sampling/sampling_logp_difference/mean": 0.028838323429226875, + "step": 47, + "step_time": 21.226045075803995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 514.25, + "completions/mean_terminated_length": 514.25, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "entropy": 1.1054812744259834, + "epoch": 0.096, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3244606554508209, + "kl": 0.0157591889728792, + "learning_rate": 3e-05, + "loss": 0.09024985134601593, + "num_tokens": 453945.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.6745388507843018, + "sampling/importance_sampling_ratio/mean": 0.7607913017272949, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650559186935425, + "sampling/sampling_logp_difference/mean": 0.026116060093045235, + "step": 48, + "step_time": 17.565261672716588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 585.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 502.4375, + "completions/mean_terminated_length": 502.4375, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.2281213253736496, + "epoch": 0.098, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4033137857913971, + "kl": 0.015613102470524609, + "learning_rate": 3e-05, + "loss": -0.2898017466068268, + "num_tokens": 463576.0, + "reward": 6.4375, + "reward_std": 1.2632629871368408, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.2632629871368408, + "sampling/importance_sampling_ratio/max": 2.6974196434020996, + "sampling/importance_sampling_ratio/mean": 0.7124314308166504, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3675417900085449, + "sampling/sampling_logp_difference/mean": 0.028341906145215034, + "step": 49, + "step_time": 29.838082558009773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 530.1875, + "completions/mean_terminated_length": 530.1875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.1955150067806244, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23144562542438507, + "kl": 0.01374751579714939, + "learning_rate": 3e-05, + "loss": -0.19065965712070465, + "num_tokens": 473915.0, + "reward": 6.3125, + "reward_std": 1.1954776048660278, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.1954776048660278, + "sampling/importance_sampling_ratio/max": 2.54063081741333, + "sampling/importance_sampling_ratio/mean": 0.8162041902542114, + "sampling/importance_sampling_ratio/min": 0.1628064066171646, + "sampling/sampling_logp_difference/max": 0.2777421474456787, + "sampling/sampling_logp_difference/mean": 0.02853373810648918, + "step": 50, + "step_time": 16.047010839451104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 549.875, + "completions/mean_terminated_length": 549.875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1870752647519112, + "epoch": 0.102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4241364896297455, + "kl": 0.013923745544161648, + "learning_rate": 3e-05, + "loss": -0.1761355698108673, + "num_tokens": 484577.0, + "reward": 6.5625, + "reward_std": 1.0935417413711548, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 1.0935417413711548, + "sampling/importance_sampling_ratio/max": 2.929507255554199, + "sampling/importance_sampling_ratio/mean": 0.6905896663665771, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5649824142456055, + "sampling/sampling_logp_difference/mean": 0.028025619685649872, + "step": 51, + "step_time": 44.488836522214115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 552.375, + "completions/mean_terminated_length": 552.375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.098166175186634, + "epoch": 0.104, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.32851356267929077, + "kl": 0.01297539210645482, + "learning_rate": 3e-05, + "loss": -0.06503897905349731, + "num_tokens": 495015.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.9652340412139893, + "sampling/importance_sampling_ratio/mean": 0.9612224102020264, + "sampling/importance_sampling_ratio/min": 0.040177375078201294, + "sampling/sampling_logp_difference/max": 0.3454720973968506, + "sampling/sampling_logp_difference/mean": 0.02687419019639492, + "step": 52, + "step_time": 20.21497478755191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 578.375, + "completions/mean_terminated_length": 578.375, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2311968132853508, + "epoch": 0.106, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21921825408935547, + "kl": 0.017025968758389354, + "learning_rate": 3e-05, + "loss": -0.18975484371185303, + "num_tokens": 505909.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.0094237327575684, + "sampling/importance_sampling_ratio/mean": 0.5587533116340637, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.39138758182525635, + "sampling/sampling_logp_difference/mean": 0.028095029294490814, + "step": 53, + "step_time": 31.140278964303434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 473.25, + "completions/mean_terminated_length": 473.25, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 1.2682743221521378, + "epoch": 0.108, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27410373091697693, + "kl": 0.018500705540645868, + "learning_rate": 3e-05, + "loss": 0.14847250282764435, + "num_tokens": 515073.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.875750780105591, + "sampling/importance_sampling_ratio/mean": 0.7429271936416626, + "sampling/importance_sampling_ratio/min": 0.09779425710439682, + "sampling/sampling_logp_difference/max": 0.5433444976806641, + "sampling/sampling_logp_difference/mean": 0.02810005284845829, + "step": 54, + "step_time": 18.365382733754814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 549.0, + "completions/mean_terminated_length": 549.0, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.3073157891631126, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29323700070381165, + "kl": 0.016703857632819563, + "learning_rate": 3e-05, + "loss": 0.05967015400528908, + "num_tokens": 525377.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 1.4832638502120972, + "sampling/importance_sampling_ratio/mean": 0.6094669103622437, + "sampling/importance_sampling_ratio/min": 0.08072065562009811, + "sampling/sampling_logp_difference/max": 0.39328718185424805, + "sampling/sampling_logp_difference/mean": 0.02906947024166584, + "step": 55, + "step_time": 30.47015379369259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 576.625, + "completions/mean_terminated_length": 576.625, + "completions/min_length": 500.0, + "completions/min_terminated_length": 500.0, + "entropy": 1.2820357605814934, + "epoch": 0.112, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.200644388794899, + "kl": 0.018819268501829356, + "learning_rate": 3e-05, + "loss": -0.04828515648841858, + "num_tokens": 536163.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.3941831588745117, + "sampling/importance_sampling_ratio/mean": 0.5633801221847534, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3664684295654297, + "sampling/sampling_logp_difference/mean": 0.02962428703904152, + "step": 56, + "step_time": 16.172011949121952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 640.25, + "completions/mean_terminated_length": 640.25, + "completions/min_length": 556.0, + "completions/min_terminated_length": 556.0, + "entropy": 1.4191219061613083, + "epoch": 0.114, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19633841514587402, + "kl": 0.02060725452611223, + "learning_rate": 3e-05, + "loss": -0.12029920518398285, + "num_tokens": 548143.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.3877830505371094, + "sampling/importance_sampling_ratio/mean": 0.6956661343574524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33841991424560547, + "sampling/sampling_logp_difference/mean": 0.028747636824846268, + "step": 57, + "step_time": 26.892430102452636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 646.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 560.4375, + "completions/mean_terminated_length": 560.4375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.3337246850132942, + "epoch": 0.116, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24178451299667358, + "kl": 0.023009511292912066, + "learning_rate": 3e-05, + "loss": -0.02738652005791664, + "num_tokens": 558710.0, + "reward": 6.0625, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.4787031412124634, + "sampling/importance_sampling_ratio/mean": 0.4813012480735779, + "sampling/importance_sampling_ratio/min": 0.05691095441579819, + "sampling/sampling_logp_difference/max": 0.3029825687408447, + "sampling/sampling_logp_difference/mean": 0.029777564108371735, + "step": 58, + "step_time": 17.878377372398973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 697.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 595.25, + "completions/mean_terminated_length": 595.25, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.3107040077447891, + "epoch": 0.118, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1262614130973816, + "kl": 0.022026430582627654, + "learning_rate": 3e-05, + "loss": 0.04775794595479965, + "num_tokens": 569826.0, + "reward": 6.4375, + "reward_std": 1.0307763814926147, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.0307763814926147, + "sampling/importance_sampling_ratio/max": 2.1841602325439453, + "sampling/importance_sampling_ratio/mean": 0.5139275193214417, + "sampling/importance_sampling_ratio/min": 0.026369251310825348, + "sampling/sampling_logp_difference/max": 0.42272377014160156, + "sampling/sampling_logp_difference/mean": 0.028494788333773613, + "step": 59, + "step_time": 24.42572767334059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 549.1875, + "completions/mean_terminated_length": 549.1875, + "completions/min_length": 489.0, + "completions/min_terminated_length": 489.0, + "entropy": 1.1738965958356857, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.588756799697876, + "kl": 0.025482238037511706, + "learning_rate": 3e-05, + "loss": 0.2925710678100586, + "num_tokens": 580229.0, + "reward": 6.0625, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 2.8465583324432373, + "sampling/importance_sampling_ratio/mean": 1.1227651834487915, + "sampling/importance_sampling_ratio/min": 0.1096419170498848, + "sampling/sampling_logp_difference/max": 0.4628920555114746, + "sampling/sampling_logp_difference/mean": 0.02885228767991066, + "step": 60, + "step_time": 21.57787229306996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 692.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 577.3125, + "completions/mean_terminated_length": 577.3125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.1913195550441742, + "epoch": 0.122, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2779391407966614, + "kl": 0.02629381464794278, + "learning_rate": 3e-05, + "loss": 0.12304374575614929, + "num_tokens": 591178.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.1122686862945557, + "sampling/importance_sampling_ratio/mean": 0.650765061378479, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.47726941108703613, + "sampling/sampling_logp_difference/mean": 0.027112768962979317, + "step": 61, + "step_time": 23.71764762001112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 628.3125, + "completions/mean_terminated_length": 628.3125, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3224291801452637, + "epoch": 0.124, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1645125448703766, + "kl": 0.025764177553355694, + "learning_rate": 3e-05, + "loss": 0.17419062554836273, + "num_tokens": 603055.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1662254333496094, + "sampling/importance_sampling_ratio/mean": 0.5809424519538879, + "sampling/importance_sampling_ratio/min": 0.03490918502211571, + "sampling/sampling_logp_difference/max": 0.4525270462036133, + "sampling/sampling_logp_difference/mean": 0.028948483988642693, + "step": 62, + "step_time": 35.74759274255484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 597.375, + "completions/mean_terminated_length": 597.375, + "completions/min_length": 478.0, + "completions/min_terminated_length": 478.0, + "entropy": 1.1552416533231735, + "epoch": 0.126, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20707836747169495, + "kl": 0.026473846402950585, + "learning_rate": 3e-05, + "loss": -0.019145065918564796, + "num_tokens": 614341.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.0268709659576416, + "sampling/importance_sampling_ratio/mean": 0.7066210508346558, + "sampling/importance_sampling_ratio/min": 0.1595209240913391, + "sampling/sampling_logp_difference/max": 0.42907285690307617, + "sampling/sampling_logp_difference/mean": 0.028000790625810623, + "step": 63, + "step_time": 39.37250621803105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 568.5, + "completions/mean_terminated_length": 568.5, + "completions/min_length": 509.0, + "completions/min_terminated_length": 509.0, + "entropy": 1.2810933291912079, + "epoch": 0.128, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21552008390426636, + "kl": 0.029041914734989405, + "learning_rate": 3e-05, + "loss": 0.037037670612335205, + "num_tokens": 625165.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.1896748542785645, + "sampling/importance_sampling_ratio/mean": 0.6408567428588867, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.374176025390625, + "sampling/sampling_logp_difference/mean": 0.02858484350144863, + "step": 64, + "step_time": 18.69576471252367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 622.75, + "completions/mean_terminated_length": 622.75, + "completions/min_length": 545.0, + "completions/min_terminated_length": 545.0, + "entropy": 1.32467532902956, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2287176102399826, + "kl": 0.023987084976397455, + "learning_rate": 3e-05, + "loss": 0.0731797143816948, + "num_tokens": 636633.0, + "reward": 6.375, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 1.4288272857666016, + "sampling/importance_sampling_ratio/mean": 0.5977773666381836, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4503474235534668, + "sampling/sampling_logp_difference/mean": 0.02755960263311863, + "step": 65, + "step_time": 27.502957582939416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 767.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 639.25, + "completions/mean_terminated_length": 639.25, + "completions/min_length": 554.0, + "completions/min_terminated_length": 554.0, + "entropy": 1.400998167693615, + "epoch": 0.132, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27425676584243774, + "kl": 0.028104660217650235, + "learning_rate": 3e-05, + "loss": 0.10324293375015259, + "num_tokens": 648597.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.971261501312256, + "sampling/importance_sampling_ratio/mean": 0.6433250904083252, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4819211959838867, + "sampling/sampling_logp_difference/mean": 0.029852069914340973, + "step": 66, + "step_time": 26.79405551031232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 541.375, + "completions/mean_terminated_length": 541.375, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.2635268718004227, + "epoch": 0.134, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13181555271148682, + "kl": 0.03373931790702045, + "learning_rate": 3e-05, + "loss": -0.11447598040103912, + "num_tokens": 658875.0, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 1.1883972883224487, + "sampling/importance_sampling_ratio/mean": 0.4192371368408203, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.44372403621673584, + "sampling/sampling_logp_difference/mean": 0.02911720983684063, + "step": 67, + "step_time": 27.6137525443919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 842.0, + "completions/max_terminated_length": 842.0, + "completions/mean_length": 689.4375, + "completions/mean_terminated_length": 689.4375, + "completions/min_length": 544.0, + "completions/min_terminated_length": 544.0, + "entropy": 1.2496536895632744, + "epoch": 0.136, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20560002326965332, + "kl": 0.026702777307946235, + "learning_rate": 3e-05, + "loss": -0.10130438208580017, + "num_tokens": 671690.0, + "reward": 5.875, + "reward_std": 2.0289571285247803, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 2.0289571285247803, + "sampling/importance_sampling_ratio/max": 2.8383262157440186, + "sampling/importance_sampling_ratio/mean": 0.9476768374443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35082435607910156, + "sampling/sampling_logp_difference/mean": 0.028364315629005432, + "step": 68, + "step_time": 29.35345455724746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 701.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 614.25, + "completions/mean_terminated_length": 614.25, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.1948458775877953, + "epoch": 0.138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20431844890117645, + "kl": 0.03377161466050893, + "learning_rate": 3e-05, + "loss": -0.0560678169131279, + "num_tokens": 683046.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.162766933441162, + "sampling/importance_sampling_ratio/mean": 0.5678162574768066, + "sampling/importance_sampling_ratio/min": 0.05678822472691536, + "sampling/sampling_logp_difference/max": 0.5758893489837646, + "sampling/sampling_logp_difference/mean": 0.028125843033194542, + "step": 69, + "step_time": 27.574916049838066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 588.3125, + "completions/mean_terminated_length": 588.3125, + "completions/min_length": 532.0, + "completions/min_terminated_length": 532.0, + "entropy": 1.2782762721180916, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26451998949050903, + "kl": 0.03907236340455711, + "learning_rate": 3e-05, + "loss": 0.17035090923309326, + "num_tokens": 694323.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.950176477432251, + "sampling/importance_sampling_ratio/mean": 0.45171743631362915, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46894216537475586, + "sampling/sampling_logp_difference/mean": 0.02863166108727455, + "step": 70, + "step_time": 19.817490340210497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 597.125, + "completions/mean_terminated_length": 597.125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2051330730319023, + "epoch": 0.142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5549097657203674, + "kl": 0.03670362115371972, + "learning_rate": 3e-05, + "loss": 0.4998631179332733, + "num_tokens": 705773.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.9404170513153076, + "sampling/importance_sampling_ratio/mean": 0.7522475123405457, + "sampling/importance_sampling_ratio/min": 0.05456363409757614, + "sampling/sampling_logp_difference/max": 0.4324514865875244, + "sampling/sampling_logp_difference/mean": 0.028340937569737434, + "step": 71, + "step_time": 41.66373607888818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 691.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 604.125, + "completions/mean_terminated_length": 604.125, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.18794547021389, + "epoch": 0.144, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10015435516834259, + "kl": 0.03911226138006896, + "learning_rate": 3e-05, + "loss": -0.02419177070260048, + "num_tokens": 717159.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1231037378311157, + "sampling/importance_sampling_ratio/mean": 0.4037884473800659, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6400742530822754, + "sampling/sampling_logp_difference/mean": 0.028367817401885986, + "step": 72, + "step_time": 23.86539705330506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 721.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 597.1875, + "completions/mean_terminated_length": 597.1875, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.2641174346208572, + "epoch": 0.146, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07918722927570343, + "kl": 0.03177848691120744, + "learning_rate": 3e-05, + "loss": -0.027635926380753517, + "num_tokens": 728402.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.5195796489715576, + "sampling/importance_sampling_ratio/mean": 0.4324396848678589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6819734573364258, + "sampling/sampling_logp_difference/mean": 0.030029678717255592, + "step": 73, + "step_time": 29.29490938829258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 640.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 574.375, + "completions/mean_terminated_length": 574.375, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3364054411649704, + "epoch": 0.148, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3853444755077362, + "kl": 0.03433372196741402, + "learning_rate": 3e-05, + "loss": -0.031142480671405792, + "num_tokens": 739632.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6853926181793213, + "sampling/importance_sampling_ratio/mean": 0.9200767874717712, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42021608352661133, + "sampling/sampling_logp_difference/mean": 0.030795859172940254, + "step": 74, + "step_time": 34.52008486771956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 618.25, + "completions/mean_terminated_length": 618.25, + "completions/min_length": 539.0, + "completions/min_terminated_length": 539.0, + "entropy": 1.365300178527832, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18101376295089722, + "kl": 0.02779634529724717, + "learning_rate": 3e-05, + "loss": -0.2718795835971832, + "num_tokens": 751164.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.8397568464279175, + "sampling/importance_sampling_ratio/mean": 0.5582400560379028, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42606019973754883, + "sampling/sampling_logp_difference/mean": 0.028895940631628036, + "step": 75, + "step_time": 18.76476171752438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 799.0, + "completions/max_terminated_length": 799.0, + "completions/mean_length": 603.375, + "completions/mean_terminated_length": 603.375, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2181015871465206, + "epoch": 0.152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1852622777223587, + "kl": 0.03176840906962752, + "learning_rate": 3e-05, + "loss": -0.10660596191883087, + "num_tokens": 762370.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.317336082458496, + "sampling/importance_sampling_ratio/mean": 0.550554633140564, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.481325626373291, + "sampling/sampling_logp_difference/mean": 0.028557566925883293, + "step": 76, + "step_time": 27.090129756834358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 572.9375, + "completions/mean_terminated_length": 572.9375, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2986655682325363, + "epoch": 0.154, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1531844586133957, + "kl": 0.03523328877054155, + "learning_rate": 3e-05, + "loss": -0.20856647193431854, + "num_tokens": 773169.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 2.855010986328125, + "sampling/importance_sampling_ratio/mean": 0.8239375352859497, + "sampling/importance_sampling_ratio/min": 0.1521405428647995, + "sampling/sampling_logp_difference/max": 0.4692528247833252, + "sampling/sampling_logp_difference/mean": 0.029906686395406723, + "step": 77, + "step_time": 25.004970545414835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 607.1875, + "completions/mean_terminated_length": 607.1875, + "completions/min_length": 540.0, + "completions/min_terminated_length": 540.0, + "entropy": 1.1003647185862064, + "epoch": 0.156, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14113759994506836, + "kl": 0.025135049945674837, + "learning_rate": 3e-05, + "loss": -0.10802068561315536, + "num_tokens": 784724.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.0096027851104736, + "sampling/importance_sampling_ratio/mean": 0.613497257232666, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4082927703857422, + "sampling/sampling_logp_difference/mean": 0.027884263545274734, + "step": 78, + "step_time": 29.614154959097505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 568.0625, + "completions/mean_terminated_length": 568.0625, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.266264721751213, + "epoch": 0.158, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18073000013828278, + "kl": 0.028119354974478483, + "learning_rate": 3e-05, + "loss": -0.0687609314918518, + "num_tokens": 795517.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.277299404144287, + "sampling/importance_sampling_ratio/mean": 0.3485238552093506, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4658241271972656, + "sampling/sampling_logp_difference/mean": 0.029626762494444847, + "step": 79, + "step_time": 23.5287338164635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 565.8125, + "completions/mean_terminated_length": 565.8125, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2773499339818954, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.31362995505332947, + "kl": 0.028471783734858036, + "learning_rate": 3e-05, + "loss": 0.059164684265851974, + "num_tokens": 806258.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.5347814559936523, + "sampling/importance_sampling_ratio/mean": 0.7027873396873474, + "sampling/importance_sampling_ratio/min": 0.06356429308652878, + "sampling/sampling_logp_difference/max": 0.4123659133911133, + "sampling/sampling_logp_difference/mean": 0.02946357987821102, + "step": 80, + "step_time": 24.251087154261768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 600.0, + "completions/max_terminated_length": 600.0, + "completions/mean_length": 536.1875, + "completions/mean_terminated_length": 536.1875, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.211024284362793, + "epoch": 0.162, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1959177404642105, + "kl": 0.02152467134874314, + "learning_rate": 3e-05, + "loss": 0.14755703508853912, + "num_tokens": 816717.0, + "reward": 6.0, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.764387369155884, + "sampling/importance_sampling_ratio/mean": 0.8167816400527954, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.28648805618286133, + "sampling/sampling_logp_difference/mean": 0.02814806066453457, + "step": 81, + "step_time": 22.752198832575232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 568.625, + "completions/mean_terminated_length": 568.625, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "entropy": 1.2584010139107704, + "epoch": 0.164, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16224367916584015, + "kl": 0.02812566957436502, + "learning_rate": 3e-05, + "loss": -0.1586945354938507, + "num_tokens": 827591.0, + "reward": 6.5, + "reward_std": 1.26491117477417, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.26491117477417, + "sampling/importance_sampling_ratio/max": 2.4172537326812744, + "sampling/importance_sampling_ratio/mean": 0.7026975154876709, + "sampling/importance_sampling_ratio/min": 0.1125984862446785, + "sampling/sampling_logp_difference/max": 0.3966444730758667, + "sampling/sampling_logp_difference/mean": 0.02937215007841587, + "step": 82, + "step_time": 22.57465209439397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 783.0, + "completions/max_terminated_length": 783.0, + "completions/mean_length": 583.5625, + "completions/mean_terminated_length": 583.5625, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2498536258935928, + "epoch": 0.166, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28551891446113586, + "kl": 0.023335880250670016, + "learning_rate": 3e-05, + "loss": 0.09868354350328445, + "num_tokens": 838760.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 1.9430233240127563, + "sampling/importance_sampling_ratio/mean": 0.7391272783279419, + "sampling/importance_sampling_ratio/min": 0.2032935619354248, + "sampling/sampling_logp_difference/max": 0.3390723466873169, + "sampling/sampling_logp_difference/mean": 0.02833949774503708, + "step": 83, + "step_time": 24.9633763525635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 581.0625, + "completions/mean_terminated_length": 581.0625, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.1582137942314148, + "epoch": 0.168, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1089889332652092, + "kl": 0.02546319324756041, + "learning_rate": 3e-05, + "loss": -0.04368923604488373, + "num_tokens": 849945.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.6800583600997925, + "sampling/importance_sampling_ratio/mean": 0.4864083528518677, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48818397521972656, + "sampling/sampling_logp_difference/mean": 0.02942320704460144, + "step": 84, + "step_time": 22.7196712391451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 553.3125, + "completions/mean_terminated_length": 553.3125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.1762890554964542, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18219847977161407, + "kl": 0.023275951389223337, + "learning_rate": 3e-05, + "loss": 0.055040232837200165, + "num_tokens": 860734.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.2940757274627686, + "sampling/importance_sampling_ratio/mean": 0.6381184458732605, + "sampling/importance_sampling_ratio/min": 0.08803392946720123, + "sampling/sampling_logp_difference/max": 0.3728243112564087, + "sampling/sampling_logp_difference/mean": 0.028103860095143318, + "step": 85, + "step_time": 28.569310082122684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 538.5, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.1561524420976639, + "epoch": 0.172, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2631295323371887, + "kl": 0.027657793601974845, + "learning_rate": 3e-05, + "loss": 0.019699495285749435, + "num_tokens": 871182.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.950981378555298, + "sampling/importance_sampling_ratio/mean": 0.5496660470962524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.358644962310791, + "sampling/sampling_logp_difference/mean": 0.02922222763299942, + "step": 86, + "step_time": 19.95468496438116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 688.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 528.625, + "completions/mean_terminated_length": 528.625, + "completions/min_length": 418.0, + "completions/min_terminated_length": 418.0, + "entropy": 1.382395550608635, + "epoch": 0.174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25233277678489685, + "kl": 0.025461523793637753, + "learning_rate": 3e-05, + "loss": -0.0012568621896207333, + "num_tokens": 881272.0, + "reward": 6.3125, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 1.940340280532837, + "sampling/importance_sampling_ratio/mean": 0.44232380390167236, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3433331251144409, + "sampling/sampling_logp_difference/mean": 0.030555889010429382, + "step": 87, + "step_time": 29.044239778537303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 537.0, + "completions/mean_terminated_length": 537.0, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.1740282103419304, + "epoch": 0.176, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1609290987253189, + "kl": 0.023582924506627023, + "learning_rate": 3e-05, + "loss": -0.0040507810190320015, + "num_tokens": 891608.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.8592076301574707, + "sampling/importance_sampling_ratio/mean": 0.4413543939590454, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3994784355163574, + "sampling/sampling_logp_difference/mean": 0.028627343475818634, + "step": 88, + "step_time": 24.642031190916896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 562.0, + "completions/mean_terminated_length": 562.0, + "completions/min_length": 490.0, + "completions/min_terminated_length": 490.0, + "entropy": 1.3354259580373764, + "epoch": 0.178, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25136521458625793, + "kl": 0.03104234568309039, + "learning_rate": 3e-05, + "loss": -0.10014888644218445, + "num_tokens": 902472.0, + "reward": 6.5, + "reward_std": 1.154700517654419, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.154700517654419, + "sampling/importance_sampling_ratio/max": 2.0172529220581055, + "sampling/importance_sampling_ratio/mean": 0.5528109073638916, + "sampling/importance_sampling_ratio/min": 0.031755149364471436, + "sampling/sampling_logp_difference/max": 0.48168420791625977, + "sampling/sampling_logp_difference/mean": 0.029525987803936005, + "step": 89, + "step_time": 23.934129936154932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 543.6875, + "completions/mean_terminated_length": 543.6875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.3072879239916801, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2416294664144516, + "kl": 0.02474795945454389, + "learning_rate": 3e-05, + "loss": 0.02994484454393387, + "num_tokens": 913003.0, + "reward": 6.125, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4122473001480103, + "sampling/importance_sampling_ratio/mean": 0.5672672390937805, + "sampling/importance_sampling_ratio/min": 0.1312582641839981, + "sampling/sampling_logp_difference/max": 0.36547136306762695, + "sampling/sampling_logp_difference/mean": 0.030115650966763496, + "step": 90, + "step_time": 16.719651044346392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 587.5, + "completions/mean_terminated_length": 587.5, + "completions/min_length": 491.0, + "completions/min_terminated_length": 491.0, + "entropy": 1.2642723061144352, + "epoch": 0.182, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11401186883449554, + "kl": 0.018764875654596835, + "learning_rate": 3e-05, + "loss": 0.17740829288959503, + "num_tokens": 923971.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.223915934562683, + "sampling/importance_sampling_ratio/mean": 0.4373893141746521, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.43628501892089844, + "sampling/sampling_logp_difference/mean": 0.027218280360102654, + "step": 91, + "step_time": 21.256958127953112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 519.3125, + "completions/mean_terminated_length": 519.3125, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.2509336844086647, + "epoch": 0.184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14000695943832397, + "kl": 0.017409664229489863, + "learning_rate": 3e-05, + "loss": -0.1092228814959526, + "num_tokens": 933880.0, + "reward": 7.125, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 2.4079525470733643, + "sampling/importance_sampling_ratio/mean": 0.6406391263008118, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3843420743942261, + "sampling/sampling_logp_difference/mean": 0.02841360680758953, + "step": 92, + "step_time": 20.99564675288275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 681.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 587.8125, + "completions/mean_terminated_length": 587.8125, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.1215453520417213, + "epoch": 0.186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16520461440086365, + "kl": 0.028165725176222622, + "learning_rate": 3e-05, + "loss": -0.15029624104499817, + "num_tokens": 945269.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.7825064659118652, + "sampling/importance_sampling_ratio/mean": 0.5902000069618225, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3571600914001465, + "sampling/sampling_logp_difference/mean": 0.02762974239885807, + "step": 93, + "step_time": 17.922352592926472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 566.0, + "completions/mean_terminated_length": 566.0, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.2864234894514084, + "epoch": 0.188, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24750091135501862, + "kl": 0.02070689742686227, + "learning_rate": 3e-05, + "loss": 0.2850193381309509, + "num_tokens": 956021.0, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "sampling/importance_sampling_ratio/max": 1.9146449565887451, + "sampling/importance_sampling_ratio/mean": 0.6849822402000427, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4927506446838379, + "sampling/sampling_logp_difference/mean": 0.029227914288640022, + "step": 94, + "step_time": 23.034203104209155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 516.6875, + "completions/mean_terminated_length": 516.6875, + "completions/min_length": 486.0, + "completions/min_terminated_length": 486.0, + "entropy": 1.2404684573411942, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11928244680166245, + "kl": 0.023086783126927912, + "learning_rate": 3e-05, + "loss": -0.032361116260290146, + "num_tokens": 965920.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.216193437576294, + "sampling/importance_sampling_ratio/mean": 0.32463955879211426, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37001991271972656, + "sampling/sampling_logp_difference/mean": 0.02843114361166954, + "step": 95, + "step_time": 15.103232022374868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 650.0, + "completions/max_terminated_length": 650.0, + "completions/mean_length": 548.4375, + "completions/mean_terminated_length": 548.4375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.2861761301755905, + "epoch": 0.192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2920665442943573, + "kl": 0.017841250344645232, + "learning_rate": 3e-05, + "loss": 0.1640928089618683, + "num_tokens": 976695.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.8954812288284302, + "sampling/importance_sampling_ratio/mean": 0.754618763923645, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31093156337738037, + "sampling/sampling_logp_difference/mean": 0.02842729538679123, + "step": 96, + "step_time": 40.74571412149817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 553.5625, + "completions/mean_terminated_length": 553.5625, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.1928813084959984, + "epoch": 0.194, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29983460903167725, + "kl": 0.020173200638964772, + "learning_rate": 3e-05, + "loss": 0.05926981568336487, + "num_tokens": 987120.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.0362496376037598, + "sampling/importance_sampling_ratio/mean": 0.6645779609680176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40996313095092773, + "sampling/sampling_logp_difference/mean": 0.02854262851178646, + "step": 97, + "step_time": 17.38945102225989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 546.1875, + "completions/mean_terminated_length": 546.1875, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "entropy": 1.365786962211132, + "epoch": 0.196, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12385546416044235, + "kl": 0.02262102661188692, + "learning_rate": 3e-05, + "loss": -0.09927551448345184, + "num_tokens": 997395.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2717533111572266, + "sampling/importance_sampling_ratio/mean": 0.5988417267799377, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35561084747314453, + "sampling/sampling_logp_difference/mean": 0.029298899695277214, + "step": 98, + "step_time": 23.35145698580891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 593.4375, + "completions/mean_terminated_length": 593.4375, + "completions/min_length": 508.0, + "completions/min_terminated_length": 508.0, + "entropy": 1.1754724085330963, + "epoch": 0.198, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2662176787853241, + "kl": 0.02589411090593785, + "learning_rate": 3e-05, + "loss": 0.2574020326137543, + "num_tokens": 1008458.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1308085918426514, + "sampling/importance_sampling_ratio/mean": 0.6420408487319946, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42550981044769287, + "sampling/sampling_logp_difference/mean": 0.02820964716374874, + "step": 99, + "step_time": 21.02622760878876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 538.4375, + "completions/mean_terminated_length": 538.4375, + "completions/min_length": 483.0, + "completions/min_terminated_length": 483.0, + "entropy": 1.3136962801218033, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2549664378166199, + "kl": 0.024644543533213437, + "learning_rate": 3e-05, + "loss": 0.06747792661190033, + "num_tokens": 1018793.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.3327062129974365, + "sampling/importance_sampling_ratio/mean": 0.7165549993515015, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4304838180541992, + "sampling/sampling_logp_difference/mean": 0.0299112256616354, + "step": 100, + "step_time": 16.768271412234753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 567.875, + "completions/mean_terminated_length": 567.875, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.15415108948946, + "epoch": 0.202, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34537598490715027, + "kl": 0.025688456720672548, + "learning_rate": 3e-05, + "loss": -0.21041882038116455, + "num_tokens": 1029751.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.282634973526001, + "sampling/importance_sampling_ratio/mean": 0.5392330288887024, + "sampling/importance_sampling_ratio/min": 0.026465320959687233, + "sampling/sampling_logp_difference/max": 0.3903813362121582, + "sampling/sampling_logp_difference/mean": 0.02962569333612919, + "step": 101, + "step_time": 16.715499105863273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 574.125, + "completions/mean_terminated_length": 574.125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.3766441270709038, + "epoch": 0.204, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27241969108581543, + "kl": 0.025680337683297694, + "learning_rate": 3e-05, + "loss": 0.24403473734855652, + "num_tokens": 1040601.0, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 2.3365371227264404, + "sampling/importance_sampling_ratio/mean": 0.6375491619110107, + "sampling/importance_sampling_ratio/min": 0.07846305519342422, + "sampling/sampling_logp_difference/max": 0.4158613681793213, + "sampling/sampling_logp_difference/mean": 0.030232973396778107, + "step": 102, + "step_time": 19.942134194541723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 557.75, + "completions/mean_terminated_length": 557.75, + "completions/min_length": 510.0, + "completions/min_terminated_length": 510.0, + "entropy": 1.3887510225176811, + "epoch": 0.206, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2711203992366791, + "kl": 0.022622586810030043, + "learning_rate": 3e-05, + "loss": -0.07210355252027512, + "num_tokens": 1051229.0, + "reward": 6.3125, + "reward_std": 1.0144785642623901, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.0144785642623901, + "sampling/importance_sampling_ratio/max": 1.8279200792312622, + "sampling/importance_sampling_ratio/mean": 0.5626887083053589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3629894256591797, + "sampling/sampling_logp_difference/mean": 0.030201904475688934, + "step": 103, + "step_time": 30.44108156999573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 544.75, + "completions/mean_terminated_length": 544.75, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.1656717136502266, + "epoch": 0.208, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27351662516593933, + "kl": 0.02198210428468883, + "learning_rate": 3e-05, + "loss": 0.06065649166703224, + "num_tokens": 1061745.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.500911235809326, + "sampling/importance_sampling_ratio/mean": 0.8908482789993286, + "sampling/importance_sampling_ratio/min": 0.05167346075177193, + "sampling/sampling_logp_difference/max": 0.3885481357574463, + "sampling/sampling_logp_difference/mean": 0.027608510106801987, + "step": 104, + "step_time": 16.46229960815981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 616.0, + "completions/max_terminated_length": 616.0, + "completions/mean_length": 562.875, + "completions/mean_terminated_length": 562.875, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.3012276738882065, + "epoch": 0.21, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21975310146808624, + "kl": 0.023721053614281118, + "learning_rate": 3e-05, + "loss": 0.06463687866926193, + "num_tokens": 1072231.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 1.7785983085632324, + "sampling/importance_sampling_ratio/mean": 0.5429776906967163, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.9359657764434814, + "sampling/sampling_logp_difference/mean": 0.030092012137174606, + "step": 105, + "step_time": 24.80119998846203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 580.0625, + "completions/mean_terminated_length": 580.0625, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.3428374752402306, + "epoch": 0.212, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23242872953414917, + "kl": 0.025716557982377708, + "learning_rate": 3e-05, + "loss": -0.004262822680175304, + "num_tokens": 1083184.0, + "reward": 6.375, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.0552361011505127, + "sampling/importance_sampling_ratio/mean": 0.6959555745124817, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7246572971343994, + "sampling/sampling_logp_difference/mean": 0.02952728420495987, + "step": 106, + "step_time": 17.27699494967237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 655.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 589.75, + "completions/mean_terminated_length": 589.75, + "completions/min_length": 547.0, + "completions/min_terminated_length": 547.0, + "entropy": 1.471379242837429, + "epoch": 0.214, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23287689685821533, + "kl": 0.025674917036667466, + "learning_rate": 3e-05, + "loss": 0.08491670340299606, + "num_tokens": 1094204.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.1368408203125, + "sampling/importance_sampling_ratio/mean": 0.5767678022384644, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36995506286621094, + "sampling/sampling_logp_difference/mean": 0.02880111336708069, + "step": 107, + "step_time": 40.17427978478372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 595.6875, + "completions/mean_terminated_length": 595.6875, + "completions/min_length": 524.0, + "completions/min_terminated_length": 524.0, + "entropy": 1.2550728917121887, + "epoch": 0.216, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09948146343231201, + "kl": 0.022876911563798785, + "learning_rate": 3e-05, + "loss": 0.04861483350396156, + "num_tokens": 1105303.0, + "reward": 6.125, + "reward_std": 1.0878112316131592, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0878112316131592, + "sampling/importance_sampling_ratio/max": 1.1923820972442627, + "sampling/importance_sampling_ratio/mean": 0.3343955874443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3477153778076172, + "sampling/sampling_logp_difference/mean": 0.029913678765296936, + "step": 108, + "step_time": 18.59066634438932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 583.5, + "completions/mean_terminated_length": 583.5, + "completions/min_length": 467.0, + "completions/min_terminated_length": 467.0, + "entropy": 1.2097205966711044, + "epoch": 0.218, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1392563134431839, + "kl": 0.0263087993953377, + "learning_rate": 3e-05, + "loss": 0.10488568246364594, + "num_tokens": 1116591.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.0463244915008545, + "sampling/importance_sampling_ratio/mean": 0.4996475875377655, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31003570556640625, + "sampling/sampling_logp_difference/mean": 0.0288787130266428, + "step": 109, + "step_time": 17.845282280817628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 571.375, + "completions/mean_terminated_length": 571.375, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.28146480768919, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5394540429115295, + "kl": 0.024339908617548645, + "learning_rate": 3e-05, + "loss": -0.23892748355865479, + "num_tokens": 1127493.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.671478509902954, + "sampling/importance_sampling_ratio/mean": 1.223832130432129, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650541305541992, + "sampling/sampling_logp_difference/mean": 0.028643080964684486, + "step": 110, + "step_time": 22.93386760680005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 493.875, + "completions/mean_terminated_length": 493.875, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 1.12203798443079, + "epoch": 0.222, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4297163188457489, + "kl": 0.024493444827385247, + "learning_rate": 3e-05, + "loss": -0.21332937479019165, + "num_tokens": 1136979.0, + "reward": 6.9375, + "reward_std": 0.6800735592842102, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.6800735592842102, + "sampling/importance_sampling_ratio/max": 2.889394760131836, + "sampling/importance_sampling_ratio/mean": 0.7321407794952393, + "sampling/importance_sampling_ratio/min": 0.02116413414478302, + "sampling/sampling_logp_difference/max": 0.49600934982299805, + "sampling/sampling_logp_difference/mean": 0.028577474877238274, + "step": 111, + "step_time": 21.056686570402235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 672.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 583.8125, + "completions/mean_terminated_length": 583.8125, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.4159239530563354, + "epoch": 0.224, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.008436380885541439, + "kl": 0.024869016953743994, + "learning_rate": 3e-05, + "loss": 0.0004943995736539364, + "num_tokens": 1148176.0, + "reward": 7.0, + "reward_std": 0.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.642366886138916, + "sampling/importance_sampling_ratio/mean": 0.7060814499855042, + "sampling/importance_sampling_ratio/min": 0.006825839169323444, + "sampling/sampling_logp_difference/max": 0.572547435760498, + "sampling/sampling_logp_difference/mean": 0.03075096383690834, + "step": 112, + "step_time": 20.1555822850205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 638.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 582.75, + "completions/mean_terminated_length": 582.75, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2932439520955086, + "epoch": 0.226, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2439681738615036, + "kl": 0.025248280493542552, + "learning_rate": 3e-05, + "loss": -0.22745110094547272, + "num_tokens": 1159380.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.5284109115600586, + "sampling/importance_sampling_ratio/mean": 1.0150926113128662, + "sampling/importance_sampling_ratio/min": 0.021104907616972923, + "sampling/sampling_logp_difference/max": 0.2513730525970459, + "sampling/sampling_logp_difference/mean": 0.02884429693222046, + "step": 113, + "step_time": 18.09852197067812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 580.0, + "completions/mean_terminated_length": 580.0, + "completions/min_length": 535.0, + "completions/min_terminated_length": 535.0, + "entropy": 1.246352232992649, + "epoch": 0.228, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23755788803100586, + "kl": 0.02679906424600631, + "learning_rate": 3e-05, + "loss": 0.3550976812839508, + "num_tokens": 1170628.0, + "reward": 7.25, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.750225305557251, + "sampling/importance_sampling_ratio/mean": 1.0749173164367676, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5576918125152588, + "sampling/sampling_logp_difference/mean": 0.03159492090344429, + "step": 114, + "step_time": 24.629896679893136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 657.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 562.6875, + "completions/mean_terminated_length": 562.6875, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.253239594399929, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3102787733078003, + "kl": 0.027133187628351152, + "learning_rate": 3e-05, + "loss": -0.05118201673030853, + "num_tokens": 1181295.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.634671926498413, + "sampling/importance_sampling_ratio/mean": 0.8949281573295593, + "sampling/importance_sampling_ratio/min": 0.09920533001422882, + "sampling/sampling_logp_difference/max": 0.6224374771118164, + "sampling/sampling_logp_difference/mean": 0.030200565233826637, + "step": 115, + "step_time": 38.715506959706545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 610.9375, + "completions/mean_terminated_length": 610.9375, + "completions/min_length": 538.0, + "completions/min_terminated_length": 538.0, + "entropy": 1.2940760999917984, + "epoch": 0.232, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14933602511882782, + "kl": 0.031228074803948402, + "learning_rate": 3e-05, + "loss": -0.05550215765833855, + "num_tokens": 1192750.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6116093397140503, + "sampling/importance_sampling_ratio/mean": 0.41488125920295715, + "sampling/importance_sampling_ratio/min": 0.009796842001378536, + "sampling/sampling_logp_difference/max": 0.5450258255004883, + "sampling/sampling_logp_difference/mean": 0.03152918070554733, + "step": 116, + "step_time": 17.821606623008847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 560.8125, + "completions/mean_terminated_length": 560.8125, + "completions/min_length": 501.0, + "completions/min_terminated_length": 501.0, + "entropy": 1.21239273250103, + "epoch": 0.234, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23778750002384186, + "kl": 0.03231424337718636, + "learning_rate": 3e-05, + "loss": -0.09421571344137192, + "num_tokens": 1203219.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.1054162979125977, + "sampling/importance_sampling_ratio/mean": 0.7292319536209106, + "sampling/importance_sampling_ratio/min": 0.010288448072969913, + "sampling/sampling_logp_difference/max": 0.8687701225280762, + "sampling/sampling_logp_difference/mean": 0.030015992000699043, + "step": 117, + "step_time": 16.80020274501294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 583.4375, + "completions/mean_terminated_length": 583.4375, + "completions/min_length": 503.0, + "completions/min_terminated_length": 503.0, + "entropy": 1.0914121232926846, + "epoch": 0.236, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.177206888794899, + "kl": 0.027808396029286087, + "learning_rate": 3e-05, + "loss": 0.010135526768863201, + "num_tokens": 1214562.0, + "reward": 6.375, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.6413226127624512, + "sampling/importance_sampling_ratio/mean": 0.5455202460289001, + "sampling/importance_sampling_ratio/min": 0.14393718540668488, + "sampling/sampling_logp_difference/max": 0.37839651107788086, + "sampling/sampling_logp_difference/mean": 0.02755727432668209, + "step": 118, + "step_time": 32.04508572584018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 802.0, + "completions/max_terminated_length": 802.0, + "completions/mean_length": 623.875, + "completions/mean_terminated_length": 623.875, + "completions/min_length": 555.0, + "completions/min_terminated_length": 555.0, + "entropy": 1.2099597677588463, + "epoch": 0.238, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10887355357408524, + "kl": 0.029803494922816753, + "learning_rate": 3e-05, + "loss": -0.05460066720843315, + "num_tokens": 1226136.0, + "reward": 6.25, + "reward_std": 1.983263373374939, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.983263373374939, + "sampling/importance_sampling_ratio/max": 1.5456031560897827, + "sampling/importance_sampling_ratio/mean": 0.4625241756439209, + "sampling/importance_sampling_ratio/min": 0.06713607162237167, + "sampling/sampling_logp_difference/max": 0.5650186538696289, + "sampling/sampling_logp_difference/mean": 0.03079008124768734, + "step": 119, + "step_time": 20.80143166380003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 621.0, + "completions/mean_terminated_length": 621.0, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.3042216151952744, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1758948117494583, + "kl": 0.03042414935771376, + "learning_rate": 3e-05, + "loss": -0.1489834189414978, + "num_tokens": 1237856.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.4328413009643555, + "sampling/importance_sampling_ratio/mean": 0.7058912515640259, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40829265117645264, + "sampling/sampling_logp_difference/mean": 0.029741039499640465, + "step": 120, + "step_time": 17.90572352707386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 593.8125, + "completions/mean_terminated_length": 593.8125, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 1.132676463574171, + "epoch": 0.242, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42553070187568665, + "kl": 0.04428348131477833, + "learning_rate": 3e-05, + "loss": 0.22666211426258087, + "num_tokens": 1249173.0, + "reward": 5.625, + "reward_std": 1.8574175834655762, + "rewards/quality_reward/mean": 5.625, + "rewards/quality_reward/std": 1.8574175834655762, + "sampling/importance_sampling_ratio/max": 2.5172836780548096, + "sampling/importance_sampling_ratio/mean": 0.6919515132904053, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6260476112365723, + "sampling/sampling_logp_difference/mean": 0.030399736016988754, + "step": 121, + "step_time": 38.02521731564775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 587.625, + "completions/mean_terminated_length": 587.625, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.2395975515246391, + "epoch": 0.244, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.248436838388443, + "kl": 0.03361149993725121, + "learning_rate": 3e-05, + "loss": 0.024570390582084656, + "num_tokens": 1260463.0, + "reward": 6.0625, + "reward_std": 1.236594796180725, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.236594796180725, + "sampling/importance_sampling_ratio/max": 2.833437442779541, + "sampling/importance_sampling_ratio/mean": 0.8825340867042542, + "sampling/importance_sampling_ratio/min": 0.06326956301927567, + "sampling/sampling_logp_difference/max": 0.540553092956543, + "sampling/sampling_logp_difference/mean": 0.030399201437830925, + "step": 122, + "step_time": 17.796182619407773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 585.3125, + "completions/mean_terminated_length": 585.3125, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.375852882862091, + "epoch": 0.246, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16029156744480133, + "kl": 0.03495740657672286, + "learning_rate": 3e-05, + "loss": -0.053390923887491226, + "num_tokens": 1271644.0, + "reward": 6.4375, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.6618704795837402, + "sampling/importance_sampling_ratio/mean": 0.5607253909111023, + "sampling/importance_sampling_ratio/min": 0.08802779763936996, + "sampling/sampling_logp_difference/max": 0.7028732299804688, + "sampling/sampling_logp_difference/mean": 0.031593482941389084, + "step": 123, + "step_time": 14.86260191956535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 630.1875, + "completions/mean_terminated_length": 630.1875, + "completions/min_length": 571.0, + "completions/min_terminated_length": 571.0, + "entropy": 1.3353190049529076, + "epoch": 0.248, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19288285076618195, + "kl": 0.033586084260605276, + "learning_rate": 3e-05, + "loss": -0.010514559224247932, + "num_tokens": 1283351.0, + "reward": 6.625, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4858638048171997, + "sampling/importance_sampling_ratio/mean": 0.5407211184501648, + "sampling/importance_sampling_ratio/min": 0.007580960635095835, + "sampling/sampling_logp_difference/max": 0.6886825561523438, + "sampling/sampling_logp_difference/mean": 0.030591927468776703, + "step": 124, + "step_time": 19.809663326013833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 599.875, + "completions/mean_terminated_length": 599.875, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.168940719217062, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15600983798503876, + "kl": 0.03213575447443873, + "learning_rate": 3e-05, + "loss": 0.10287950932979584, + "num_tokens": 1294997.0, + "reward": 7.0, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 1.832617163658142, + "sampling/importance_sampling_ratio/mean": 0.6814589500427246, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5988303422927856, + "sampling/sampling_logp_difference/mean": 0.030487919226288795, + "step": 125, + "step_time": 36.94939920771867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 634.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 582.25, + "completions/mean_terminated_length": 582.25, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.0670793130993843, + "epoch": 0.252, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3560245931148529, + "kl": 0.025753034162335098, + "learning_rate": 3e-05, + "loss": 0.16848862171173096, + "num_tokens": 1305993.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.17899489402771, + "sampling/importance_sampling_ratio/mean": 0.7458471059799194, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7731144428253174, + "sampling/sampling_logp_difference/mean": 0.029648227617144585, + "step": 126, + "step_time": 16.284966757521033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 717.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 650.8125, + "completions/mean_terminated_length": 650.8125, + "completions/min_length": 571.0, + "completions/min_terminated_length": 571.0, + "entropy": 1.2732752412557602, + "epoch": 0.254, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18122251331806183, + "kl": 0.03214431612286717, + "learning_rate": 3e-05, + "loss": 0.16780534386634827, + "num_tokens": 1318054.0, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 1.937699556350708, + "sampling/importance_sampling_ratio/mean": 0.5485143065452576, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.0368115901947021, + "sampling/sampling_logp_difference/mean": 0.03218457102775574, + "step": 127, + "step_time": 19.58785921242088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 743.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 617.375, + "completions/mean_terminated_length": 617.375, + "completions/min_length": 530.0, + "completions/min_terminated_length": 530.0, + "entropy": 1.306506209075451, + "epoch": 0.256, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1356409788131714, + "kl": 0.026579287368804216, + "learning_rate": 3e-05, + "loss": 0.06286599487066269, + "num_tokens": 1329500.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.106013298034668, + "sampling/importance_sampling_ratio/mean": 0.4681059420108795, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.676828145980835, + "sampling/sampling_logp_difference/mean": 0.032271042466163635, + "step": 128, + "step_time": 17.268729500938207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 599.0, + "completions/mean_terminated_length": 599.0, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.3493447452783585, + "epoch": 0.258, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2950517237186432, + "kl": 0.02614310395438224, + "learning_rate": 3e-05, + "loss": -0.059055861085653305, + "num_tokens": 1341020.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.7381844520568848, + "sampling/importance_sampling_ratio/mean": 0.6402126550674438, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5858409404754639, + "sampling/sampling_logp_difference/mean": 0.031067587435245514, + "step": 129, + "step_time": 28.869210730306804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 710.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 585.75, + "completions/mean_terminated_length": 585.75, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2956265732645988, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42696425318717957, + "kl": 0.02683100081048906, + "learning_rate": 3e-05, + "loss": 0.003650778206065297, + "num_tokens": 1351928.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.9060652256011963, + "sampling/importance_sampling_ratio/mean": 0.8535536527633667, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4930081367492676, + "sampling/sampling_logp_difference/mean": 0.031333789229393005, + "step": 130, + "step_time": 18.11609491892159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 568.5625, + "completions/mean_terminated_length": 568.5625, + "completions/min_length": 470.0, + "completions/min_terminated_length": 470.0, + "entropy": 1.2097233161330223, + "epoch": 0.262, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20076203346252441, + "kl": 0.024291134322993457, + "learning_rate": 3e-05, + "loss": -0.01662549562752247, + "num_tokens": 1362825.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.6266331672668457, + "sampling/importance_sampling_ratio/mean": 0.6302506327629089, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.963031530380249, + "sampling/sampling_logp_difference/mean": 0.0319429412484169, + "step": 131, + "step_time": 24.060474771540612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 741.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 605.9375, + "completions/mean_terminated_length": 605.9375, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.2823583781719208, + "epoch": 0.264, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.184434711933136, + "kl": 0.02043789450544864, + "learning_rate": 3e-05, + "loss": -0.0666906088590622, + "num_tokens": 1374296.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.831458568572998, + "sampling/importance_sampling_ratio/mean": 0.5425443649291992, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.671515941619873, + "sampling/sampling_logp_difference/mean": 0.030500290915369987, + "step": 132, + "step_time": 17.10482985060662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 665.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 591.375, + "completions/mean_terminated_length": 591.375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.1699804589152336, + "epoch": 0.266, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4572683274745941, + "kl": 0.025262096198275685, + "learning_rate": 3e-05, + "loss": 0.6353421211242676, + "num_tokens": 1385318.0, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.3584084510803223, + "sampling/importance_sampling_ratio/mean": 0.7995439767837524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3869748115539551, + "sampling/sampling_logp_difference/mean": 0.028536029160022736, + "step": 133, + "step_time": 36.004622367210686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 652.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 554.25, + "completions/mean_terminated_length": 554.25, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1336797252297401, + "epoch": 0.268, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.256655752658844, + "kl": 0.022568197746295482, + "learning_rate": 3e-05, + "loss": 0.026529084891080856, + "num_tokens": 1396234.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 1.9517148733139038, + "sampling/importance_sampling_ratio/mean": 0.8202446699142456, + "sampling/importance_sampling_ratio/min": 0.022647401317954063, + "sampling/sampling_logp_difference/max": 1.145315170288086, + "sampling/sampling_logp_difference/mean": 0.02867746911942959, + "step": 134, + "step_time": 20.057327402289957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 641.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 573.5, + "completions/mean_terminated_length": 573.5, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.1878332123160362, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30302342772483826, + "kl": 0.02298387698829174, + "learning_rate": 3e-05, + "loss": 0.2014756053686142, + "num_tokens": 1407322.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.9701545238494873, + "sampling/importance_sampling_ratio/mean": 0.6178270578384399, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4479391574859619, + "sampling/sampling_logp_difference/mean": 0.03068450093269348, + "step": 135, + "step_time": 36.01238542608917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 534.0, + "completions/mean_terminated_length": 534.0, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2640416622161865, + "epoch": 0.272, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39115583896636963, + "kl": 0.020954113570041955, + "learning_rate": 3e-05, + "loss": 0.3642374873161316, + "num_tokens": 1418010.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.0828306674957275, + "sampling/importance_sampling_ratio/mean": 0.7149391770362854, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5613884925842285, + "sampling/sampling_logp_difference/mean": 0.0298798568546772, + "step": 136, + "step_time": 20.20259432308376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 538.8125, + "completions/mean_terminated_length": 538.8125, + "completions/min_length": 451.0, + "completions/min_terminated_length": 451.0, + "entropy": 1.2438515722751617, + "epoch": 0.274, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4104433059692383, + "kl": 0.02069689182098955, + "learning_rate": 3e-05, + "loss": -0.15938539803028107, + "num_tokens": 1428335.0, + "reward": 6.875, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.013305425643921, + "sampling/importance_sampling_ratio/mean": 0.5245123505592346, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46263813972473145, + "sampling/sampling_logp_difference/mean": 0.030781995505094528, + "step": 137, + "step_time": 18.362532567232847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 542.4375, + "completions/mean_terminated_length": 542.4375, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2741251289844513, + "epoch": 0.276, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41347458958625793, + "kl": 0.020388772361911833, + "learning_rate": 3e-05, + "loss": 0.3297041654586792, + "num_tokens": 1438734.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.6671712398529053, + "sampling/importance_sampling_ratio/mean": 0.822363555431366, + "sampling/importance_sampling_ratio/min": 0.016625043004751205, + "sampling/sampling_logp_difference/max": 0.40987062454223633, + "sampling/sampling_logp_difference/mean": 0.02844768762588501, + "step": 138, + "step_time": 34.91616539563984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 516.3125, + "completions/mean_terminated_length": 516.3125, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.2030403539538383, + "epoch": 0.278, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3896540701389313, + "kl": 0.022598200594075024, + "learning_rate": 3e-05, + "loss": -0.25778308510780334, + "num_tokens": 1448611.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.9001679420471191, + "sampling/importance_sampling_ratio/mean": 0.6815162897109985, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.2820701599121094, + "sampling/sampling_logp_difference/mean": 0.027655858546495438, + "step": 139, + "step_time": 20.81112790806219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 512.8125, + "completions/mean_terminated_length": 512.8125, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "entropy": 1.124063789844513, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2338583916425705, + "kl": 0.022081921808421612, + "learning_rate": 3e-05, + "loss": 0.09288515895605087, + "num_tokens": 1458456.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.7245709896087646, + "sampling/importance_sampling_ratio/mean": 0.7086957693099976, + "sampling/importance_sampling_ratio/min": 0.13776090741157532, + "sampling/sampling_logp_difference/max": 0.4399615526199341, + "sampling/sampling_logp_difference/mean": 0.02669401653110981, + "step": 140, + "step_time": 22.541061893571168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 521.9375, + "completions/mean_terminated_length": 521.9375, + "completions/min_length": 460.0, + "completions/min_terminated_length": 460.0, + "entropy": 1.3581550270318985, + "epoch": 0.282, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1792808622121811, + "kl": 0.02498150523751974, + "learning_rate": 3e-05, + "loss": 0.08992868661880493, + "num_tokens": 1468623.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.6533762216567993, + "sampling/importance_sampling_ratio/mean": 0.5165826082229614, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42975759506225586, + "sampling/sampling_logp_difference/mean": 0.028398238122463226, + "step": 141, + "step_time": 20.935550182592124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 454.625, + "completions/mean_terminated_length": 454.625, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 1.2165675312280655, + "epoch": 0.284, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2593871057033539, + "kl": 0.024267837987281382, + "learning_rate": 3e-05, + "loss": 0.24750135838985443, + "num_tokens": 1477449.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 1.5698224306106567, + "sampling/importance_sampling_ratio/mean": 0.6540807485580444, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3056577444076538, + "sampling/sampling_logp_difference/mean": 0.029166901484131813, + "step": 142, + "step_time": 17.03540184069425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 490.625, + "completions/mean_terminated_length": 490.625, + "completions/min_length": 446.0, + "completions/min_terminated_length": 446.0, + "entropy": 1.1259984448552132, + "epoch": 0.286, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37285444140434265, + "kl": 0.019997276307549328, + "learning_rate": 3e-05, + "loss": 0.330167293548584, + "num_tokens": 1486931.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.156266927719116, + "sampling/importance_sampling_ratio/mean": 0.7264441251754761, + "sampling/importance_sampling_ratio/min": 0.07088703662157059, + "sampling/sampling_logp_difference/max": 0.42168426513671875, + "sampling/sampling_logp_difference/mean": 0.02719968557357788, + "step": 143, + "step_time": 15.82226228993386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 529.0625, + "completions/mean_terminated_length": 496.0666809082031, + "completions/min_length": 421.0, + "completions/min_terminated_length": 421.0, + "entropy": 1.0426596216857433, + "epoch": 0.288, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11882677674293518, + "kl": 0.027060870255809277, + "learning_rate": 3e-05, + "loss": 0.07405021786689758, + "num_tokens": 1496916.0, + "reward": 5.75, + "reward_std": 1.732050895690918, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.732050895690918, + "sampling/importance_sampling_ratio/max": 0.8768613934516907, + "sampling/importance_sampling_ratio/mean": 0.39511895179748535, + "sampling/importance_sampling_ratio/min": 0.11731626838445663, + "sampling/sampling_logp_difference/max": 0.6632819175720215, + "sampling/sampling_logp_difference/mean": 0.02569635957479477, + "step": 144, + "step_time": 21.100794151891023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 502.5625, + "completions/mean_terminated_length": 502.5625, + "completions/min_length": 452.0, + "completions/min_terminated_length": 452.0, + "entropy": 1.2609772458672523, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25876912474632263, + "kl": 0.028287828317843378, + "learning_rate": 3e-05, + "loss": -0.18078479170799255, + "num_tokens": 1507157.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.90565824508667, + "sampling/importance_sampling_ratio/mean": 0.7513852119445801, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3683091402053833, + "sampling/sampling_logp_difference/mean": 0.02768835797905922, + "step": 145, + "step_time": 19.17377450503409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 586.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 478.25, + "completions/mean_terminated_length": 478.25, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "entropy": 1.2076954543590546, + "epoch": 0.292, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4150021970272064, + "kl": 0.027647150680422783, + "learning_rate": 3e-05, + "loss": -0.07925756275653839, + "num_tokens": 1516833.0, + "reward": 6.5, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.0522961616516113, + "sampling/importance_sampling_ratio/mean": 0.6499220132827759, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40656137466430664, + "sampling/sampling_logp_difference/mean": 0.028425993397831917, + "step": 146, + "step_time": 19.426104408223182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 467.1875, + "completions/mean_terminated_length": 467.1875, + "completions/min_length": 396.0, + "completions/min_terminated_length": 396.0, + "entropy": 1.1012553870677948, + "epoch": 0.294, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33721745014190674, + "kl": 0.02999569766689092, + "learning_rate": 3e-05, + "loss": 0.04165569692850113, + "num_tokens": 1526028.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.507693290710449, + "sampling/importance_sampling_ratio/mean": 0.8091086149215698, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4148428440093994, + "sampling/sampling_logp_difference/mean": 0.028098180890083313, + "step": 147, + "step_time": 32.705999514088035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 467.0, + "completions/mean_terminated_length": 467.0, + "completions/min_length": 385.0, + "completions/min_terminated_length": 385.0, + "entropy": 1.2554677203297615, + "epoch": 0.296, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26061347126960754, + "kl": 0.0399768726201728, + "learning_rate": 3e-05, + "loss": -0.018139198422431946, + "num_tokens": 1535348.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.5010173320770264, + "sampling/importance_sampling_ratio/mean": 0.6952720880508423, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35097575187683105, + "sampling/sampling_logp_difference/mean": 0.028018539771437645, + "step": 148, + "step_time": 34.92355508869514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 486.75, + "completions/mean_terminated_length": 486.75, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "entropy": 1.2563373371958733, + "epoch": 0.298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2081325203180313, + "kl": 0.03278218396008015, + "learning_rate": 3e-05, + "loss": -0.01242863480001688, + "num_tokens": 1544912.0, + "reward": 5.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.0925099849700928, + "sampling/importance_sampling_ratio/mean": 0.4945816695690155, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45126640796661377, + "sampling/sampling_logp_difference/mean": 0.030079778283834457, + "step": 149, + "step_time": 26.784944237209857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 571.0, + "completions/max_terminated_length": 571.0, + "completions/mean_length": 483.0625, + "completions/mean_terminated_length": 483.0625, + "completions/min_length": 366.0, + "completions/min_terminated_length": 366.0, + "entropy": 1.1289120316505432, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40016695857048035, + "kl": 0.032314015785232186, + "learning_rate": 3e-05, + "loss": -0.1471974402666092, + "num_tokens": 1554417.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.876359701156616, + "sampling/importance_sampling_ratio/mean": 0.8288668394088745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.438828706741333, + "sampling/sampling_logp_difference/mean": 0.027187082916498184, + "step": 150, + "step_time": 25.687996607273817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 504.25, + "completions/mean_terminated_length": 504.25, + "completions/min_length": 436.0, + "completions/min_terminated_length": 436.0, + "entropy": 1.1718142479658127, + "epoch": 0.302, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3958420157432556, + "kl": 0.02723738143686205, + "learning_rate": 3e-05, + "loss": -0.3493230938911438, + "num_tokens": 1564101.0, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "sampling/importance_sampling_ratio/max": 2.8077433109283447, + "sampling/importance_sampling_ratio/mean": 0.7897872924804688, + "sampling/importance_sampling_ratio/min": 0.06443088501691818, + "sampling/sampling_logp_difference/max": 0.48229074478149414, + "sampling/sampling_logp_difference/mean": 0.02742597460746765, + "step": 151, + "step_time": 34.508475522045046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 519.0, + "completions/max_terminated_length": 519.0, + "completions/mean_length": 296.0, + "completions/mean_terminated_length": 296.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 1.0032543204724789, + "epoch": 0.304, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1944788247346878, + "kl": 0.030508540105074644, + "learning_rate": 3e-05, + "loss": -0.2918842136859894, + "num_tokens": 1570429.0, + "reward": 3.4375, + "reward_std": 3.558440685272217, + "rewards/quality_reward/mean": 3.4375, + "rewards/quality_reward/std": 3.558440685272217, + "sampling/importance_sampling_ratio/max": 1.85885751247406, + "sampling/importance_sampling_ratio/mean": 1.0381181240081787, + "sampling/importance_sampling_ratio/min": 0.1504185050725937, + "sampling/sampling_logp_difference/max": 0.4975461959838867, + "sampling/sampling_logp_difference/mean": 0.02710951678454876, + "step": 152, + "step_time": 22.075861463323236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 514.4375, + "completions/mean_terminated_length": 514.4375, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2780758067965508, + "epoch": 0.306, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4307408928871155, + "kl": 0.02628148818621412, + "learning_rate": 3e-05, + "loss": 0.33224302530288696, + "num_tokens": 1580372.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.5702013969421387, + "sampling/importance_sampling_ratio/mean": 0.8224437832832336, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48117589950561523, + "sampling/sampling_logp_difference/mean": 0.027627088129520416, + "step": 153, + "step_time": 18.07258096849546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 529.0, + "completions/max_terminated_length": 529.0, + "completions/mean_length": 467.125, + "completions/mean_terminated_length": 467.125, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.161174800246954, + "epoch": 0.308, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1959461271762848, + "kl": 0.026262085768394172, + "learning_rate": 3e-05, + "loss": 0.05762449651956558, + "num_tokens": 1589438.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.2318003177642822, + "sampling/importance_sampling_ratio/mean": 0.7094592452049255, + "sampling/importance_sampling_ratio/min": 0.11490790545940399, + "sampling/sampling_logp_difference/max": 0.43965983390808105, + "sampling/sampling_logp_difference/mean": 0.02732112817466259, + "step": 154, + "step_time": 22.3326059714891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 396.875, + "completions/mean_terminated_length": 396.875, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 1.1771309785544872, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4554482400417328, + "kl": 0.03402461623772979, + "learning_rate": 3e-05, + "loss": -0.19043315947055817, + "num_tokens": 1597588.0, + "reward": 5.5625, + "reward_std": 1.3149778842926025, + "rewards/quality_reward/mean": 5.5625, + "rewards/quality_reward/std": 1.3149778842926025, + "sampling/importance_sampling_ratio/max": 2.1980347633361816, + "sampling/importance_sampling_ratio/mean": 0.7672010064125061, + "sampling/importance_sampling_ratio/min": 0.10123267024755478, + "sampling/sampling_logp_difference/max": 0.5363807678222656, + "sampling/sampling_logp_difference/mean": 0.029761571437120438, + "step": 155, + "step_time": 18.508908156771213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 520.5, + "completions/mean_terminated_length": 520.5, + "completions/min_length": 422.0, + "completions/min_terminated_length": 422.0, + "entropy": 1.123583823442459, + "epoch": 0.312, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20946863293647766, + "kl": 0.02810170315206051, + "learning_rate": 3e-05, + "loss": 0.07504862546920776, + "num_tokens": 1607508.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.429632306098938, + "sampling/importance_sampling_ratio/mean": 0.631747305393219, + "sampling/importance_sampling_ratio/min": 0.016474967822432518, + "sampling/sampling_logp_difference/max": 0.40722954273223877, + "sampling/sampling_logp_difference/mean": 0.028051164001226425, + "step": 156, + "step_time": 17.622006124351174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 623.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 471.8125, + "completions/mean_terminated_length": 471.8125, + "completions/min_length": 374.0, + "completions/min_terminated_length": 374.0, + "entropy": 1.0345656536519527, + "epoch": 0.314, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3838019073009491, + "kl": 0.029524097801186144, + "learning_rate": 3e-05, + "loss": 0.29842275381088257, + "num_tokens": 1617121.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.64119291305542, + "sampling/importance_sampling_ratio/mean": 0.5510131120681763, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48647427558898926, + "sampling/sampling_logp_difference/mean": 0.02675374038517475, + "step": 157, + "step_time": 34.35223956173286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 479.3125, + "completions/mean_terminated_length": 479.3125, + "completions/min_length": 415.0, + "completions/min_terminated_length": 415.0, + "entropy": 1.197593629360199, + "epoch": 0.316, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14107273519039154, + "kl": 0.02758750319480896, + "learning_rate": 3e-05, + "loss": 0.07440078258514404, + "num_tokens": 1626462.0, + "reward": 6.1875, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.719329833984375, + "sampling/importance_sampling_ratio/mean": 0.515890896320343, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37659645080566406, + "sampling/sampling_logp_difference/mean": 0.02727513015270233, + "step": 158, + "step_time": 30.49356387415901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 465.4375, + "completions/mean_terminated_length": 465.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.182745985686779, + "epoch": 0.318, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22129040956497192, + "kl": 0.02928700065240264, + "learning_rate": 3e-05, + "loss": 0.0027256053872406483, + "num_tokens": 1635613.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.2551939487457275, + "sampling/importance_sampling_ratio/mean": 0.4655284583568573, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3532288074493408, + "sampling/sampling_logp_difference/mean": 0.027347734197974205, + "step": 159, + "step_time": 23.563114238902926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 480.9375, + "completions/mean_terminated_length": 480.9375, + "completions/min_length": 378.0, + "completions/min_terminated_length": 378.0, + "entropy": 1.0412059053778648, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22958238422870636, + "kl": 0.025641236337833107, + "learning_rate": 3e-05, + "loss": -0.1119004413485527, + "num_tokens": 1645060.0, + "reward": 5.8125, + "reward_std": 2.9033026695251465, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 2.9033026695251465, + "sampling/importance_sampling_ratio/max": 2.3590943813323975, + "sampling/importance_sampling_ratio/mean": 0.6600984334945679, + "sampling/importance_sampling_ratio/min": 0.16755303740501404, + "sampling/sampling_logp_difference/max": 0.3485531806945801, + "sampling/sampling_logp_difference/mean": 0.02425791323184967, + "step": 160, + "step_time": 17.564059282653034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 511.5625, + "completions/mean_terminated_length": 511.5625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.0806752033531666, + "epoch": 0.322, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2202390879392624, + "kl": 0.03178418125025928, + "learning_rate": 3e-05, + "loss": -0.24258574843406677, + "num_tokens": 1654901.0, + "reward": 6.5, + "reward_std": 0.9660918116569519, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.9660918116569519, + "sampling/importance_sampling_ratio/max": 2.616337776184082, + "sampling/importance_sampling_ratio/mean": 0.8924014568328857, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37401676177978516, + "sampling/sampling_logp_difference/mean": 0.02693682350218296, + "step": 161, + "step_time": 22.71096785319969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 484.1875, + "completions/mean_terminated_length": 484.1875, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.2182030156254768, + "epoch": 0.324, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2978271245956421, + "kl": 0.026724245748482645, + "learning_rate": 3e-05, + "loss": -0.0895138755440712, + "num_tokens": 1664568.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.0902533531188965, + "sampling/importance_sampling_ratio/mean": 0.9070306420326233, + "sampling/importance_sampling_ratio/min": 0.20267778635025024, + "sampling/sampling_logp_difference/max": 0.3564906120300293, + "sampling/sampling_logp_difference/mean": 0.02701719105243683, + "step": 162, + "step_time": 24.52080715773627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 543.0, + "completions/max_terminated_length": 543.0, + "completions/mean_length": 506.9375, + "completions/mean_terminated_length": 506.9375, + "completions/min_length": 452.0, + "completions/min_terminated_length": 452.0, + "entropy": 1.2944460734724998, + "epoch": 0.326, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13530702888965607, + "kl": 0.03180140187032521, + "learning_rate": 3e-05, + "loss": 0.016086462885141373, + "num_tokens": 1674423.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.1556015014648438, + "sampling/importance_sampling_ratio/mean": 0.583191990852356, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.1959445476531982, + "sampling/sampling_logp_difference/mean": 0.028113799169659615, + "step": 163, + "step_time": 21.659780140966177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 525.0, + "completions/max_terminated_length": 525.0, + "completions/mean_length": 490.5625, + "completions/mean_terminated_length": 490.5625, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.289131723344326, + "epoch": 0.328, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5087841153144836, + "kl": 0.026518055819906294, + "learning_rate": 3e-05, + "loss": 0.2851857542991638, + "num_tokens": 1683864.0, + "reward": 6.5, + "reward_std": 1.0327955484390259, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.0327955484390259, + "sampling/importance_sampling_ratio/max": 2.531486988067627, + "sampling/importance_sampling_ratio/mean": 0.8939677476882935, + "sampling/importance_sampling_ratio/min": 0.09362189471721649, + "sampling/sampling_logp_difference/max": 0.38115930557250977, + "sampling/sampling_logp_difference/mean": 0.028977636247873306, + "step": 164, + "step_time": 22.67840038659051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 570.0, + "completions/max_terminated_length": 570.0, + "completions/mean_length": 508.5, + "completions/mean_terminated_length": 508.5, + "completions/min_length": 465.0, + "completions/min_terminated_length": 465.0, + "entropy": 1.2583990097045898, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5895075798034668, + "kl": 0.03340678755193949, + "learning_rate": 3e-05, + "loss": 0.5024052858352661, + "num_tokens": 1693592.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.5420279502868652, + "sampling/importance_sampling_ratio/mean": 0.8359720706939697, + "sampling/importance_sampling_ratio/min": 0.13951139152050018, + "sampling/sampling_logp_difference/max": 0.25212621688842773, + "sampling/sampling_logp_difference/mean": 0.027791393920779228, + "step": 165, + "step_time": 23.054075544700027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 522.625, + "completions/mean_terminated_length": 522.625, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.2723611742258072, + "epoch": 0.332, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4107528030872345, + "kl": 0.028830039431340992, + "learning_rate": 3e-05, + "loss": 0.43730953335762024, + "num_tokens": 1703722.0, + "reward": 7.25, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 1.8613929748535156, + "sampling/importance_sampling_ratio/mean": 0.5325981378555298, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33945512771606445, + "sampling/sampling_logp_difference/mean": 0.028407979756593704, + "step": 166, + "step_time": 14.466566514223814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 485.4375, + "completions/mean_terminated_length": 485.4375, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.159641794860363, + "epoch": 0.334, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23576849699020386, + "kl": 0.03369407169520855, + "learning_rate": 3e-05, + "loss": -0.17192532122135162, + "num_tokens": 1713129.0, + "reward": 6.9375, + "reward_std": 0.8539125919342041, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.8539125919342041, + "sampling/importance_sampling_ratio/max": 2.364237070083618, + "sampling/importance_sampling_ratio/mean": 0.835480809211731, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4001603126525879, + "sampling/sampling_logp_difference/mean": 0.027760744094848633, + "step": 167, + "step_time": 15.427942908834666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 531.125, + "completions/mean_terminated_length": 531.125, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.1074568964540958, + "epoch": 0.336, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14523518085479736, + "kl": 0.032465216936543584, + "learning_rate": 3e-05, + "loss": -0.10911832749843597, + "num_tokens": 1723371.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.233501672744751, + "sampling/importance_sampling_ratio/mean": 0.5449534058570862, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42547130584716797, + "sampling/sampling_logp_difference/mean": 0.026400625705718994, + "step": 168, + "step_time": 14.74156094249338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 629.0, + "completions/max_terminated_length": 629.0, + "completions/mean_length": 518.25, + "completions/mean_terminated_length": 518.25, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.2571639120578766, + "epoch": 0.338, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3119359314441681, + "kl": 0.03448521322570741, + "learning_rate": 3e-05, + "loss": 0.14656513929367065, + "num_tokens": 1733279.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.882887601852417, + "sampling/importance_sampling_ratio/mean": 0.8524343967437744, + "sampling/importance_sampling_ratio/min": 0.061056625097990036, + "sampling/sampling_logp_difference/max": 0.34301328659057617, + "sampling/sampling_logp_difference/mean": 0.027896685525774956, + "step": 169, + "step_time": 22.03652939805761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 552.8125, + "completions/mean_terminated_length": 552.8125, + "completions/min_length": 454.0, + "completions/min_terminated_length": 454.0, + "entropy": 1.2771715074777603, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23340070247650146, + "kl": 0.03460722556337714, + "learning_rate": 3e-05, + "loss": -0.11407067626714706, + "num_tokens": 1743740.0, + "reward": 6.8125, + "reward_std": 0.6551081538200378, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.6551081538200378, + "sampling/importance_sampling_ratio/max": 1.6628351211547852, + "sampling/importance_sampling_ratio/mean": 0.5497220754623413, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.314577579498291, + "sampling/sampling_logp_difference/mean": 0.03021315485239029, + "step": 170, + "step_time": 18.62061845092103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 564.0, + "completions/max_terminated_length": 564.0, + "completions/mean_length": 495.1875, + "completions/mean_terminated_length": 495.1875, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "entropy": 1.1425480283796787, + "epoch": 0.342, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07674646377563477, + "kl": 0.0316173325991258, + "learning_rate": 3e-05, + "loss": -0.012545892037451267, + "num_tokens": 1753231.0, + "reward": 6.4375, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.37643563747406, + "sampling/importance_sampling_ratio/mean": 0.4176323413848877, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5454483032226562, + "sampling/sampling_logp_difference/mean": 0.027837729081511497, + "step": 171, + "step_time": 35.69278695946559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 536.5, + "completions/mean_terminated_length": 536.5, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "entropy": 1.3311709240078926, + "epoch": 0.344, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5406383275985718, + "kl": 0.03873496490996331, + "learning_rate": 3e-05, + "loss": 0.2167063057422638, + "num_tokens": 1763511.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.74458646774292, + "sampling/importance_sampling_ratio/mean": 1.1003804206848145, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5733523368835449, + "sampling/sampling_logp_difference/mean": 0.028938323259353638, + "step": 172, + "step_time": 19.092736863531172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 508.0625, + "completions/mean_terminated_length": 508.0625, + "completions/min_length": 437.0, + "completions/min_terminated_length": 437.0, + "entropy": 1.22428647428751, + "epoch": 0.346, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11082098633050919, + "kl": 0.03200511261820793, + "learning_rate": 3e-05, + "loss": 0.07203174382448196, + "num_tokens": 1773304.0, + "reward": 7.125, + "reward_std": 0.5, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8160909414291382, + "sampling/importance_sampling_ratio/mean": 0.5755537748336792, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.595893383026123, + "sampling/sampling_logp_difference/mean": 0.028249088674783707, + "step": 173, + "step_time": 36.1855756030418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 720.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 556.9375, + "completions/mean_terminated_length": 556.9375, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.1718761064112186, + "epoch": 0.348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3865050971508026, + "kl": 0.03591357555706054, + "learning_rate": 3e-05, + "loss": 0.2996499538421631, + "num_tokens": 1784039.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.0816619396209717, + "sampling/importance_sampling_ratio/mean": 0.5800511240959167, + "sampling/importance_sampling_ratio/min": 0.09306051582098007, + "sampling/sampling_logp_difference/max": 0.41143274307250977, + "sampling/sampling_logp_difference/mean": 0.027585921809077263, + "step": 174, + "step_time": 20.269209342077374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 494.6875, + "completions/mean_terminated_length": 494.6875, + "completions/min_length": 442.0, + "completions/min_terminated_length": 442.0, + "entropy": 1.3337711468338966, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11888998746871948, + "kl": 0.03842530632391572, + "learning_rate": 3e-05, + "loss": 0.07849398255348206, + "num_tokens": 1793682.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.3145290613174438, + "sampling/importance_sampling_ratio/mean": 0.4274219870567322, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4447822570800781, + "sampling/sampling_logp_difference/mean": 0.031242625787854195, + "step": 175, + "step_time": 17.293509372044355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 544.8125, + "completions/mean_terminated_length": 544.8125, + "completions/min_length": 462.0, + "completions/min_terminated_length": 462.0, + "entropy": 1.19626072794199, + "epoch": 0.352, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2991865277290344, + "kl": 0.036185722798109055, + "learning_rate": 3e-05, + "loss": 0.11461115628480911, + "num_tokens": 1804039.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5985729694366455, + "sampling/importance_sampling_ratio/mean": 0.5072454810142517, + "sampling/importance_sampling_ratio/min": 0.07339605689048767, + "sampling/sampling_logp_difference/max": 0.3649592399597168, + "sampling/sampling_logp_difference/mean": 0.026338135823607445, + "step": 176, + "step_time": 40.05168053135276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 488.0625, + "completions/mean_terminated_length": 488.0625, + "completions/min_length": 419.0, + "completions/min_terminated_length": 419.0, + "entropy": 1.2391329184174538, + "epoch": 0.354, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21207627654075623, + "kl": 0.036609378294087946, + "learning_rate": 3e-05, + "loss": 0.11777876317501068, + "num_tokens": 1813440.0, + "reward": 7.0, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.3352530002593994, + "sampling/importance_sampling_ratio/mean": 0.6533275246620178, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35143423080444336, + "sampling/sampling_logp_difference/mean": 0.027743803337216377, + "step": 177, + "step_time": 33.59382761735469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 570.0, + "completions/max_terminated_length": 570.0, + "completions/mean_length": 495.125, + "completions/mean_terminated_length": 495.125, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "entropy": 1.3244052603840828, + "epoch": 0.356, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2076549232006073, + "kl": 0.03601150680333376, + "learning_rate": 3e-05, + "loss": -0.20823253691196442, + "num_tokens": 1823018.0, + "reward": 7.0, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.9768388271331787, + "sampling/importance_sampling_ratio/mean": 0.8220031261444092, + "sampling/importance_sampling_ratio/min": 0.08218635618686676, + "sampling/sampling_logp_difference/max": 0.3826625347137451, + "sampling/sampling_logp_difference/mean": 0.02992408722639084, + "step": 178, + "step_time": 19.288791641127318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 497.875, + "completions/mean_terminated_length": 497.875, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.2238815650343895, + "epoch": 0.358, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5341953039169312, + "kl": 0.0400471081957221, + "learning_rate": 3e-05, + "loss": 0.06042468547821045, + "num_tokens": 1832736.0, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.1878349781036377, + "sampling/importance_sampling_ratio/mean": 1.0708422660827637, + "sampling/importance_sampling_ratio/min": 0.037978146225214005, + "sampling/sampling_logp_difference/max": 0.5151598453521729, + "sampling/sampling_logp_difference/mean": 0.029129499569535255, + "step": 179, + "step_time": 17.113372664432973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 541.6875, + "completions/mean_terminated_length": 541.6875, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.4256544262170792, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21081708371639252, + "kl": 0.03922082995995879, + "learning_rate": 3e-05, + "loss": -0.10235662013292313, + "num_tokens": 1843203.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 1.1632962226867676, + "sampling/importance_sampling_ratio/mean": 0.4484874904155731, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31549549102783203, + "sampling/sampling_logp_difference/mean": 0.02854372188448906, + "step": 180, + "step_time": 38.8069160906598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 541.0, + "completions/mean_terminated_length": 541.0, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.2127383947372437, + "epoch": 0.362, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28472819924354553, + "kl": 0.039078159374184906, + "learning_rate": 3e-05, + "loss": -0.1838480830192566, + "num_tokens": 1853555.0, + "reward": 6.1875, + "reward_std": 0.6551081538200378, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.6551081538200378, + "sampling/importance_sampling_ratio/max": 1.6971478462219238, + "sampling/importance_sampling_ratio/mean": 0.5436820983886719, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4628629684448242, + "sampling/sampling_logp_difference/mean": 0.028904814273118973, + "step": 181, + "step_time": 22.927347922697663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 521.1875, + "completions/mean_terminated_length": 521.1875, + "completions/min_length": 481.0, + "completions/min_terminated_length": 481.0, + "entropy": 1.205168604850769, + "epoch": 0.364, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40580859780311584, + "kl": 0.03652556415181607, + "learning_rate": 3e-05, + "loss": 0.11645574867725372, + "num_tokens": 1863670.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.6977338790893555, + "sampling/importance_sampling_ratio/mean": 0.7627977132797241, + "sampling/importance_sampling_ratio/min": 0.0672435387969017, + "sampling/sampling_logp_difference/max": 0.42972230911254883, + "sampling/sampling_logp_difference/mean": 0.02727590501308441, + "step": 182, + "step_time": 17.11851307330653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 501.6875, + "completions/mean_terminated_length": 501.6875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2342532575130463, + "epoch": 0.366, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30436721444129944, + "kl": 0.04035243031103164, + "learning_rate": 3e-05, + "loss": 0.07254824787378311, + "num_tokens": 1873353.0, + "reward": 7.0, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 1.7811214923858643, + "sampling/importance_sampling_ratio/mean": 0.6635246276855469, + "sampling/importance_sampling_ratio/min": 0.05660989508032799, + "sampling/sampling_logp_difference/max": 0.3192565441131592, + "sampling/sampling_logp_difference/mean": 0.028735067695379257, + "step": 183, + "step_time": 15.842315018642694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 540.625, + "completions/mean_terminated_length": 540.625, + "completions/min_length": 488.0, + "completions/min_terminated_length": 488.0, + "entropy": 1.3501724749803543, + "epoch": 0.368, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15960462391376495, + "kl": 0.04095173126552254, + "learning_rate": 3e-05, + "loss": -0.020260833203792572, + "num_tokens": 1883907.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 1.9560747146606445, + "sampling/importance_sampling_ratio/mean": 0.6561183929443359, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.9810755252838135, + "sampling/sampling_logp_difference/mean": 0.029474109411239624, + "step": 184, + "step_time": 27.30614952277392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 532.6875, + "completions/mean_terminated_length": 532.6875, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "entropy": 1.3397118523716927, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19153976440429688, + "kl": 0.045232257107272744, + "learning_rate": 3e-05, + "loss": 0.07327698916196823, + "num_tokens": 1894262.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 2.3196122646331787, + "sampling/importance_sampling_ratio/mean": 0.7404133677482605, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.378201961517334, + "sampling/sampling_logp_difference/mean": 0.02841799519956112, + "step": 185, + "step_time": 17.66303364513442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 626.0, + "completions/max_terminated_length": 626.0, + "completions/mean_length": 547.875, + "completions/mean_terminated_length": 547.875, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.4480287805199623, + "epoch": 0.372, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1921214759349823, + "kl": 0.04523745132610202, + "learning_rate": 3e-05, + "loss": -0.1904258131980896, + "num_tokens": 1904748.0, + "reward": 6.1875, + "reward_std": 1.0468206405639648, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.0468206405639648, + "sampling/importance_sampling_ratio/max": 2.1734814643859863, + "sampling/importance_sampling_ratio/mean": 0.8759132027626038, + "sampling/importance_sampling_ratio/min": 0.1473371982574463, + "sampling/sampling_logp_difference/max": 0.43239259719848633, + "sampling/sampling_logp_difference/mean": 0.030228231102228165, + "step": 186, + "step_time": 15.321936973370612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 522.5, + "completions/mean_terminated_length": 522.5, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.1799098625779152, + "epoch": 0.374, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1446281522512436, + "kl": 0.04069687286391854, + "learning_rate": 3e-05, + "loss": -0.005613957531750202, + "num_tokens": 1915044.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1428344249725342, + "sampling/importance_sampling_ratio/mean": 0.49870407581329346, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3580789566040039, + "sampling/sampling_logp_difference/mean": 0.028958076611161232, + "step": 187, + "step_time": 20.772348938975483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 526.6875, + "completions/mean_terminated_length": 526.6875, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.1814791783690453, + "epoch": 0.376, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09093533456325531, + "kl": 0.048393386183306575, + "learning_rate": 3e-05, + "loss": -0.09858276695013046, + "num_tokens": 1925055.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.5353080034255981, + "sampling/importance_sampling_ratio/mean": 0.47519102692604065, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.455765962600708, + "sampling/sampling_logp_difference/mean": 0.02906900830566883, + "step": 188, + "step_time": 19.42380119021982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 663.0, + "completions/max_terminated_length": 663.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 538.5, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.2316770479083061, + "epoch": 0.378, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20003275573253632, + "kl": 0.04744360945187509, + "learning_rate": 3e-05, + "loss": -0.16722381114959717, + "num_tokens": 1935415.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.9816064834594727, + "sampling/importance_sampling_ratio/mean": 0.680183470249176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4760274887084961, + "sampling/sampling_logp_difference/mean": 0.028748787939548492, + "step": 189, + "step_time": 19.73181858472526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 555.125, + "completions/mean_terminated_length": 555.125, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.198552466928959, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13213400542736053, + "kl": 0.04299823543988168, + "learning_rate": 3e-05, + "loss": 0.04805057868361473, + "num_tokens": 1945985.0, + "reward": 6.625, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 1.563953161239624, + "sampling/importance_sampling_ratio/mean": 0.41058292984962463, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3399984836578369, + "sampling/sampling_logp_difference/mean": 0.0274125337600708, + "step": 190, + "step_time": 17.826407154556364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 684.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 539.25, + "completions/mean_terminated_length": 539.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.179109387099743, + "epoch": 0.382, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2910110354423523, + "kl": 0.046097030164673924, + "learning_rate": 3e-05, + "loss": -0.3787975311279297, + "num_tokens": 1956445.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.4134271144866943, + "sampling/importance_sampling_ratio/mean": 0.8701735734939575, + "sampling/importance_sampling_ratio/min": 0.1316290944814682, + "sampling/sampling_logp_difference/max": 0.3755226135253906, + "sampling/sampling_logp_difference/mean": 0.029267774894833565, + "step": 191, + "step_time": 24.135659996420145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 540.0, + "completions/mean_terminated_length": 540.0, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.2047618329524994, + "epoch": 0.384, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20911987125873566, + "kl": 0.04525213362649083, + "learning_rate": 3e-05, + "loss": 0.013828473165631294, + "num_tokens": 1966693.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 1.9282022714614868, + "sampling/importance_sampling_ratio/mean": 0.5034524202346802, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4313013553619385, + "sampling/sampling_logp_difference/mean": 0.02878478728234768, + "step": 192, + "step_time": 14.677200393751264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 597.0, + "completions/max_terminated_length": 597.0, + "completions/mean_length": 553.125, + "completions/mean_terminated_length": 553.125, + "completions/min_length": 501.0, + "completions/min_terminated_length": 501.0, + "entropy": 1.1857876032590866, + "epoch": 0.386, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3986797034740448, + "kl": 0.04699963750317693, + "learning_rate": 3e-05, + "loss": -0.06190801039338112, + "num_tokens": 1977311.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.877324104309082, + "sampling/importance_sampling_ratio/mean": 0.9780403971672058, + "sampling/importance_sampling_ratio/min": 0.02624017745256424, + "sampling/sampling_logp_difference/max": 0.7719376087188721, + "sampling/sampling_logp_difference/mean": 0.02950127050280571, + "step": 193, + "step_time": 22.189579842612147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 531.5625, + "completions/mean_terminated_length": 531.5625, + "completions/min_length": 465.0, + "completions/min_terminated_length": 465.0, + "entropy": 1.3334204703569412, + "epoch": 0.388, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22821271419525146, + "kl": 0.04149021068587899, + "learning_rate": 3e-05, + "loss": -0.15963155031204224, + "num_tokens": 1987680.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 2.790942907333374, + "sampling/importance_sampling_ratio/mean": 0.7382668256759644, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45777153968811035, + "sampling/sampling_logp_difference/mean": 0.029790451750159264, + "step": 194, + "step_time": 21.228061076253653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 505.0625, + "completions/mean_terminated_length": 505.0625, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.2570307329297066, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3617555797100067, + "kl": 0.042452862951904535, + "learning_rate": 3e-05, + "loss": -0.2362610548734665, + "num_tokens": 1997481.0, + "reward": 6.625, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.3454530239105225, + "sampling/importance_sampling_ratio/mean": 0.9099248647689819, + "sampling/importance_sampling_ratio/min": 0.09889467060565948, + "sampling/sampling_logp_difference/max": 0.6245463490486145, + "sampling/sampling_logp_difference/mean": 0.029685894027352333, + "step": 195, + "step_time": 17.880568680819124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 544.6875, + "completions/mean_terminated_length": 544.6875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.208221659064293, + "epoch": 0.392, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35804808139801025, + "kl": 0.03823408088646829, + "learning_rate": 3e-05, + "loss": 0.026877164840698242, + "num_tokens": 2007860.0, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 2.670585870742798, + "sampling/importance_sampling_ratio/mean": 0.6493271589279175, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4234185218811035, + "sampling/sampling_logp_difference/mean": 0.02899312786757946, + "step": 196, + "step_time": 20.456977635622025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 582.875, + "completions/mean_terminated_length": 582.875, + "completions/min_length": 517.0, + "completions/min_terminated_length": 517.0, + "entropy": 1.3732100576162338, + "epoch": 0.394, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.05589874088764191, + "kl": 0.03486072865780443, + "learning_rate": 3e-05, + "loss": -0.019679736346006393, + "num_tokens": 2018946.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1570472717285156, + "sampling/importance_sampling_ratio/mean": 0.35759687423706055, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33884429931640625, + "sampling/sampling_logp_difference/mean": 0.029303058981895447, + "step": 197, + "step_time": 16.522779263556004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 533.5, + "completions/mean_terminated_length": 533.5, + "completions/min_length": 479.0, + "completions/min_terminated_length": 479.0, + "entropy": 1.1664377599954605, + "epoch": 0.396, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23638494312763214, + "kl": 0.037777561345137656, + "learning_rate": 3e-05, + "loss": 0.0447416789829731, + "num_tokens": 2029178.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.484117865562439, + "sampling/importance_sampling_ratio/mean": 0.539449155330658, + "sampling/importance_sampling_ratio/min": 0.09831889718770981, + "sampling/sampling_logp_difference/max": 0.37561988830566406, + "sampling/sampling_logp_difference/mean": 0.029459550976753235, + "step": 198, + "step_time": 30.065524043980986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 536.0625, + "completions/mean_terminated_length": 536.0625, + "completions/min_length": 472.0, + "completions/min_terminated_length": 472.0, + "entropy": 1.254080481827259, + "epoch": 0.398, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1923029124736786, + "kl": 0.03572200902272016, + "learning_rate": 3e-05, + "loss": -0.09766081720590591, + "num_tokens": 2039347.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.6362762451171875, + "sampling/importance_sampling_ratio/mean": 0.5547572374343872, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35316991806030273, + "sampling/sampling_logp_difference/mean": 0.027488065883517265, + "step": 199, + "step_time": 30.96936017088592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 551.5, + "completions/mean_terminated_length": 551.5, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.2724409252405167, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22953365743160248, + "kl": 0.036497756373137236, + "learning_rate": 3e-05, + "loss": 0.014221633784472942, + "num_tokens": 2049795.0, + "reward": 6.5625, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.4539027214050293, + "sampling/importance_sampling_ratio/mean": 0.6678089499473572, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7396450042724609, + "sampling/sampling_logp_difference/mean": 0.0300765261054039, + "step": 200, + "step_time": 16.165886579081416 + } + ], + "logging_steps": 1, + "max_steps": 300, + "num_input_tokens_seen": 2049795, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/outputs/E0-baseline/checkpoint-200/training_args.bin b/outputs/E0-baseline/checkpoint-200/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c3a2693682d14c331dbd12e8872552fc28d54b49 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-200/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6 +size 7697 diff --git a/outputs/E0-baseline/checkpoint-250/README.md b/outputs/E0-baseline/checkpoint-250/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88990785a4695219513364eafae195070ddd3182 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-250/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507 +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.20.0 \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-250/adapter_config.json b/outputs/E0-baseline/checkpoint-250/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3b383bef044f277e3ff79f0fc5addabd3c441593 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-250/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.20.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-250/adapter_model.safetensors b/outputs/E0-baseline/checkpoint-250/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c8869eddab8e8ff96a01271296ac68f16acc93df --- /dev/null +++ b/outputs/E0-baseline/checkpoint-250/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ec658a42b607a64873f110f786c01afb192f0abf7fbc70a1f4ee67e79f279a +size 264308896 diff --git a/outputs/E0-baseline/checkpoint-250/chat_template.jinja b/outputs/E0-baseline/checkpoint-250/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-250/chat_template.jinja @@ -0,0 +1,61 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-250/tokenizer.json b/outputs/E0-baseline/checkpoint-250/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-250/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/outputs/E0-baseline/checkpoint-250/tokenizer_config.json b/outputs/E0-baseline/checkpoint-250/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..86d67326516569b4f1cec3ac7b51b8903062c648 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-250/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/outputs/E0-baseline/checkpoint-250/trainer_state.json b/outputs/E0-baseline/checkpoint-250/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..575278def069e7c9219db66ea286cfab6c438112 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-250/trainer_state.json @@ -0,0 +1,8284 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5, + "eval_steps": 500, + "global_step": 250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 529.0, + "completions/mean_terminated_length": 529.0, + "completions/min_length": 482.0, + "completions/min_terminated_length": 482.0, + "entropy": 1.2025159299373627, + "epoch": 0.002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22668755054473877, + "kl": 0.0, + "learning_rate": 0.0, + "loss": 0.2398601919412613, + "num_tokens": 10400.0, + "reward": 6.375, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6477458477020264, + "sampling/importance_sampling_ratio/mean": 0.5064857602119446, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40894174575805664, + "sampling/sampling_logp_difference/mean": 0.026567919179797173, + "step": 1, + "step_time": 12.760562099982053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 644.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 562.25, + "completions/mean_terminated_length": 562.25, + "completions/min_length": 497.0, + "completions/min_terminated_length": 497.0, + "entropy": 1.1930748969316483, + "epoch": 0.004, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12967805564403534, + "kl": 0.0, + "learning_rate": 3e-06, + "loss": -0.08571265637874603, + "num_tokens": 20924.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 1.0037232637405396, + "sampling/importance_sampling_ratio/mean": 0.41275694966316223, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45699238777160645, + "sampling/sampling_logp_difference/mean": 0.025086138397455215, + "step": 2, + "step_time": 11.406366533134133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 483.625, + "completions/mean_terminated_length": 483.625, + "completions/min_length": 407.0, + "completions/min_terminated_length": 407.0, + "entropy": 1.1096689626574516, + "epoch": 0.006, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22966289520263672, + "kl": 0.0008355328354809899, + "learning_rate": 6e-06, + "loss": 0.020913563668727875, + "num_tokens": 30222.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.927120566368103, + "sampling/importance_sampling_ratio/mean": 0.6114993095397949, + "sampling/importance_sampling_ratio/min": 0.11067161709070206, + "sampling/sampling_logp_difference/max": 0.4620504379272461, + "sampling/sampling_logp_difference/mean": 0.024864500388503075, + "step": 3, + "step_time": 26.480680393986404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 608.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 524.75, + "completions/mean_terminated_length": 524.75, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.211122527718544, + "epoch": 0.008, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30378466844558716, + "kl": 0.0008403196770814247, + "learning_rate": 9e-06, + "loss": -0.12959149479866028, + "num_tokens": 40410.0, + "reward": 6.25, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.406888961791992, + "sampling/importance_sampling_ratio/mean": 0.5457419753074646, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3680229187011719, + "sampling/sampling_logp_difference/mean": 0.02656388282775879, + "step": 4, + "step_time": 22.95301443943754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 487.5625, + "completions/mean_terminated_length": 487.5625, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "entropy": 1.247180238366127, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5199307799339294, + "kl": 0.0008723233368073124, + "learning_rate": 1.2e-05, + "loss": 0.11524428427219391, + "num_tokens": 49915.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.1544158458709717, + "sampling/importance_sampling_ratio/mean": 0.963020920753479, + "sampling/importance_sampling_ratio/min": 0.04948288947343826, + "sampling/sampling_logp_difference/max": 0.4774587154388428, + "sampling/sampling_logp_difference/mean": 0.02625642716884613, + "step": 5, + "step_time": 22.37928077671677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 533.0, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 394.0, + "completions/min_terminated_length": 394.0, + "entropy": 1.1693861335515976, + "epoch": 0.012, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27643197774887085, + "kl": 0.0009261858467652928, + "learning_rate": 1.5e-05, + "loss": 0.15093231201171875, + "num_tokens": 60219.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 2.125091791152954, + "sampling/importance_sampling_ratio/mean": 0.7733402252197266, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7049552202224731, + "sampling/sampling_logp_difference/mean": 0.025986071676015854, + "step": 6, + "step_time": 21.00841654697433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 505.6875, + "completions/mean_terminated_length": 505.6875, + "completions/min_length": 425.0, + "completions/min_terminated_length": 425.0, + "entropy": 1.2473183795809746, + "epoch": 0.014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2166663259267807, + "kl": 0.0009701631606731098, + "learning_rate": 1.8e-05, + "loss": -0.08582288026809692, + "num_tokens": 69902.0, + "reward": 7.1875, + "reward_std": 0.75, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.8287386894226074, + "sampling/importance_sampling_ratio/mean": 0.5286832451820374, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.38030898571014404, + "sampling/sampling_logp_difference/mean": 0.0264718160033226, + "step": 7, + "step_time": 46.596127359196544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.1040107272565365, + "epoch": 0.016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18874908983707428, + "kl": 0.0010721117541834246, + "learning_rate": 2.1e-05, + "loss": -0.1946130096912384, + "num_tokens": 79501.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.3561792373657227, + "sampling/importance_sampling_ratio/mean": 0.6918502449989319, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.26114892959594727, + "sampling/sampling_logp_difference/mean": 0.02554757334291935, + "step": 8, + "step_time": 25.433595282491297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 554.0, + "completions/max_terminated_length": 554.0, + "completions/mean_length": 490.1875, + "completions/mean_terminated_length": 490.1875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1377170644700527, + "epoch": 0.018, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.204215869307518, + "kl": 0.002002388624532614, + "learning_rate": 2.4e-05, + "loss": -0.08130021393299103, + "num_tokens": 88976.0, + "reward": 6.5, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.0982444286346436, + "sampling/importance_sampling_ratio/mean": 0.5873216986656189, + "sampling/importance_sampling_ratio/min": 0.04890051856637001, + "sampling/sampling_logp_difference/max": 0.8512144088745117, + "sampling/sampling_logp_difference/mean": 0.02638406865298748, + "step": 9, + "step_time": 18.427699581719935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 500.1875, + "completions/mean_terminated_length": 500.1875, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.1230391450226307, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1985069215297699, + "kl": 0.0026735300780273974, + "learning_rate": 2.7000000000000002e-05, + "loss": -0.12387816607952118, + "num_tokens": 98603.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.983199954032898, + "sampling/importance_sampling_ratio/mean": 0.5639468431472778, + "sampling/importance_sampling_ratio/min": 0.012905921787023544, + "sampling/sampling_logp_difference/max": 0.3653905391693115, + "sampling/sampling_logp_difference/mean": 0.025383003056049347, + "step": 10, + "step_time": 14.99981931829825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 459.6875, + "completions/mean_terminated_length": 459.6875, + "completions/min_length": 379.0, + "completions/min_terminated_length": 379.0, + "entropy": 1.213625729084015, + "epoch": 0.022, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3695620596408844, + "kl": 0.00424640768324025, + "learning_rate": 3e-05, + "loss": -0.06913074851036072, + "num_tokens": 107734.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.000136375427246, + "sampling/importance_sampling_ratio/mean": 0.8144867420196533, + "sampling/importance_sampling_ratio/min": 0.06302778422832489, + "sampling/sampling_logp_difference/max": 0.3647327423095703, + "sampling/sampling_logp_difference/mean": 0.026674197986721992, + "step": 11, + "step_time": 30.46549107739702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.191277615725994, + "epoch": 0.024, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35457733273506165, + "kl": 0.007200263120466843, + "learning_rate": 3e-05, + "loss": 0.22097699344158173, + "num_tokens": 117199.0, + "reward": 5.9375, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 1.5660414695739746, + "sampling/importance_sampling_ratio/mean": 0.649204432964325, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3921785354614258, + "sampling/sampling_logp_difference/mean": 0.02726689912378788, + "step": 12, + "step_time": 15.719243939965963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 464.3125, + "completions/mean_terminated_length": 464.3125, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "entropy": 1.24251464381814, + "epoch": 0.026, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30047014355659485, + "kl": 0.0058551667898427695, + "learning_rate": 3e-05, + "loss": -0.07746122777462006, + "num_tokens": 126556.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6028401851654053, + "sampling/importance_sampling_ratio/mean": 0.7561360597610474, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4946432113647461, + "sampling/sampling_logp_difference/mean": 0.02639186754822731, + "step": 13, + "step_time": 18.08984712138772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 444.8125, + "completions/mean_terminated_length": 444.8125, + "completions/min_length": 389.0, + "completions/min_terminated_length": 389.0, + "entropy": 1.1501125395298004, + "epoch": 0.028, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24105942249298096, + "kl": 0.012796793889719993, + "learning_rate": 3e-05, + "loss": 0.10855278372764587, + "num_tokens": 135417.0, + "reward": 3.1875, + "reward_std": 3.310966730117798, + "rewards/quality_reward/mean": 3.1875, + "rewards/quality_reward/std": 3.310966730117798, + "sampling/importance_sampling_ratio/max": 2.541518211364746, + "sampling/importance_sampling_ratio/mean": 0.5903321504592896, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.8390979766845703, + "sampling/sampling_logp_difference/mean": 0.02838001400232315, + "step": 14, + "step_time": 20.055794408079237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 497.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 442.25, + "completions/mean_terminated_length": 442.25, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 1.1262825280427933, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19040776789188385, + "kl": 0.010701361010433175, + "learning_rate": 3e-05, + "loss": -0.007966680452227592, + "num_tokens": 144205.0, + "reward": 5.875, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.0754293203353882, + "sampling/importance_sampling_ratio/mean": 0.41446638107299805, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3982200622558594, + "sampling/sampling_logp_difference/mean": 0.027210108935832977, + "step": 15, + "step_time": 14.176074750721455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 478.125, + "completions/mean_terminated_length": 478.125, + "completions/min_length": 433.0, + "completions/min_terminated_length": 433.0, + "entropy": 1.2985924184322357, + "epoch": 0.032, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23625652492046356, + "kl": 0.0213887135614641, + "learning_rate": 3e-05, + "loss": -0.21546132862567902, + "num_tokens": 153543.0, + "reward": 6.5, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 2.0074336528778076, + "sampling/importance_sampling_ratio/mean": 0.49076417088508606, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5095968246459961, + "sampling/sampling_logp_difference/mean": 0.028833162039518356, + "step": 16, + "step_time": 15.848205763846636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 469.25, + "completions/mean_terminated_length": 469.25, + "completions/min_length": 423.0, + "completions/min_terminated_length": 423.0, + "entropy": 1.3148910626769066, + "epoch": 0.034, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17765119671821594, + "kl": 0.02128879475640133, + "learning_rate": 3e-05, + "loss": -0.0828079879283905, + "num_tokens": 163043.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 1.5988941192626953, + "sampling/importance_sampling_ratio/mean": 0.5021570324897766, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.591062068939209, + "sampling/sampling_logp_difference/mean": 0.030804751440882683, + "step": 17, + "step_time": 28.313011147081852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 447.0, + "completions/mean_terminated_length": 447.0, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 1.390664003789425, + "epoch": 0.036, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37244123220443726, + "kl": 0.019650122558232397, + "learning_rate": 3e-05, + "loss": -0.01184748113155365, + "num_tokens": 172379.0, + "reward": 6.0, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.675238847732544, + "sampling/importance_sampling_ratio/mean": 0.9581233263015747, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5762512683868408, + "sampling/sampling_logp_difference/mean": 0.03126702085137367, + "step": 18, + "step_time": 30.991567107848823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 498.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 447.75, + "completions/mean_terminated_length": 447.75, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 1.3287223279476166, + "epoch": 0.038, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23734751343727112, + "kl": 0.022738213243428618, + "learning_rate": 3e-05, + "loss": 0.040024783462285995, + "num_tokens": 181239.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.095722198486328, + "sampling/importance_sampling_ratio/mean": 0.6408629417419434, + "sampling/importance_sampling_ratio/min": 0.1203346848487854, + "sampling/sampling_logp_difference/max": 0.4722297191619873, + "sampling/sampling_logp_difference/mean": 0.030378391966223717, + "step": 19, + "step_time": 18.615950418636203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 452.3125, + "completions/mean_terminated_length": 452.3125, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 1.1001618690788746, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34637194871902466, + "kl": 0.015380491153337061, + "learning_rate": 3e-05, + "loss": 0.1691148728132248, + "num_tokens": 190068.0, + "reward": 6.125, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.6566128730773926, + "sampling/importance_sampling_ratio/mean": 0.7767290472984314, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.6531033515930176, + "sampling/sampling_logp_difference/mean": 0.030463142320513725, + "step": 20, + "step_time": 15.741292077116668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 457.5625, + "completions/mean_terminated_length": 457.5625, + "completions/min_length": 390.0, + "completions/min_terminated_length": 390.0, + "entropy": 1.3708399310708046, + "epoch": 0.042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2514922618865967, + "kl": 0.018277494702488184, + "learning_rate": 3e-05, + "loss": -0.13425321877002716, + "num_tokens": 198941.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.5612297058105469, + "sampling/importance_sampling_ratio/mean": 0.601022481918335, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4818992614746094, + "sampling/sampling_logp_difference/mean": 0.0315740592777729, + "step": 21, + "step_time": 17.24192419089377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 421.4375, + "completions/mean_terminated_length": 421.4375, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 1.136269599199295, + "epoch": 0.044, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2268020063638687, + "kl": 0.017973962530959398, + "learning_rate": 3e-05, + "loss": 0.010622119531035423, + "num_tokens": 207300.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.7986358404159546, + "sampling/importance_sampling_ratio/mean": 0.46136727929115295, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5335149765014648, + "sampling/sampling_logp_difference/mean": 0.02804401144385338, + "step": 22, + "step_time": 19.37282825494185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 452.6875, + "completions/mean_terminated_length": 452.6875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3299130946397781, + "epoch": 0.046, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33994877338409424, + "kl": 0.021804221265483648, + "learning_rate": 3e-05, + "loss": -0.24404363334178925, + "num_tokens": 216343.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.452249526977539, + "sampling/importance_sampling_ratio/mean": 0.747193455696106, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6328549385070801, + "sampling/sampling_logp_difference/mean": 0.02918298915028572, + "step": 23, + "step_time": 15.356728344224393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 462.4375, + "completions/mean_terminated_length": 462.4375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.379701942205429, + "epoch": 0.048, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3076202869415283, + "kl": 0.020299295720178634, + "learning_rate": 3e-05, + "loss": -0.09123071283102036, + "num_tokens": 225550.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.5845099687576294, + "sampling/importance_sampling_ratio/mean": 0.6625345945358276, + "sampling/importance_sampling_ratio/min": 0.04495502635836601, + "sampling/sampling_logp_difference/max": 0.36547183990478516, + "sampling/sampling_logp_difference/mean": 0.028946854174137115, + "step": 24, + "step_time": 14.95462113339454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.2471638694405556, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22180576622486115, + "kl": 0.018309170845896006, + "learning_rate": 3e-05, + "loss": -0.1412944793701172, + "num_tokens": 235005.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.6407876014709473, + "sampling/importance_sampling_ratio/mean": 0.7712795734405518, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4363563060760498, + "sampling/sampling_logp_difference/mean": 0.02898716926574707, + "step": 25, + "step_time": 16.46686205593869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 455.75, + "completions/mean_terminated_length": 455.75, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 1.28530602902174, + "epoch": 0.052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37535253167152405, + "kl": 0.020504546992015094, + "learning_rate": 3e-05, + "loss": 0.10839397460222244, + "num_tokens": 243953.0, + "reward": 6.25, + "reward_std": 1.0645813941955566, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.0645813941955566, + "sampling/importance_sampling_ratio/max": 2.0567266941070557, + "sampling/importance_sampling_ratio/mean": 0.5800145864486694, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4523625373840332, + "sampling/sampling_logp_difference/mean": 0.027610119432210922, + "step": 26, + "step_time": 22.121026155073196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.0, + "completions/max_terminated_length": 514.0, + "completions/mean_length": 462.75, + "completions/mean_terminated_length": 462.75, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.2253629937767982, + "epoch": 0.054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2340388149023056, + "kl": 0.020236384589225054, + "learning_rate": 3e-05, + "loss": 0.04823978245258331, + "num_tokens": 253237.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.1560932397842407, + "sampling/importance_sampling_ratio/mean": 0.40753045678138733, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35022830963134766, + "sampling/sampling_logp_difference/mean": 0.028367744758725166, + "step": 27, + "step_time": 15.10967448912561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 538.0, + "completions/max_terminated_length": 538.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "entropy": 1.3779077678918839, + "epoch": 0.056, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5531017184257507, + "kl": 0.01706669357372448, + "learning_rate": 3e-05, + "loss": 0.0933581069111824, + "num_tokens": 262454.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.5711586475372314, + "sampling/importance_sampling_ratio/mean": 0.7730721831321716, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.343827486038208, + "sampling/sampling_logp_difference/mean": 0.028883326798677444, + "step": 28, + "step_time": 38.59647103771567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 477.0, + "completions/mean_terminated_length": 477.0, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.3791070505976677, + "epoch": 0.058, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33771538734436035, + "kl": 0.02141271048458293, + "learning_rate": 3e-05, + "loss": 0.010216176509857178, + "num_tokens": 271918.0, + "reward": 5.75, + "reward_std": 1.2909945249557495, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.2909945249557495, + "sampling/importance_sampling_ratio/max": 2.4999797344207764, + "sampling/importance_sampling_ratio/mean": 1.0250636339187622, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3258817195892334, + "sampling/sampling_logp_difference/mean": 0.029029540717601776, + "step": 29, + "step_time": 23.610272648278624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 471.1875, + "completions/mean_terminated_length": 471.1875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.4652926102280617, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21955685317516327, + "kl": 0.019562674744520336, + "learning_rate": 3e-05, + "loss": -0.014814459718763828, + "num_tokens": 281305.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 2.802098274230957, + "sampling/importance_sampling_ratio/mean": 0.866391658782959, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6354451179504395, + "sampling/sampling_logp_difference/mean": 0.03177585452795029, + "step": 30, + "step_time": 16.961607525125146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 490.5, + "completions/mean_terminated_length": 490.5, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 1.1957123205065727, + "epoch": 0.062, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12176825106143951, + "kl": 0.026934220048133284, + "learning_rate": 3e-05, + "loss": -0.08307373523712158, + "num_tokens": 291409.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.610858678817749, + "sampling/importance_sampling_ratio/mean": 0.6937527656555176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4253416061401367, + "sampling/sampling_logp_difference/mean": 0.02871524728834629, + "step": 31, + "step_time": 15.012207658961415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 462.0, + "completions/mean_terminated_length": 462.0, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3018206283450127, + "epoch": 0.064, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4979296624660492, + "kl": 0.03539742121938616, + "learning_rate": 3e-05, + "loss": 0.0017175457905977964, + "num_tokens": 300649.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8527640104293823, + "sampling/importance_sampling_ratio/mean": 0.7824680209159851, + "sampling/importance_sampling_ratio/min": 0.07447884231805801, + "sampling/sampling_logp_difference/max": 0.5221483707427979, + "sampling/sampling_logp_difference/mean": 0.029107660055160522, + "step": 32, + "step_time": 22.78309725271538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 474.0, + "completions/mean_terminated_length": 474.0, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.321175642311573, + "epoch": 0.066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22016967833042145, + "kl": 0.029592803912237287, + "learning_rate": 3e-05, + "loss": 0.05625719577074051, + "num_tokens": 309889.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.9875097274780273, + "sampling/importance_sampling_ratio/mean": 0.7832435369491577, + "sampling/importance_sampling_ratio/min": 0.18006731569766998, + "sampling/sampling_logp_difference/max": 0.38585615158081055, + "sampling/sampling_logp_difference/mean": 0.027828343212604523, + "step": 33, + "step_time": 44.51360382232815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 459.9375, + "completions/mean_terminated_length": 459.9375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.2222414836287498, + "epoch": 0.068, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2873040437698364, + "kl": 0.02840927499346435, + "learning_rate": 3e-05, + "loss": -0.037432070821523666, + "num_tokens": 318904.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 2.6647069454193115, + "sampling/importance_sampling_ratio/mean": 0.5744763016700745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.41987133026123047, + "sampling/sampling_logp_difference/mean": 0.029470665380358696, + "step": 34, + "step_time": 133.03877451224253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 522.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 462.1875, + "completions/mean_terminated_length": 462.1875, + "completions/min_length": 413.0, + "completions/min_terminated_length": 413.0, + "entropy": 1.1665974482893944, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2490653246641159, + "kl": 0.025841041060630232, + "learning_rate": 3e-05, + "loss": -0.20422951877117157, + "num_tokens": 328011.0, + "reward": 6.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 2.510730266571045, + "sampling/importance_sampling_ratio/mean": 0.7954420447349548, + "sampling/importance_sampling_ratio/min": 0.020566223189234734, + "sampling/sampling_logp_difference/max": 0.36430132389068604, + "sampling/sampling_logp_difference/mean": 0.026844775304198265, + "step": 35, + "step_time": 21.5843787365593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 549.0, + "completions/max_terminated_length": 549.0, + "completions/mean_length": 492.0, + "completions/mean_terminated_length": 492.0, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3467887043952942, + "epoch": 0.072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15268851816654205, + "kl": 0.023660800594370812, + "learning_rate": 3e-05, + "loss": -0.11188814043998718, + "num_tokens": 337731.0, + "reward": 6.5625, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.6270908117294312, + "sampling/importance_sampling_ratio/mean": 0.614537239074707, + "sampling/importance_sampling_ratio/min": 0.10853960365056992, + "sampling/sampling_logp_difference/max": 0.4023854732513428, + "sampling/sampling_logp_difference/mean": 0.028961554169654846, + "step": 36, + "step_time": 18.843947287183255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.0, + "completions/max_terminated_length": 513.0, + "completions/mean_length": 460.75, + "completions/mean_terminated_length": 460.75, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "entropy": 1.2476315572857857, + "epoch": 0.074, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42548227310180664, + "kl": 0.022181478852871805, + "learning_rate": 3e-05, + "loss": 0.37223517894744873, + "num_tokens": 346815.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.6624510288238525, + "sampling/importance_sampling_ratio/mean": 0.7942180633544922, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4037543535232544, + "sampling/sampling_logp_difference/mean": 0.028780322521924973, + "step": 37, + "step_time": 42.04662919091061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 452.0, + "completions/mean_terminated_length": 452.0, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "entropy": 1.1745503433048725, + "epoch": 0.076, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16980381309986115, + "kl": 0.017483733594417572, + "learning_rate": 3e-05, + "loss": -0.09029137343168259, + "num_tokens": 355711.0, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "sampling/importance_sampling_ratio/max": 2.3201518058776855, + "sampling/importance_sampling_ratio/mean": 0.721072793006897, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.30433225631713867, + "sampling/sampling_logp_difference/mean": 0.026646045967936516, + "step": 38, + "step_time": 38.63075139513239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 466.8125, + "completions/mean_terminated_length": 466.8125, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2664988860487938, + "epoch": 0.078, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21305795013904572, + "kl": 0.021121050289366394, + "learning_rate": 3e-05, + "loss": -0.03154226019978523, + "num_tokens": 364860.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.0297553539276123, + "sampling/importance_sampling_ratio/mean": 0.674609363079071, + "sampling/importance_sampling_ratio/min": 0.11245065927505493, + "sampling/sampling_logp_difference/max": 0.37443917989730835, + "sampling/sampling_logp_difference/mean": 0.028257746249437332, + "step": 39, + "step_time": 30.028073193039745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 485.1875, + "completions/mean_terminated_length": 485.1875, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.3458357080817223, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12235487997531891, + "kl": 0.018535695446189493, + "learning_rate": 3e-05, + "loss": -0.035816628485918045, + "num_tokens": 374607.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.242042303085327, + "sampling/importance_sampling_ratio/mean": 0.5344723463058472, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36985254287719727, + "sampling/sampling_logp_difference/mean": 0.029600802809000015, + "step": 40, + "step_time": 15.446288945619017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 498.875, + "completions/mean_terminated_length": 498.875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3402792811393738, + "epoch": 0.082, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15642686188220978, + "kl": 0.013967045990284532, + "learning_rate": 3e-05, + "loss": 0.0011727213859558105, + "num_tokens": 384317.0, + "reward": 6.375, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5691092014312744, + "sampling/importance_sampling_ratio/mean": 0.3696203827857971, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6487679481506348, + "sampling/sampling_logp_difference/mean": 0.03018251620233059, + "step": 41, + "step_time": 18.15720977121964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.3317564576864243, + "epoch": 0.084, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3236794173717499, + "kl": 0.01707366103073582, + "learning_rate": 3e-05, + "loss": 0.10363321751356125, + "num_tokens": 393934.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.0568668842315674, + "sampling/importance_sampling_ratio/mean": 0.6415404677391052, + "sampling/importance_sampling_ratio/min": 0.07682990282773972, + "sampling/sampling_logp_difference/max": 0.7769032716751099, + "sampling/sampling_logp_difference/mean": 0.02936164103448391, + "step": 42, + "step_time": 15.066733688581735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 510.9375, + "completions/mean_terminated_length": 510.9375, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2099780030548573, + "epoch": 0.086, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1771102100610733, + "kl": 0.01784718461567536, + "learning_rate": 3e-05, + "loss": -0.027566466480493546, + "num_tokens": 403893.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.3430161476135254, + "sampling/importance_sampling_ratio/mean": 0.7769261598587036, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36153650283813477, + "sampling/sampling_logp_difference/mean": 0.028799494728446007, + "step": 43, + "step_time": 16.90863296529278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 494.1875, + "completions/mean_terminated_length": 494.1875, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "entropy": 1.2924985438585281, + "epoch": 0.088, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3432070314884186, + "kl": 0.014529847539961338, + "learning_rate": 3e-05, + "loss": -0.04157561436295509, + "num_tokens": 413312.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.726816177368164, + "sampling/importance_sampling_ratio/mean": 0.8989821672439575, + "sampling/importance_sampling_ratio/min": 0.07410597801208496, + "sampling/sampling_logp_difference/max": 0.31444358825683594, + "sampling/sampling_logp_difference/mean": 0.028122060000896454, + "step": 44, + "step_time": 16.7880685236305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 543.5, + "completions/mean_terminated_length": 543.5, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.206408604979515, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36233776807785034, + "kl": 0.015796773659531027, + "learning_rate": 3e-05, + "loss": 0.029176680371165276, + "num_tokens": 423624.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.677195429801941, + "sampling/importance_sampling_ratio/mean": 0.6537867188453674, + "sampling/importance_sampling_ratio/min": 0.09822077304124832, + "sampling/sampling_logp_difference/max": 0.31381869316101074, + "sampling/sampling_logp_difference/mean": 0.02717999368906021, + "step": 45, + "step_time": 23.35960763087496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 643.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 534.5625, + "completions/mean_terminated_length": 534.5625, + "completions/min_length": 453.0, + "completions/min_terminated_length": 453.0, + "entropy": 1.2577891275286674, + "epoch": 0.092, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20219561457633972, + "kl": 0.014481160265859216, + "learning_rate": 3e-05, + "loss": 0.18850615620613098, + "num_tokens": 433817.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.203894853591919, + "sampling/importance_sampling_ratio/mean": 0.697495698928833, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35237741470336914, + "sampling/sampling_logp_difference/mean": 0.02723248302936554, + "step": 46, + "step_time": 19.56032704282552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 538.25, + "completions/mean_terminated_length": 538.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.3271892964839935, + "epoch": 0.094, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22262753546237946, + "kl": 0.012554377142805606, + "learning_rate": 3e-05, + "loss": 0.06984467804431915, + "num_tokens": 444045.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 2.2261502742767334, + "sampling/importance_sampling_ratio/mean": 0.6712950468063354, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4496123790740967, + "sampling/sampling_logp_difference/mean": 0.028838323429226875, + "step": 47, + "step_time": 21.226045075803995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 514.25, + "completions/mean_terminated_length": 514.25, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "entropy": 1.1054812744259834, + "epoch": 0.096, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3244606554508209, + "kl": 0.0157591889728792, + "learning_rate": 3e-05, + "loss": 0.09024985134601593, + "num_tokens": 453945.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.6745388507843018, + "sampling/importance_sampling_ratio/mean": 0.7607913017272949, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650559186935425, + "sampling/sampling_logp_difference/mean": 0.026116060093045235, + "step": 48, + "step_time": 17.565261672716588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 585.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 502.4375, + "completions/mean_terminated_length": 502.4375, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.2281213253736496, + "epoch": 0.098, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4033137857913971, + "kl": 0.015613102470524609, + "learning_rate": 3e-05, + "loss": -0.2898017466068268, + "num_tokens": 463576.0, + "reward": 6.4375, + "reward_std": 1.2632629871368408, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.2632629871368408, + "sampling/importance_sampling_ratio/max": 2.6974196434020996, + "sampling/importance_sampling_ratio/mean": 0.7124314308166504, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3675417900085449, + "sampling/sampling_logp_difference/mean": 0.028341906145215034, + "step": 49, + "step_time": 29.838082558009773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 530.1875, + "completions/mean_terminated_length": 530.1875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.1955150067806244, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23144562542438507, + "kl": 0.01374751579714939, + "learning_rate": 3e-05, + "loss": -0.19065965712070465, + "num_tokens": 473915.0, + "reward": 6.3125, + "reward_std": 1.1954776048660278, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.1954776048660278, + "sampling/importance_sampling_ratio/max": 2.54063081741333, + "sampling/importance_sampling_ratio/mean": 0.8162041902542114, + "sampling/importance_sampling_ratio/min": 0.1628064066171646, + "sampling/sampling_logp_difference/max": 0.2777421474456787, + "sampling/sampling_logp_difference/mean": 0.02853373810648918, + "step": 50, + "step_time": 16.047010839451104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 549.875, + "completions/mean_terminated_length": 549.875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1870752647519112, + "epoch": 0.102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4241364896297455, + "kl": 0.013923745544161648, + "learning_rate": 3e-05, + "loss": -0.1761355698108673, + "num_tokens": 484577.0, + "reward": 6.5625, + "reward_std": 1.0935417413711548, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 1.0935417413711548, + "sampling/importance_sampling_ratio/max": 2.929507255554199, + "sampling/importance_sampling_ratio/mean": 0.6905896663665771, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5649824142456055, + "sampling/sampling_logp_difference/mean": 0.028025619685649872, + "step": 51, + "step_time": 44.488836522214115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 552.375, + "completions/mean_terminated_length": 552.375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.098166175186634, + "epoch": 0.104, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.32851356267929077, + "kl": 0.01297539210645482, + "learning_rate": 3e-05, + "loss": -0.06503897905349731, + "num_tokens": 495015.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.9652340412139893, + "sampling/importance_sampling_ratio/mean": 0.9612224102020264, + "sampling/importance_sampling_ratio/min": 0.040177375078201294, + "sampling/sampling_logp_difference/max": 0.3454720973968506, + "sampling/sampling_logp_difference/mean": 0.02687419019639492, + "step": 52, + "step_time": 20.21497478755191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 578.375, + "completions/mean_terminated_length": 578.375, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2311968132853508, + "epoch": 0.106, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21921825408935547, + "kl": 0.017025968758389354, + "learning_rate": 3e-05, + "loss": -0.18975484371185303, + "num_tokens": 505909.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.0094237327575684, + "sampling/importance_sampling_ratio/mean": 0.5587533116340637, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.39138758182525635, + "sampling/sampling_logp_difference/mean": 0.028095029294490814, + "step": 53, + "step_time": 31.140278964303434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 473.25, + "completions/mean_terminated_length": 473.25, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 1.2682743221521378, + "epoch": 0.108, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27410373091697693, + "kl": 0.018500705540645868, + "learning_rate": 3e-05, + "loss": 0.14847250282764435, + "num_tokens": 515073.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.875750780105591, + "sampling/importance_sampling_ratio/mean": 0.7429271936416626, + "sampling/importance_sampling_ratio/min": 0.09779425710439682, + "sampling/sampling_logp_difference/max": 0.5433444976806641, + "sampling/sampling_logp_difference/mean": 0.02810005284845829, + "step": 54, + "step_time": 18.365382733754814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 549.0, + "completions/mean_terminated_length": 549.0, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.3073157891631126, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29323700070381165, + "kl": 0.016703857632819563, + "learning_rate": 3e-05, + "loss": 0.05967015400528908, + "num_tokens": 525377.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 1.4832638502120972, + "sampling/importance_sampling_ratio/mean": 0.6094669103622437, + "sampling/importance_sampling_ratio/min": 0.08072065562009811, + "sampling/sampling_logp_difference/max": 0.39328718185424805, + "sampling/sampling_logp_difference/mean": 0.02906947024166584, + "step": 55, + "step_time": 30.47015379369259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 576.625, + "completions/mean_terminated_length": 576.625, + "completions/min_length": 500.0, + "completions/min_terminated_length": 500.0, + "entropy": 1.2820357605814934, + "epoch": 0.112, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.200644388794899, + "kl": 0.018819268501829356, + "learning_rate": 3e-05, + "loss": -0.04828515648841858, + "num_tokens": 536163.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.3941831588745117, + "sampling/importance_sampling_ratio/mean": 0.5633801221847534, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3664684295654297, + "sampling/sampling_logp_difference/mean": 0.02962428703904152, + "step": 56, + "step_time": 16.172011949121952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 640.25, + "completions/mean_terminated_length": 640.25, + "completions/min_length": 556.0, + "completions/min_terminated_length": 556.0, + "entropy": 1.4191219061613083, + "epoch": 0.114, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19633841514587402, + "kl": 0.02060725452611223, + "learning_rate": 3e-05, + "loss": -0.12029920518398285, + "num_tokens": 548143.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.3877830505371094, + "sampling/importance_sampling_ratio/mean": 0.6956661343574524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33841991424560547, + "sampling/sampling_logp_difference/mean": 0.028747636824846268, + "step": 57, + "step_time": 26.892430102452636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 646.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 560.4375, + "completions/mean_terminated_length": 560.4375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.3337246850132942, + "epoch": 0.116, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24178451299667358, + "kl": 0.023009511292912066, + "learning_rate": 3e-05, + "loss": -0.02738652005791664, + "num_tokens": 558710.0, + "reward": 6.0625, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.4787031412124634, + "sampling/importance_sampling_ratio/mean": 0.4813012480735779, + "sampling/importance_sampling_ratio/min": 0.05691095441579819, + "sampling/sampling_logp_difference/max": 0.3029825687408447, + "sampling/sampling_logp_difference/mean": 0.029777564108371735, + "step": 58, + "step_time": 17.878377372398973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 697.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 595.25, + "completions/mean_terminated_length": 595.25, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.3107040077447891, + "epoch": 0.118, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1262614130973816, + "kl": 0.022026430582627654, + "learning_rate": 3e-05, + "loss": 0.04775794595479965, + "num_tokens": 569826.0, + "reward": 6.4375, + "reward_std": 1.0307763814926147, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.0307763814926147, + "sampling/importance_sampling_ratio/max": 2.1841602325439453, + "sampling/importance_sampling_ratio/mean": 0.5139275193214417, + "sampling/importance_sampling_ratio/min": 0.026369251310825348, + "sampling/sampling_logp_difference/max": 0.42272377014160156, + "sampling/sampling_logp_difference/mean": 0.028494788333773613, + "step": 59, + "step_time": 24.42572767334059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 549.1875, + "completions/mean_terminated_length": 549.1875, + "completions/min_length": 489.0, + "completions/min_terminated_length": 489.0, + "entropy": 1.1738965958356857, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.588756799697876, + "kl": 0.025482238037511706, + "learning_rate": 3e-05, + "loss": 0.2925710678100586, + "num_tokens": 580229.0, + "reward": 6.0625, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 2.8465583324432373, + "sampling/importance_sampling_ratio/mean": 1.1227651834487915, + "sampling/importance_sampling_ratio/min": 0.1096419170498848, + "sampling/sampling_logp_difference/max": 0.4628920555114746, + "sampling/sampling_logp_difference/mean": 0.02885228767991066, + "step": 60, + "step_time": 21.57787229306996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 692.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 577.3125, + "completions/mean_terminated_length": 577.3125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.1913195550441742, + "epoch": 0.122, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2779391407966614, + "kl": 0.02629381464794278, + "learning_rate": 3e-05, + "loss": 0.12304374575614929, + "num_tokens": 591178.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.1122686862945557, + "sampling/importance_sampling_ratio/mean": 0.650765061378479, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.47726941108703613, + "sampling/sampling_logp_difference/mean": 0.027112768962979317, + "step": 61, + "step_time": 23.71764762001112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 628.3125, + "completions/mean_terminated_length": 628.3125, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3224291801452637, + "epoch": 0.124, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1645125448703766, + "kl": 0.025764177553355694, + "learning_rate": 3e-05, + "loss": 0.17419062554836273, + "num_tokens": 603055.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1662254333496094, + "sampling/importance_sampling_ratio/mean": 0.5809424519538879, + "sampling/importance_sampling_ratio/min": 0.03490918502211571, + "sampling/sampling_logp_difference/max": 0.4525270462036133, + "sampling/sampling_logp_difference/mean": 0.028948483988642693, + "step": 62, + "step_time": 35.74759274255484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 597.375, + "completions/mean_terminated_length": 597.375, + "completions/min_length": 478.0, + "completions/min_terminated_length": 478.0, + "entropy": 1.1552416533231735, + "epoch": 0.126, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20707836747169495, + "kl": 0.026473846402950585, + "learning_rate": 3e-05, + "loss": -0.019145065918564796, + "num_tokens": 614341.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.0268709659576416, + "sampling/importance_sampling_ratio/mean": 0.7066210508346558, + "sampling/importance_sampling_ratio/min": 0.1595209240913391, + "sampling/sampling_logp_difference/max": 0.42907285690307617, + "sampling/sampling_logp_difference/mean": 0.028000790625810623, + "step": 63, + "step_time": 39.37250621803105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 568.5, + "completions/mean_terminated_length": 568.5, + "completions/min_length": 509.0, + "completions/min_terminated_length": 509.0, + "entropy": 1.2810933291912079, + "epoch": 0.128, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21552008390426636, + "kl": 0.029041914734989405, + "learning_rate": 3e-05, + "loss": 0.037037670612335205, + "num_tokens": 625165.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.1896748542785645, + "sampling/importance_sampling_ratio/mean": 0.6408567428588867, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.374176025390625, + "sampling/sampling_logp_difference/mean": 0.02858484350144863, + "step": 64, + "step_time": 18.69576471252367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 622.75, + "completions/mean_terminated_length": 622.75, + "completions/min_length": 545.0, + "completions/min_terminated_length": 545.0, + "entropy": 1.32467532902956, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2287176102399826, + "kl": 0.023987084976397455, + "learning_rate": 3e-05, + "loss": 0.0731797143816948, + "num_tokens": 636633.0, + "reward": 6.375, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 1.4288272857666016, + "sampling/importance_sampling_ratio/mean": 0.5977773666381836, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4503474235534668, + "sampling/sampling_logp_difference/mean": 0.02755960263311863, + "step": 65, + "step_time": 27.502957582939416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 767.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 639.25, + "completions/mean_terminated_length": 639.25, + "completions/min_length": 554.0, + "completions/min_terminated_length": 554.0, + "entropy": 1.400998167693615, + "epoch": 0.132, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27425676584243774, + "kl": 0.028104660217650235, + "learning_rate": 3e-05, + "loss": 0.10324293375015259, + "num_tokens": 648597.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.971261501312256, + "sampling/importance_sampling_ratio/mean": 0.6433250904083252, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4819211959838867, + "sampling/sampling_logp_difference/mean": 0.029852069914340973, + "step": 66, + "step_time": 26.79405551031232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 541.375, + "completions/mean_terminated_length": 541.375, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.2635268718004227, + "epoch": 0.134, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13181555271148682, + "kl": 0.03373931790702045, + "learning_rate": 3e-05, + "loss": -0.11447598040103912, + "num_tokens": 658875.0, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 1.1883972883224487, + "sampling/importance_sampling_ratio/mean": 0.4192371368408203, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.44372403621673584, + "sampling/sampling_logp_difference/mean": 0.02911720983684063, + "step": 67, + "step_time": 27.6137525443919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 842.0, + "completions/max_terminated_length": 842.0, + "completions/mean_length": 689.4375, + "completions/mean_terminated_length": 689.4375, + "completions/min_length": 544.0, + "completions/min_terminated_length": 544.0, + "entropy": 1.2496536895632744, + "epoch": 0.136, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20560002326965332, + "kl": 0.026702777307946235, + "learning_rate": 3e-05, + "loss": -0.10130438208580017, + "num_tokens": 671690.0, + "reward": 5.875, + "reward_std": 2.0289571285247803, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 2.0289571285247803, + "sampling/importance_sampling_ratio/max": 2.8383262157440186, + "sampling/importance_sampling_ratio/mean": 0.9476768374443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35082435607910156, + "sampling/sampling_logp_difference/mean": 0.028364315629005432, + "step": 68, + "step_time": 29.35345455724746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 701.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 614.25, + "completions/mean_terminated_length": 614.25, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.1948458775877953, + "epoch": 0.138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20431844890117645, + "kl": 0.03377161466050893, + "learning_rate": 3e-05, + "loss": -0.0560678169131279, + "num_tokens": 683046.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.162766933441162, + "sampling/importance_sampling_ratio/mean": 0.5678162574768066, + "sampling/importance_sampling_ratio/min": 0.05678822472691536, + "sampling/sampling_logp_difference/max": 0.5758893489837646, + "sampling/sampling_logp_difference/mean": 0.028125843033194542, + "step": 69, + "step_time": 27.574916049838066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 588.3125, + "completions/mean_terminated_length": 588.3125, + "completions/min_length": 532.0, + "completions/min_terminated_length": 532.0, + "entropy": 1.2782762721180916, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26451998949050903, + "kl": 0.03907236340455711, + "learning_rate": 3e-05, + "loss": 0.17035090923309326, + "num_tokens": 694323.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.950176477432251, + "sampling/importance_sampling_ratio/mean": 0.45171743631362915, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46894216537475586, + "sampling/sampling_logp_difference/mean": 0.02863166108727455, + "step": 70, + "step_time": 19.817490340210497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 597.125, + "completions/mean_terminated_length": 597.125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2051330730319023, + "epoch": 0.142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5549097657203674, + "kl": 0.03670362115371972, + "learning_rate": 3e-05, + "loss": 0.4998631179332733, + "num_tokens": 705773.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.9404170513153076, + "sampling/importance_sampling_ratio/mean": 0.7522475123405457, + "sampling/importance_sampling_ratio/min": 0.05456363409757614, + "sampling/sampling_logp_difference/max": 0.4324514865875244, + "sampling/sampling_logp_difference/mean": 0.028340937569737434, + "step": 71, + "step_time": 41.66373607888818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 691.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 604.125, + "completions/mean_terminated_length": 604.125, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.18794547021389, + "epoch": 0.144, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10015435516834259, + "kl": 0.03911226138006896, + "learning_rate": 3e-05, + "loss": -0.02419177070260048, + "num_tokens": 717159.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1231037378311157, + "sampling/importance_sampling_ratio/mean": 0.4037884473800659, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6400742530822754, + "sampling/sampling_logp_difference/mean": 0.028367817401885986, + "step": 72, + "step_time": 23.86539705330506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 721.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 597.1875, + "completions/mean_terminated_length": 597.1875, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.2641174346208572, + "epoch": 0.146, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07918722927570343, + "kl": 0.03177848691120744, + "learning_rate": 3e-05, + "loss": -0.027635926380753517, + "num_tokens": 728402.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.5195796489715576, + "sampling/importance_sampling_ratio/mean": 0.4324396848678589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6819734573364258, + "sampling/sampling_logp_difference/mean": 0.030029678717255592, + "step": 73, + "step_time": 29.29490938829258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 640.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 574.375, + "completions/mean_terminated_length": 574.375, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3364054411649704, + "epoch": 0.148, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3853444755077362, + "kl": 0.03433372196741402, + "learning_rate": 3e-05, + "loss": -0.031142480671405792, + "num_tokens": 739632.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6853926181793213, + "sampling/importance_sampling_ratio/mean": 0.9200767874717712, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42021608352661133, + "sampling/sampling_logp_difference/mean": 0.030795859172940254, + "step": 74, + "step_time": 34.52008486771956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 618.25, + "completions/mean_terminated_length": 618.25, + "completions/min_length": 539.0, + "completions/min_terminated_length": 539.0, + "entropy": 1.365300178527832, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18101376295089722, + "kl": 0.02779634529724717, + "learning_rate": 3e-05, + "loss": -0.2718795835971832, + "num_tokens": 751164.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.8397568464279175, + "sampling/importance_sampling_ratio/mean": 0.5582400560379028, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42606019973754883, + "sampling/sampling_logp_difference/mean": 0.028895940631628036, + "step": 75, + "step_time": 18.76476171752438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 799.0, + "completions/max_terminated_length": 799.0, + "completions/mean_length": 603.375, + "completions/mean_terminated_length": 603.375, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2181015871465206, + "epoch": 0.152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1852622777223587, + "kl": 0.03176840906962752, + "learning_rate": 3e-05, + "loss": -0.10660596191883087, + "num_tokens": 762370.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.317336082458496, + "sampling/importance_sampling_ratio/mean": 0.550554633140564, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.481325626373291, + "sampling/sampling_logp_difference/mean": 0.028557566925883293, + "step": 76, + "step_time": 27.090129756834358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 572.9375, + "completions/mean_terminated_length": 572.9375, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2986655682325363, + "epoch": 0.154, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1531844586133957, + "kl": 0.03523328877054155, + "learning_rate": 3e-05, + "loss": -0.20856647193431854, + "num_tokens": 773169.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 2.855010986328125, + "sampling/importance_sampling_ratio/mean": 0.8239375352859497, + "sampling/importance_sampling_ratio/min": 0.1521405428647995, + "sampling/sampling_logp_difference/max": 0.4692528247833252, + "sampling/sampling_logp_difference/mean": 0.029906686395406723, + "step": 77, + "step_time": 25.004970545414835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 607.1875, + "completions/mean_terminated_length": 607.1875, + "completions/min_length": 540.0, + "completions/min_terminated_length": 540.0, + "entropy": 1.1003647185862064, + "epoch": 0.156, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14113759994506836, + "kl": 0.025135049945674837, + "learning_rate": 3e-05, + "loss": -0.10802068561315536, + "num_tokens": 784724.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.0096027851104736, + "sampling/importance_sampling_ratio/mean": 0.613497257232666, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4082927703857422, + "sampling/sampling_logp_difference/mean": 0.027884263545274734, + "step": 78, + "step_time": 29.614154959097505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 568.0625, + "completions/mean_terminated_length": 568.0625, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.266264721751213, + "epoch": 0.158, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18073000013828278, + "kl": 0.028119354974478483, + "learning_rate": 3e-05, + "loss": -0.0687609314918518, + "num_tokens": 795517.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.277299404144287, + "sampling/importance_sampling_ratio/mean": 0.3485238552093506, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4658241271972656, + "sampling/sampling_logp_difference/mean": 0.029626762494444847, + "step": 79, + "step_time": 23.5287338164635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 565.8125, + "completions/mean_terminated_length": 565.8125, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2773499339818954, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.31362995505332947, + "kl": 0.028471783734858036, + "learning_rate": 3e-05, + "loss": 0.059164684265851974, + "num_tokens": 806258.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.5347814559936523, + "sampling/importance_sampling_ratio/mean": 0.7027873396873474, + "sampling/importance_sampling_ratio/min": 0.06356429308652878, + "sampling/sampling_logp_difference/max": 0.4123659133911133, + "sampling/sampling_logp_difference/mean": 0.02946357987821102, + "step": 80, + "step_time": 24.251087154261768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 600.0, + "completions/max_terminated_length": 600.0, + "completions/mean_length": 536.1875, + "completions/mean_terminated_length": 536.1875, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.211024284362793, + "epoch": 0.162, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1959177404642105, + "kl": 0.02152467134874314, + "learning_rate": 3e-05, + "loss": 0.14755703508853912, + "num_tokens": 816717.0, + "reward": 6.0, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.764387369155884, + "sampling/importance_sampling_ratio/mean": 0.8167816400527954, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.28648805618286133, + "sampling/sampling_logp_difference/mean": 0.02814806066453457, + "step": 81, + "step_time": 22.752198832575232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 568.625, + "completions/mean_terminated_length": 568.625, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "entropy": 1.2584010139107704, + "epoch": 0.164, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16224367916584015, + "kl": 0.02812566957436502, + "learning_rate": 3e-05, + "loss": -0.1586945354938507, + "num_tokens": 827591.0, + "reward": 6.5, + "reward_std": 1.26491117477417, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.26491117477417, + "sampling/importance_sampling_ratio/max": 2.4172537326812744, + "sampling/importance_sampling_ratio/mean": 0.7026975154876709, + "sampling/importance_sampling_ratio/min": 0.1125984862446785, + "sampling/sampling_logp_difference/max": 0.3966444730758667, + "sampling/sampling_logp_difference/mean": 0.02937215007841587, + "step": 82, + "step_time": 22.57465209439397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 783.0, + "completions/max_terminated_length": 783.0, + "completions/mean_length": 583.5625, + "completions/mean_terminated_length": 583.5625, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2498536258935928, + "epoch": 0.166, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28551891446113586, + "kl": 0.023335880250670016, + "learning_rate": 3e-05, + "loss": 0.09868354350328445, + "num_tokens": 838760.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 1.9430233240127563, + "sampling/importance_sampling_ratio/mean": 0.7391272783279419, + "sampling/importance_sampling_ratio/min": 0.2032935619354248, + "sampling/sampling_logp_difference/max": 0.3390723466873169, + "sampling/sampling_logp_difference/mean": 0.02833949774503708, + "step": 83, + "step_time": 24.9633763525635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 581.0625, + "completions/mean_terminated_length": 581.0625, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.1582137942314148, + "epoch": 0.168, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1089889332652092, + "kl": 0.02546319324756041, + "learning_rate": 3e-05, + "loss": -0.04368923604488373, + "num_tokens": 849945.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.6800583600997925, + "sampling/importance_sampling_ratio/mean": 0.4864083528518677, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48818397521972656, + "sampling/sampling_logp_difference/mean": 0.02942320704460144, + "step": 84, + "step_time": 22.7196712391451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 553.3125, + "completions/mean_terminated_length": 553.3125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.1762890554964542, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18219847977161407, + "kl": 0.023275951389223337, + "learning_rate": 3e-05, + "loss": 0.055040232837200165, + "num_tokens": 860734.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.2940757274627686, + "sampling/importance_sampling_ratio/mean": 0.6381184458732605, + "sampling/importance_sampling_ratio/min": 0.08803392946720123, + "sampling/sampling_logp_difference/max": 0.3728243112564087, + "sampling/sampling_logp_difference/mean": 0.028103860095143318, + "step": 85, + "step_time": 28.569310082122684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 538.5, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.1561524420976639, + "epoch": 0.172, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2631295323371887, + "kl": 0.027657793601974845, + "learning_rate": 3e-05, + "loss": 0.019699495285749435, + "num_tokens": 871182.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.950981378555298, + "sampling/importance_sampling_ratio/mean": 0.5496660470962524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.358644962310791, + "sampling/sampling_logp_difference/mean": 0.02922222763299942, + "step": 86, + "step_time": 19.95468496438116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 688.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 528.625, + "completions/mean_terminated_length": 528.625, + "completions/min_length": 418.0, + "completions/min_terminated_length": 418.0, + "entropy": 1.382395550608635, + "epoch": 0.174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25233277678489685, + "kl": 0.025461523793637753, + "learning_rate": 3e-05, + "loss": -0.0012568621896207333, + "num_tokens": 881272.0, + "reward": 6.3125, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 1.940340280532837, + "sampling/importance_sampling_ratio/mean": 0.44232380390167236, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3433331251144409, + "sampling/sampling_logp_difference/mean": 0.030555889010429382, + "step": 87, + "step_time": 29.044239778537303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 537.0, + "completions/mean_terminated_length": 537.0, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.1740282103419304, + "epoch": 0.176, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1609290987253189, + "kl": 0.023582924506627023, + "learning_rate": 3e-05, + "loss": -0.0040507810190320015, + "num_tokens": 891608.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.8592076301574707, + "sampling/importance_sampling_ratio/mean": 0.4413543939590454, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3994784355163574, + "sampling/sampling_logp_difference/mean": 0.028627343475818634, + "step": 88, + "step_time": 24.642031190916896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 562.0, + "completions/mean_terminated_length": 562.0, + "completions/min_length": 490.0, + "completions/min_terminated_length": 490.0, + "entropy": 1.3354259580373764, + "epoch": 0.178, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25136521458625793, + "kl": 0.03104234568309039, + "learning_rate": 3e-05, + "loss": -0.10014888644218445, + "num_tokens": 902472.0, + "reward": 6.5, + "reward_std": 1.154700517654419, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.154700517654419, + "sampling/importance_sampling_ratio/max": 2.0172529220581055, + "sampling/importance_sampling_ratio/mean": 0.5528109073638916, + "sampling/importance_sampling_ratio/min": 0.031755149364471436, + "sampling/sampling_logp_difference/max": 0.48168420791625977, + "sampling/sampling_logp_difference/mean": 0.029525987803936005, + "step": 89, + "step_time": 23.934129936154932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 543.6875, + "completions/mean_terminated_length": 543.6875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.3072879239916801, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2416294664144516, + "kl": 0.02474795945454389, + "learning_rate": 3e-05, + "loss": 0.02994484454393387, + "num_tokens": 913003.0, + "reward": 6.125, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4122473001480103, + "sampling/importance_sampling_ratio/mean": 0.5672672390937805, + "sampling/importance_sampling_ratio/min": 0.1312582641839981, + "sampling/sampling_logp_difference/max": 0.36547136306762695, + "sampling/sampling_logp_difference/mean": 0.030115650966763496, + "step": 90, + "step_time": 16.719651044346392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 587.5, + "completions/mean_terminated_length": 587.5, + "completions/min_length": 491.0, + "completions/min_terminated_length": 491.0, + "entropy": 1.2642723061144352, + "epoch": 0.182, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11401186883449554, + "kl": 0.018764875654596835, + "learning_rate": 3e-05, + "loss": 0.17740829288959503, + "num_tokens": 923971.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.223915934562683, + "sampling/importance_sampling_ratio/mean": 0.4373893141746521, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.43628501892089844, + "sampling/sampling_logp_difference/mean": 0.027218280360102654, + "step": 91, + "step_time": 21.256958127953112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 519.3125, + "completions/mean_terminated_length": 519.3125, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.2509336844086647, + "epoch": 0.184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14000695943832397, + "kl": 0.017409664229489863, + "learning_rate": 3e-05, + "loss": -0.1092228814959526, + "num_tokens": 933880.0, + "reward": 7.125, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 2.4079525470733643, + "sampling/importance_sampling_ratio/mean": 0.6406391263008118, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3843420743942261, + "sampling/sampling_logp_difference/mean": 0.02841360680758953, + "step": 92, + "step_time": 20.99564675288275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 681.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 587.8125, + "completions/mean_terminated_length": 587.8125, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.1215453520417213, + "epoch": 0.186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16520461440086365, + "kl": 0.028165725176222622, + "learning_rate": 3e-05, + "loss": -0.15029624104499817, + "num_tokens": 945269.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.7825064659118652, + "sampling/importance_sampling_ratio/mean": 0.5902000069618225, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3571600914001465, + "sampling/sampling_logp_difference/mean": 0.02762974239885807, + "step": 93, + "step_time": 17.922352592926472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 566.0, + "completions/mean_terminated_length": 566.0, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.2864234894514084, + "epoch": 0.188, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24750091135501862, + "kl": 0.02070689742686227, + "learning_rate": 3e-05, + "loss": 0.2850193381309509, + "num_tokens": 956021.0, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "sampling/importance_sampling_ratio/max": 1.9146449565887451, + "sampling/importance_sampling_ratio/mean": 0.6849822402000427, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4927506446838379, + "sampling/sampling_logp_difference/mean": 0.029227914288640022, + "step": 94, + "step_time": 23.034203104209155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 516.6875, + "completions/mean_terminated_length": 516.6875, + "completions/min_length": 486.0, + "completions/min_terminated_length": 486.0, + "entropy": 1.2404684573411942, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11928244680166245, + "kl": 0.023086783126927912, + "learning_rate": 3e-05, + "loss": -0.032361116260290146, + "num_tokens": 965920.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.216193437576294, + "sampling/importance_sampling_ratio/mean": 0.32463955879211426, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37001991271972656, + "sampling/sampling_logp_difference/mean": 0.02843114361166954, + "step": 95, + "step_time": 15.103232022374868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 650.0, + "completions/max_terminated_length": 650.0, + "completions/mean_length": 548.4375, + "completions/mean_terminated_length": 548.4375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.2861761301755905, + "epoch": 0.192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2920665442943573, + "kl": 0.017841250344645232, + "learning_rate": 3e-05, + "loss": 0.1640928089618683, + "num_tokens": 976695.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.8954812288284302, + "sampling/importance_sampling_ratio/mean": 0.754618763923645, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31093156337738037, + "sampling/sampling_logp_difference/mean": 0.02842729538679123, + "step": 96, + "step_time": 40.74571412149817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 553.5625, + "completions/mean_terminated_length": 553.5625, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.1928813084959984, + "epoch": 0.194, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29983460903167725, + "kl": 0.020173200638964772, + "learning_rate": 3e-05, + "loss": 0.05926981568336487, + "num_tokens": 987120.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.0362496376037598, + "sampling/importance_sampling_ratio/mean": 0.6645779609680176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40996313095092773, + "sampling/sampling_logp_difference/mean": 0.02854262851178646, + "step": 97, + "step_time": 17.38945102225989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 546.1875, + "completions/mean_terminated_length": 546.1875, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "entropy": 1.365786962211132, + "epoch": 0.196, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12385546416044235, + "kl": 0.02262102661188692, + "learning_rate": 3e-05, + "loss": -0.09927551448345184, + "num_tokens": 997395.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2717533111572266, + "sampling/importance_sampling_ratio/mean": 0.5988417267799377, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35561084747314453, + "sampling/sampling_logp_difference/mean": 0.029298899695277214, + "step": 98, + "step_time": 23.35145698580891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 593.4375, + "completions/mean_terminated_length": 593.4375, + "completions/min_length": 508.0, + "completions/min_terminated_length": 508.0, + "entropy": 1.1754724085330963, + "epoch": 0.198, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2662176787853241, + "kl": 0.02589411090593785, + "learning_rate": 3e-05, + "loss": 0.2574020326137543, + "num_tokens": 1008458.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1308085918426514, + "sampling/importance_sampling_ratio/mean": 0.6420408487319946, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42550981044769287, + "sampling/sampling_logp_difference/mean": 0.02820964716374874, + "step": 99, + "step_time": 21.02622760878876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 538.4375, + "completions/mean_terminated_length": 538.4375, + "completions/min_length": 483.0, + "completions/min_terminated_length": 483.0, + "entropy": 1.3136962801218033, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2549664378166199, + "kl": 0.024644543533213437, + "learning_rate": 3e-05, + "loss": 0.06747792661190033, + "num_tokens": 1018793.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.3327062129974365, + "sampling/importance_sampling_ratio/mean": 0.7165549993515015, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4304838180541992, + "sampling/sampling_logp_difference/mean": 0.0299112256616354, + "step": 100, + "step_time": 16.768271412234753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 567.875, + "completions/mean_terminated_length": 567.875, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.15415108948946, + "epoch": 0.202, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34537598490715027, + "kl": 0.025688456720672548, + "learning_rate": 3e-05, + "loss": -0.21041882038116455, + "num_tokens": 1029751.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.282634973526001, + "sampling/importance_sampling_ratio/mean": 0.5392330288887024, + "sampling/importance_sampling_ratio/min": 0.026465320959687233, + "sampling/sampling_logp_difference/max": 0.3903813362121582, + "sampling/sampling_logp_difference/mean": 0.02962569333612919, + "step": 101, + "step_time": 16.715499105863273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 574.125, + "completions/mean_terminated_length": 574.125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.3766441270709038, + "epoch": 0.204, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27241969108581543, + "kl": 0.025680337683297694, + "learning_rate": 3e-05, + "loss": 0.24403473734855652, + "num_tokens": 1040601.0, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 2.3365371227264404, + "sampling/importance_sampling_ratio/mean": 0.6375491619110107, + "sampling/importance_sampling_ratio/min": 0.07846305519342422, + "sampling/sampling_logp_difference/max": 0.4158613681793213, + "sampling/sampling_logp_difference/mean": 0.030232973396778107, + "step": 102, + "step_time": 19.942134194541723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 557.75, + "completions/mean_terminated_length": 557.75, + "completions/min_length": 510.0, + "completions/min_terminated_length": 510.0, + "entropy": 1.3887510225176811, + "epoch": 0.206, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2711203992366791, + "kl": 0.022622586810030043, + "learning_rate": 3e-05, + "loss": -0.07210355252027512, + "num_tokens": 1051229.0, + "reward": 6.3125, + "reward_std": 1.0144785642623901, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.0144785642623901, + "sampling/importance_sampling_ratio/max": 1.8279200792312622, + "sampling/importance_sampling_ratio/mean": 0.5626887083053589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3629894256591797, + "sampling/sampling_logp_difference/mean": 0.030201904475688934, + "step": 103, + "step_time": 30.44108156999573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 544.75, + "completions/mean_terminated_length": 544.75, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.1656717136502266, + "epoch": 0.208, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27351662516593933, + "kl": 0.02198210428468883, + "learning_rate": 3e-05, + "loss": 0.06065649166703224, + "num_tokens": 1061745.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.500911235809326, + "sampling/importance_sampling_ratio/mean": 0.8908482789993286, + "sampling/importance_sampling_ratio/min": 0.05167346075177193, + "sampling/sampling_logp_difference/max": 0.3885481357574463, + "sampling/sampling_logp_difference/mean": 0.027608510106801987, + "step": 104, + "step_time": 16.46229960815981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 616.0, + "completions/max_terminated_length": 616.0, + "completions/mean_length": 562.875, + "completions/mean_terminated_length": 562.875, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.3012276738882065, + "epoch": 0.21, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21975310146808624, + "kl": 0.023721053614281118, + "learning_rate": 3e-05, + "loss": 0.06463687866926193, + "num_tokens": 1072231.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 1.7785983085632324, + "sampling/importance_sampling_ratio/mean": 0.5429776906967163, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.9359657764434814, + "sampling/sampling_logp_difference/mean": 0.030092012137174606, + "step": 105, + "step_time": 24.80119998846203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 580.0625, + "completions/mean_terminated_length": 580.0625, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.3428374752402306, + "epoch": 0.212, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23242872953414917, + "kl": 0.025716557982377708, + "learning_rate": 3e-05, + "loss": -0.004262822680175304, + "num_tokens": 1083184.0, + "reward": 6.375, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.0552361011505127, + "sampling/importance_sampling_ratio/mean": 0.6959555745124817, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7246572971343994, + "sampling/sampling_logp_difference/mean": 0.02952728420495987, + "step": 106, + "step_time": 17.27699494967237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 655.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 589.75, + "completions/mean_terminated_length": 589.75, + "completions/min_length": 547.0, + "completions/min_terminated_length": 547.0, + "entropy": 1.471379242837429, + "epoch": 0.214, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23287689685821533, + "kl": 0.025674917036667466, + "learning_rate": 3e-05, + "loss": 0.08491670340299606, + "num_tokens": 1094204.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.1368408203125, + "sampling/importance_sampling_ratio/mean": 0.5767678022384644, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36995506286621094, + "sampling/sampling_logp_difference/mean": 0.02880111336708069, + "step": 107, + "step_time": 40.17427978478372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 595.6875, + "completions/mean_terminated_length": 595.6875, + "completions/min_length": 524.0, + "completions/min_terminated_length": 524.0, + "entropy": 1.2550728917121887, + "epoch": 0.216, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09948146343231201, + "kl": 0.022876911563798785, + "learning_rate": 3e-05, + "loss": 0.04861483350396156, + "num_tokens": 1105303.0, + "reward": 6.125, + "reward_std": 1.0878112316131592, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0878112316131592, + "sampling/importance_sampling_ratio/max": 1.1923820972442627, + "sampling/importance_sampling_ratio/mean": 0.3343955874443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3477153778076172, + "sampling/sampling_logp_difference/mean": 0.029913678765296936, + "step": 108, + "step_time": 18.59066634438932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 583.5, + "completions/mean_terminated_length": 583.5, + "completions/min_length": 467.0, + "completions/min_terminated_length": 467.0, + "entropy": 1.2097205966711044, + "epoch": 0.218, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1392563134431839, + "kl": 0.0263087993953377, + "learning_rate": 3e-05, + "loss": 0.10488568246364594, + "num_tokens": 1116591.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.0463244915008545, + "sampling/importance_sampling_ratio/mean": 0.4996475875377655, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31003570556640625, + "sampling/sampling_logp_difference/mean": 0.0288787130266428, + "step": 109, + "step_time": 17.845282280817628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 571.375, + "completions/mean_terminated_length": 571.375, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.28146480768919, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5394540429115295, + "kl": 0.024339908617548645, + "learning_rate": 3e-05, + "loss": -0.23892748355865479, + "num_tokens": 1127493.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.671478509902954, + "sampling/importance_sampling_ratio/mean": 1.223832130432129, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650541305541992, + "sampling/sampling_logp_difference/mean": 0.028643080964684486, + "step": 110, + "step_time": 22.93386760680005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 493.875, + "completions/mean_terminated_length": 493.875, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 1.12203798443079, + "epoch": 0.222, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4297163188457489, + "kl": 0.024493444827385247, + "learning_rate": 3e-05, + "loss": -0.21332937479019165, + "num_tokens": 1136979.0, + "reward": 6.9375, + "reward_std": 0.6800735592842102, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.6800735592842102, + "sampling/importance_sampling_ratio/max": 2.889394760131836, + "sampling/importance_sampling_ratio/mean": 0.7321407794952393, + "sampling/importance_sampling_ratio/min": 0.02116413414478302, + "sampling/sampling_logp_difference/max": 0.49600934982299805, + "sampling/sampling_logp_difference/mean": 0.028577474877238274, + "step": 111, + "step_time": 21.056686570402235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 672.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 583.8125, + "completions/mean_terminated_length": 583.8125, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.4159239530563354, + "epoch": 0.224, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.008436380885541439, + "kl": 0.024869016953743994, + "learning_rate": 3e-05, + "loss": 0.0004943995736539364, + "num_tokens": 1148176.0, + "reward": 7.0, + "reward_std": 0.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.642366886138916, + "sampling/importance_sampling_ratio/mean": 0.7060814499855042, + "sampling/importance_sampling_ratio/min": 0.006825839169323444, + "sampling/sampling_logp_difference/max": 0.572547435760498, + "sampling/sampling_logp_difference/mean": 0.03075096383690834, + "step": 112, + "step_time": 20.1555822850205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 638.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 582.75, + "completions/mean_terminated_length": 582.75, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2932439520955086, + "epoch": 0.226, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2439681738615036, + "kl": 0.025248280493542552, + "learning_rate": 3e-05, + "loss": -0.22745110094547272, + "num_tokens": 1159380.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.5284109115600586, + "sampling/importance_sampling_ratio/mean": 1.0150926113128662, + "sampling/importance_sampling_ratio/min": 0.021104907616972923, + "sampling/sampling_logp_difference/max": 0.2513730525970459, + "sampling/sampling_logp_difference/mean": 0.02884429693222046, + "step": 113, + "step_time": 18.09852197067812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 580.0, + "completions/mean_terminated_length": 580.0, + "completions/min_length": 535.0, + "completions/min_terminated_length": 535.0, + "entropy": 1.246352232992649, + "epoch": 0.228, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23755788803100586, + "kl": 0.02679906424600631, + "learning_rate": 3e-05, + "loss": 0.3550976812839508, + "num_tokens": 1170628.0, + "reward": 7.25, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.750225305557251, + "sampling/importance_sampling_ratio/mean": 1.0749173164367676, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5576918125152588, + "sampling/sampling_logp_difference/mean": 0.03159492090344429, + "step": 114, + "step_time": 24.629896679893136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 657.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 562.6875, + "completions/mean_terminated_length": 562.6875, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.253239594399929, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3102787733078003, + "kl": 0.027133187628351152, + "learning_rate": 3e-05, + "loss": -0.05118201673030853, + "num_tokens": 1181295.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.634671926498413, + "sampling/importance_sampling_ratio/mean": 0.8949281573295593, + "sampling/importance_sampling_ratio/min": 0.09920533001422882, + "sampling/sampling_logp_difference/max": 0.6224374771118164, + "sampling/sampling_logp_difference/mean": 0.030200565233826637, + "step": 115, + "step_time": 38.715506959706545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 610.9375, + "completions/mean_terminated_length": 610.9375, + "completions/min_length": 538.0, + "completions/min_terminated_length": 538.0, + "entropy": 1.2940760999917984, + "epoch": 0.232, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14933602511882782, + "kl": 0.031228074803948402, + "learning_rate": 3e-05, + "loss": -0.05550215765833855, + "num_tokens": 1192750.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6116093397140503, + "sampling/importance_sampling_ratio/mean": 0.41488125920295715, + "sampling/importance_sampling_ratio/min": 0.009796842001378536, + "sampling/sampling_logp_difference/max": 0.5450258255004883, + "sampling/sampling_logp_difference/mean": 0.03152918070554733, + "step": 116, + "step_time": 17.821606623008847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 560.8125, + "completions/mean_terminated_length": 560.8125, + "completions/min_length": 501.0, + "completions/min_terminated_length": 501.0, + "entropy": 1.21239273250103, + "epoch": 0.234, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23778750002384186, + "kl": 0.03231424337718636, + "learning_rate": 3e-05, + "loss": -0.09421571344137192, + "num_tokens": 1203219.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.1054162979125977, + "sampling/importance_sampling_ratio/mean": 0.7292319536209106, + "sampling/importance_sampling_ratio/min": 0.010288448072969913, + "sampling/sampling_logp_difference/max": 0.8687701225280762, + "sampling/sampling_logp_difference/mean": 0.030015992000699043, + "step": 117, + "step_time": 16.80020274501294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 583.4375, + "completions/mean_terminated_length": 583.4375, + "completions/min_length": 503.0, + "completions/min_terminated_length": 503.0, + "entropy": 1.0914121232926846, + "epoch": 0.236, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.177206888794899, + "kl": 0.027808396029286087, + "learning_rate": 3e-05, + "loss": 0.010135526768863201, + "num_tokens": 1214562.0, + "reward": 6.375, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.6413226127624512, + "sampling/importance_sampling_ratio/mean": 0.5455202460289001, + "sampling/importance_sampling_ratio/min": 0.14393718540668488, + "sampling/sampling_logp_difference/max": 0.37839651107788086, + "sampling/sampling_logp_difference/mean": 0.02755727432668209, + "step": 118, + "step_time": 32.04508572584018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 802.0, + "completions/max_terminated_length": 802.0, + "completions/mean_length": 623.875, + "completions/mean_terminated_length": 623.875, + "completions/min_length": 555.0, + "completions/min_terminated_length": 555.0, + "entropy": 1.2099597677588463, + "epoch": 0.238, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10887355357408524, + "kl": 0.029803494922816753, + "learning_rate": 3e-05, + "loss": -0.05460066720843315, + "num_tokens": 1226136.0, + "reward": 6.25, + "reward_std": 1.983263373374939, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.983263373374939, + "sampling/importance_sampling_ratio/max": 1.5456031560897827, + "sampling/importance_sampling_ratio/mean": 0.4625241756439209, + "sampling/importance_sampling_ratio/min": 0.06713607162237167, + "sampling/sampling_logp_difference/max": 0.5650186538696289, + "sampling/sampling_logp_difference/mean": 0.03079008124768734, + "step": 119, + "step_time": 20.80143166380003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 621.0, + "completions/mean_terminated_length": 621.0, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.3042216151952744, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1758948117494583, + "kl": 0.03042414935771376, + "learning_rate": 3e-05, + "loss": -0.1489834189414978, + "num_tokens": 1237856.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.4328413009643555, + "sampling/importance_sampling_ratio/mean": 0.7058912515640259, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40829265117645264, + "sampling/sampling_logp_difference/mean": 0.029741039499640465, + "step": 120, + "step_time": 17.90572352707386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 593.8125, + "completions/mean_terminated_length": 593.8125, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 1.132676463574171, + "epoch": 0.242, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42553070187568665, + "kl": 0.04428348131477833, + "learning_rate": 3e-05, + "loss": 0.22666211426258087, + "num_tokens": 1249173.0, + "reward": 5.625, + "reward_std": 1.8574175834655762, + "rewards/quality_reward/mean": 5.625, + "rewards/quality_reward/std": 1.8574175834655762, + "sampling/importance_sampling_ratio/max": 2.5172836780548096, + "sampling/importance_sampling_ratio/mean": 0.6919515132904053, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6260476112365723, + "sampling/sampling_logp_difference/mean": 0.030399736016988754, + "step": 121, + "step_time": 38.02521731564775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 587.625, + "completions/mean_terminated_length": 587.625, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.2395975515246391, + "epoch": 0.244, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.248436838388443, + "kl": 0.03361149993725121, + "learning_rate": 3e-05, + "loss": 0.024570390582084656, + "num_tokens": 1260463.0, + "reward": 6.0625, + "reward_std": 1.236594796180725, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.236594796180725, + "sampling/importance_sampling_ratio/max": 2.833437442779541, + "sampling/importance_sampling_ratio/mean": 0.8825340867042542, + "sampling/importance_sampling_ratio/min": 0.06326956301927567, + "sampling/sampling_logp_difference/max": 0.540553092956543, + "sampling/sampling_logp_difference/mean": 0.030399201437830925, + "step": 122, + "step_time": 17.796182619407773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 585.3125, + "completions/mean_terminated_length": 585.3125, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.375852882862091, + "epoch": 0.246, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16029156744480133, + "kl": 0.03495740657672286, + "learning_rate": 3e-05, + "loss": -0.053390923887491226, + "num_tokens": 1271644.0, + "reward": 6.4375, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.6618704795837402, + "sampling/importance_sampling_ratio/mean": 0.5607253909111023, + "sampling/importance_sampling_ratio/min": 0.08802779763936996, + "sampling/sampling_logp_difference/max": 0.7028732299804688, + "sampling/sampling_logp_difference/mean": 0.031593482941389084, + "step": 123, + "step_time": 14.86260191956535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 630.1875, + "completions/mean_terminated_length": 630.1875, + "completions/min_length": 571.0, + "completions/min_terminated_length": 571.0, + "entropy": 1.3353190049529076, + "epoch": 0.248, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19288285076618195, + "kl": 0.033586084260605276, + "learning_rate": 3e-05, + "loss": -0.010514559224247932, + "num_tokens": 1283351.0, + "reward": 6.625, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4858638048171997, + "sampling/importance_sampling_ratio/mean": 0.5407211184501648, + "sampling/importance_sampling_ratio/min": 0.007580960635095835, + "sampling/sampling_logp_difference/max": 0.6886825561523438, + "sampling/sampling_logp_difference/mean": 0.030591927468776703, + "step": 124, + "step_time": 19.809663326013833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 599.875, + "completions/mean_terminated_length": 599.875, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.168940719217062, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15600983798503876, + "kl": 0.03213575447443873, + "learning_rate": 3e-05, + "loss": 0.10287950932979584, + "num_tokens": 1294997.0, + "reward": 7.0, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 1.832617163658142, + "sampling/importance_sampling_ratio/mean": 0.6814589500427246, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5988303422927856, + "sampling/sampling_logp_difference/mean": 0.030487919226288795, + "step": 125, + "step_time": 36.94939920771867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 634.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 582.25, + "completions/mean_terminated_length": 582.25, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.0670793130993843, + "epoch": 0.252, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3560245931148529, + "kl": 0.025753034162335098, + "learning_rate": 3e-05, + "loss": 0.16848862171173096, + "num_tokens": 1305993.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.17899489402771, + "sampling/importance_sampling_ratio/mean": 0.7458471059799194, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7731144428253174, + "sampling/sampling_logp_difference/mean": 0.029648227617144585, + "step": 126, + "step_time": 16.284966757521033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 717.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 650.8125, + "completions/mean_terminated_length": 650.8125, + "completions/min_length": 571.0, + "completions/min_terminated_length": 571.0, + "entropy": 1.2732752412557602, + "epoch": 0.254, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18122251331806183, + "kl": 0.03214431612286717, + "learning_rate": 3e-05, + "loss": 0.16780534386634827, + "num_tokens": 1318054.0, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 1.937699556350708, + "sampling/importance_sampling_ratio/mean": 0.5485143065452576, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.0368115901947021, + "sampling/sampling_logp_difference/mean": 0.03218457102775574, + "step": 127, + "step_time": 19.58785921242088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 743.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 617.375, + "completions/mean_terminated_length": 617.375, + "completions/min_length": 530.0, + "completions/min_terminated_length": 530.0, + "entropy": 1.306506209075451, + "epoch": 0.256, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1356409788131714, + "kl": 0.026579287368804216, + "learning_rate": 3e-05, + "loss": 0.06286599487066269, + "num_tokens": 1329500.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.106013298034668, + "sampling/importance_sampling_ratio/mean": 0.4681059420108795, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.676828145980835, + "sampling/sampling_logp_difference/mean": 0.032271042466163635, + "step": 128, + "step_time": 17.268729500938207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 599.0, + "completions/mean_terminated_length": 599.0, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.3493447452783585, + "epoch": 0.258, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2950517237186432, + "kl": 0.02614310395438224, + "learning_rate": 3e-05, + "loss": -0.059055861085653305, + "num_tokens": 1341020.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.7381844520568848, + "sampling/importance_sampling_ratio/mean": 0.6402126550674438, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5858409404754639, + "sampling/sampling_logp_difference/mean": 0.031067587435245514, + "step": 129, + "step_time": 28.869210730306804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 710.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 585.75, + "completions/mean_terminated_length": 585.75, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2956265732645988, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42696425318717957, + "kl": 0.02683100081048906, + "learning_rate": 3e-05, + "loss": 0.003650778206065297, + "num_tokens": 1351928.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.9060652256011963, + "sampling/importance_sampling_ratio/mean": 0.8535536527633667, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4930081367492676, + "sampling/sampling_logp_difference/mean": 0.031333789229393005, + "step": 130, + "step_time": 18.11609491892159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 568.5625, + "completions/mean_terminated_length": 568.5625, + "completions/min_length": 470.0, + "completions/min_terminated_length": 470.0, + "entropy": 1.2097233161330223, + "epoch": 0.262, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20076203346252441, + "kl": 0.024291134322993457, + "learning_rate": 3e-05, + "loss": -0.01662549562752247, + "num_tokens": 1362825.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.6266331672668457, + "sampling/importance_sampling_ratio/mean": 0.6302506327629089, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.963031530380249, + "sampling/sampling_logp_difference/mean": 0.0319429412484169, + "step": 131, + "step_time": 24.060474771540612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 741.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 605.9375, + "completions/mean_terminated_length": 605.9375, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.2823583781719208, + "epoch": 0.264, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.184434711933136, + "kl": 0.02043789450544864, + "learning_rate": 3e-05, + "loss": -0.0666906088590622, + "num_tokens": 1374296.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.831458568572998, + "sampling/importance_sampling_ratio/mean": 0.5425443649291992, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.671515941619873, + "sampling/sampling_logp_difference/mean": 0.030500290915369987, + "step": 132, + "step_time": 17.10482985060662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 665.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 591.375, + "completions/mean_terminated_length": 591.375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.1699804589152336, + "epoch": 0.266, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4572683274745941, + "kl": 0.025262096198275685, + "learning_rate": 3e-05, + "loss": 0.6353421211242676, + "num_tokens": 1385318.0, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.3584084510803223, + "sampling/importance_sampling_ratio/mean": 0.7995439767837524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3869748115539551, + "sampling/sampling_logp_difference/mean": 0.028536029160022736, + "step": 133, + "step_time": 36.004622367210686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 652.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 554.25, + "completions/mean_terminated_length": 554.25, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1336797252297401, + "epoch": 0.268, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.256655752658844, + "kl": 0.022568197746295482, + "learning_rate": 3e-05, + "loss": 0.026529084891080856, + "num_tokens": 1396234.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 1.9517148733139038, + "sampling/importance_sampling_ratio/mean": 0.8202446699142456, + "sampling/importance_sampling_ratio/min": 0.022647401317954063, + "sampling/sampling_logp_difference/max": 1.145315170288086, + "sampling/sampling_logp_difference/mean": 0.02867746911942959, + "step": 134, + "step_time": 20.057327402289957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 641.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 573.5, + "completions/mean_terminated_length": 573.5, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.1878332123160362, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30302342772483826, + "kl": 0.02298387698829174, + "learning_rate": 3e-05, + "loss": 0.2014756053686142, + "num_tokens": 1407322.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.9701545238494873, + "sampling/importance_sampling_ratio/mean": 0.6178270578384399, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4479391574859619, + "sampling/sampling_logp_difference/mean": 0.03068450093269348, + "step": 135, + "step_time": 36.01238542608917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 534.0, + "completions/mean_terminated_length": 534.0, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2640416622161865, + "epoch": 0.272, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39115583896636963, + "kl": 0.020954113570041955, + "learning_rate": 3e-05, + "loss": 0.3642374873161316, + "num_tokens": 1418010.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.0828306674957275, + "sampling/importance_sampling_ratio/mean": 0.7149391770362854, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5613884925842285, + "sampling/sampling_logp_difference/mean": 0.0298798568546772, + "step": 136, + "step_time": 20.20259432308376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 538.8125, + "completions/mean_terminated_length": 538.8125, + "completions/min_length": 451.0, + "completions/min_terminated_length": 451.0, + "entropy": 1.2438515722751617, + "epoch": 0.274, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4104433059692383, + "kl": 0.02069689182098955, + "learning_rate": 3e-05, + "loss": -0.15938539803028107, + "num_tokens": 1428335.0, + "reward": 6.875, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.013305425643921, + "sampling/importance_sampling_ratio/mean": 0.5245123505592346, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46263813972473145, + "sampling/sampling_logp_difference/mean": 0.030781995505094528, + "step": 137, + "step_time": 18.362532567232847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 542.4375, + "completions/mean_terminated_length": 542.4375, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2741251289844513, + "epoch": 0.276, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41347458958625793, + "kl": 0.020388772361911833, + "learning_rate": 3e-05, + "loss": 0.3297041654586792, + "num_tokens": 1438734.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.6671712398529053, + "sampling/importance_sampling_ratio/mean": 0.822363555431366, + "sampling/importance_sampling_ratio/min": 0.016625043004751205, + "sampling/sampling_logp_difference/max": 0.40987062454223633, + "sampling/sampling_logp_difference/mean": 0.02844768762588501, + "step": 138, + "step_time": 34.91616539563984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 516.3125, + "completions/mean_terminated_length": 516.3125, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.2030403539538383, + "epoch": 0.278, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3896540701389313, + "kl": 0.022598200594075024, + "learning_rate": 3e-05, + "loss": -0.25778308510780334, + "num_tokens": 1448611.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.9001679420471191, + "sampling/importance_sampling_ratio/mean": 0.6815162897109985, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.2820701599121094, + "sampling/sampling_logp_difference/mean": 0.027655858546495438, + "step": 139, + "step_time": 20.81112790806219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 512.8125, + "completions/mean_terminated_length": 512.8125, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "entropy": 1.124063789844513, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2338583916425705, + "kl": 0.022081921808421612, + "learning_rate": 3e-05, + "loss": 0.09288515895605087, + "num_tokens": 1458456.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.7245709896087646, + "sampling/importance_sampling_ratio/mean": 0.7086957693099976, + "sampling/importance_sampling_ratio/min": 0.13776090741157532, + "sampling/sampling_logp_difference/max": 0.4399615526199341, + "sampling/sampling_logp_difference/mean": 0.02669401653110981, + "step": 140, + "step_time": 22.541061893571168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 521.9375, + "completions/mean_terminated_length": 521.9375, + "completions/min_length": 460.0, + "completions/min_terminated_length": 460.0, + "entropy": 1.3581550270318985, + "epoch": 0.282, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1792808622121811, + "kl": 0.02498150523751974, + "learning_rate": 3e-05, + "loss": 0.08992868661880493, + "num_tokens": 1468623.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.6533762216567993, + "sampling/importance_sampling_ratio/mean": 0.5165826082229614, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42975759506225586, + "sampling/sampling_logp_difference/mean": 0.028398238122463226, + "step": 141, + "step_time": 20.935550182592124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 454.625, + "completions/mean_terminated_length": 454.625, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 1.2165675312280655, + "epoch": 0.284, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2593871057033539, + "kl": 0.024267837987281382, + "learning_rate": 3e-05, + "loss": 0.24750135838985443, + "num_tokens": 1477449.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 1.5698224306106567, + "sampling/importance_sampling_ratio/mean": 0.6540807485580444, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3056577444076538, + "sampling/sampling_logp_difference/mean": 0.029166901484131813, + "step": 142, + "step_time": 17.03540184069425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 490.625, + "completions/mean_terminated_length": 490.625, + "completions/min_length": 446.0, + "completions/min_terminated_length": 446.0, + "entropy": 1.1259984448552132, + "epoch": 0.286, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37285444140434265, + "kl": 0.019997276307549328, + "learning_rate": 3e-05, + "loss": 0.330167293548584, + "num_tokens": 1486931.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.156266927719116, + "sampling/importance_sampling_ratio/mean": 0.7264441251754761, + "sampling/importance_sampling_ratio/min": 0.07088703662157059, + "sampling/sampling_logp_difference/max": 0.42168426513671875, + "sampling/sampling_logp_difference/mean": 0.02719968557357788, + "step": 143, + "step_time": 15.82226228993386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 529.0625, + "completions/mean_terminated_length": 496.0666809082031, + "completions/min_length": 421.0, + "completions/min_terminated_length": 421.0, + "entropy": 1.0426596216857433, + "epoch": 0.288, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11882677674293518, + "kl": 0.027060870255809277, + "learning_rate": 3e-05, + "loss": 0.07405021786689758, + "num_tokens": 1496916.0, + "reward": 5.75, + "reward_std": 1.732050895690918, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.732050895690918, + "sampling/importance_sampling_ratio/max": 0.8768613934516907, + "sampling/importance_sampling_ratio/mean": 0.39511895179748535, + "sampling/importance_sampling_ratio/min": 0.11731626838445663, + "sampling/sampling_logp_difference/max": 0.6632819175720215, + "sampling/sampling_logp_difference/mean": 0.02569635957479477, + "step": 144, + "step_time": 21.100794151891023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 502.5625, + "completions/mean_terminated_length": 502.5625, + "completions/min_length": 452.0, + "completions/min_terminated_length": 452.0, + "entropy": 1.2609772458672523, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25876912474632263, + "kl": 0.028287828317843378, + "learning_rate": 3e-05, + "loss": -0.18078479170799255, + "num_tokens": 1507157.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.90565824508667, + "sampling/importance_sampling_ratio/mean": 0.7513852119445801, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3683091402053833, + "sampling/sampling_logp_difference/mean": 0.02768835797905922, + "step": 145, + "step_time": 19.17377450503409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 586.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 478.25, + "completions/mean_terminated_length": 478.25, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "entropy": 1.2076954543590546, + "epoch": 0.292, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4150021970272064, + "kl": 0.027647150680422783, + "learning_rate": 3e-05, + "loss": -0.07925756275653839, + "num_tokens": 1516833.0, + "reward": 6.5, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.0522961616516113, + "sampling/importance_sampling_ratio/mean": 0.6499220132827759, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40656137466430664, + "sampling/sampling_logp_difference/mean": 0.028425993397831917, + "step": 146, + "step_time": 19.426104408223182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 467.1875, + "completions/mean_terminated_length": 467.1875, + "completions/min_length": 396.0, + "completions/min_terminated_length": 396.0, + "entropy": 1.1012553870677948, + "epoch": 0.294, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33721745014190674, + "kl": 0.02999569766689092, + "learning_rate": 3e-05, + "loss": 0.04165569692850113, + "num_tokens": 1526028.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.507693290710449, + "sampling/importance_sampling_ratio/mean": 0.8091086149215698, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4148428440093994, + "sampling/sampling_logp_difference/mean": 0.028098180890083313, + "step": 147, + "step_time": 32.705999514088035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 467.0, + "completions/mean_terminated_length": 467.0, + "completions/min_length": 385.0, + "completions/min_terminated_length": 385.0, + "entropy": 1.2554677203297615, + "epoch": 0.296, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26061347126960754, + "kl": 0.0399768726201728, + "learning_rate": 3e-05, + "loss": -0.018139198422431946, + "num_tokens": 1535348.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.5010173320770264, + "sampling/importance_sampling_ratio/mean": 0.6952720880508423, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35097575187683105, + "sampling/sampling_logp_difference/mean": 0.028018539771437645, + "step": 148, + "step_time": 34.92355508869514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 486.75, + "completions/mean_terminated_length": 486.75, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "entropy": 1.2563373371958733, + "epoch": 0.298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2081325203180313, + "kl": 0.03278218396008015, + "learning_rate": 3e-05, + "loss": -0.01242863480001688, + "num_tokens": 1544912.0, + "reward": 5.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.0925099849700928, + "sampling/importance_sampling_ratio/mean": 0.4945816695690155, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45126640796661377, + "sampling/sampling_logp_difference/mean": 0.030079778283834457, + "step": 149, + "step_time": 26.784944237209857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 571.0, + "completions/max_terminated_length": 571.0, + "completions/mean_length": 483.0625, + "completions/mean_terminated_length": 483.0625, + "completions/min_length": 366.0, + "completions/min_terminated_length": 366.0, + "entropy": 1.1289120316505432, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40016695857048035, + "kl": 0.032314015785232186, + "learning_rate": 3e-05, + "loss": -0.1471974402666092, + "num_tokens": 1554417.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.876359701156616, + "sampling/importance_sampling_ratio/mean": 0.8288668394088745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.438828706741333, + "sampling/sampling_logp_difference/mean": 0.027187082916498184, + "step": 150, + "step_time": 25.687996607273817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 504.25, + "completions/mean_terminated_length": 504.25, + "completions/min_length": 436.0, + "completions/min_terminated_length": 436.0, + "entropy": 1.1718142479658127, + "epoch": 0.302, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3958420157432556, + "kl": 0.02723738143686205, + "learning_rate": 3e-05, + "loss": -0.3493230938911438, + "num_tokens": 1564101.0, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "sampling/importance_sampling_ratio/max": 2.8077433109283447, + "sampling/importance_sampling_ratio/mean": 0.7897872924804688, + "sampling/importance_sampling_ratio/min": 0.06443088501691818, + "sampling/sampling_logp_difference/max": 0.48229074478149414, + "sampling/sampling_logp_difference/mean": 0.02742597460746765, + "step": 151, + "step_time": 34.508475522045046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 519.0, + "completions/max_terminated_length": 519.0, + "completions/mean_length": 296.0, + "completions/mean_terminated_length": 296.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 1.0032543204724789, + "epoch": 0.304, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1944788247346878, + "kl": 0.030508540105074644, + "learning_rate": 3e-05, + "loss": -0.2918842136859894, + "num_tokens": 1570429.0, + "reward": 3.4375, + "reward_std": 3.558440685272217, + "rewards/quality_reward/mean": 3.4375, + "rewards/quality_reward/std": 3.558440685272217, + "sampling/importance_sampling_ratio/max": 1.85885751247406, + "sampling/importance_sampling_ratio/mean": 1.0381181240081787, + "sampling/importance_sampling_ratio/min": 0.1504185050725937, + "sampling/sampling_logp_difference/max": 0.4975461959838867, + "sampling/sampling_logp_difference/mean": 0.02710951678454876, + "step": 152, + "step_time": 22.075861463323236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 514.4375, + "completions/mean_terminated_length": 514.4375, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2780758067965508, + "epoch": 0.306, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4307408928871155, + "kl": 0.02628148818621412, + "learning_rate": 3e-05, + "loss": 0.33224302530288696, + "num_tokens": 1580372.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.5702013969421387, + "sampling/importance_sampling_ratio/mean": 0.8224437832832336, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48117589950561523, + "sampling/sampling_logp_difference/mean": 0.027627088129520416, + "step": 153, + "step_time": 18.07258096849546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 529.0, + "completions/max_terminated_length": 529.0, + "completions/mean_length": 467.125, + "completions/mean_terminated_length": 467.125, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.161174800246954, + "epoch": 0.308, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1959461271762848, + "kl": 0.026262085768394172, + "learning_rate": 3e-05, + "loss": 0.05762449651956558, + "num_tokens": 1589438.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.2318003177642822, + "sampling/importance_sampling_ratio/mean": 0.7094592452049255, + "sampling/importance_sampling_ratio/min": 0.11490790545940399, + "sampling/sampling_logp_difference/max": 0.43965983390808105, + "sampling/sampling_logp_difference/mean": 0.02732112817466259, + "step": 154, + "step_time": 22.3326059714891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 396.875, + "completions/mean_terminated_length": 396.875, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 1.1771309785544872, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4554482400417328, + "kl": 0.03402461623772979, + "learning_rate": 3e-05, + "loss": -0.19043315947055817, + "num_tokens": 1597588.0, + "reward": 5.5625, + "reward_std": 1.3149778842926025, + "rewards/quality_reward/mean": 5.5625, + "rewards/quality_reward/std": 1.3149778842926025, + "sampling/importance_sampling_ratio/max": 2.1980347633361816, + "sampling/importance_sampling_ratio/mean": 0.7672010064125061, + "sampling/importance_sampling_ratio/min": 0.10123267024755478, + "sampling/sampling_logp_difference/max": 0.5363807678222656, + "sampling/sampling_logp_difference/mean": 0.029761571437120438, + "step": 155, + "step_time": 18.508908156771213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 520.5, + "completions/mean_terminated_length": 520.5, + "completions/min_length": 422.0, + "completions/min_terminated_length": 422.0, + "entropy": 1.123583823442459, + "epoch": 0.312, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20946863293647766, + "kl": 0.02810170315206051, + "learning_rate": 3e-05, + "loss": 0.07504862546920776, + "num_tokens": 1607508.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.429632306098938, + "sampling/importance_sampling_ratio/mean": 0.631747305393219, + "sampling/importance_sampling_ratio/min": 0.016474967822432518, + "sampling/sampling_logp_difference/max": 0.40722954273223877, + "sampling/sampling_logp_difference/mean": 0.028051164001226425, + "step": 156, + "step_time": 17.622006124351174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 623.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 471.8125, + "completions/mean_terminated_length": 471.8125, + "completions/min_length": 374.0, + "completions/min_terminated_length": 374.0, + "entropy": 1.0345656536519527, + "epoch": 0.314, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3838019073009491, + "kl": 0.029524097801186144, + "learning_rate": 3e-05, + "loss": 0.29842275381088257, + "num_tokens": 1617121.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.64119291305542, + "sampling/importance_sampling_ratio/mean": 0.5510131120681763, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48647427558898926, + "sampling/sampling_logp_difference/mean": 0.02675374038517475, + "step": 157, + "step_time": 34.35223956173286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 479.3125, + "completions/mean_terminated_length": 479.3125, + "completions/min_length": 415.0, + "completions/min_terminated_length": 415.0, + "entropy": 1.197593629360199, + "epoch": 0.316, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14107273519039154, + "kl": 0.02758750319480896, + "learning_rate": 3e-05, + "loss": 0.07440078258514404, + "num_tokens": 1626462.0, + "reward": 6.1875, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.719329833984375, + "sampling/importance_sampling_ratio/mean": 0.515890896320343, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37659645080566406, + "sampling/sampling_logp_difference/mean": 0.02727513015270233, + "step": 158, + "step_time": 30.49356387415901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 465.4375, + "completions/mean_terminated_length": 465.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.182745985686779, + "epoch": 0.318, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22129040956497192, + "kl": 0.02928700065240264, + "learning_rate": 3e-05, + "loss": 0.0027256053872406483, + "num_tokens": 1635613.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.2551939487457275, + "sampling/importance_sampling_ratio/mean": 0.4655284583568573, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3532288074493408, + "sampling/sampling_logp_difference/mean": 0.027347734197974205, + "step": 159, + "step_time": 23.563114238902926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 480.9375, + "completions/mean_terminated_length": 480.9375, + "completions/min_length": 378.0, + "completions/min_terminated_length": 378.0, + "entropy": 1.0412059053778648, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22958238422870636, + "kl": 0.025641236337833107, + "learning_rate": 3e-05, + "loss": -0.1119004413485527, + "num_tokens": 1645060.0, + "reward": 5.8125, + "reward_std": 2.9033026695251465, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 2.9033026695251465, + "sampling/importance_sampling_ratio/max": 2.3590943813323975, + "sampling/importance_sampling_ratio/mean": 0.6600984334945679, + "sampling/importance_sampling_ratio/min": 0.16755303740501404, + "sampling/sampling_logp_difference/max": 0.3485531806945801, + "sampling/sampling_logp_difference/mean": 0.02425791323184967, + "step": 160, + "step_time": 17.564059282653034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 511.5625, + "completions/mean_terminated_length": 511.5625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.0806752033531666, + "epoch": 0.322, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2202390879392624, + "kl": 0.03178418125025928, + "learning_rate": 3e-05, + "loss": -0.24258574843406677, + "num_tokens": 1654901.0, + "reward": 6.5, + "reward_std": 0.9660918116569519, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.9660918116569519, + "sampling/importance_sampling_ratio/max": 2.616337776184082, + "sampling/importance_sampling_ratio/mean": 0.8924014568328857, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37401676177978516, + "sampling/sampling_logp_difference/mean": 0.02693682350218296, + "step": 161, + "step_time": 22.71096785319969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 484.1875, + "completions/mean_terminated_length": 484.1875, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.2182030156254768, + "epoch": 0.324, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2978271245956421, + "kl": 0.026724245748482645, + "learning_rate": 3e-05, + "loss": -0.0895138755440712, + "num_tokens": 1664568.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.0902533531188965, + "sampling/importance_sampling_ratio/mean": 0.9070306420326233, + "sampling/importance_sampling_ratio/min": 0.20267778635025024, + "sampling/sampling_logp_difference/max": 0.3564906120300293, + "sampling/sampling_logp_difference/mean": 0.02701719105243683, + "step": 162, + "step_time": 24.52080715773627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 543.0, + "completions/max_terminated_length": 543.0, + "completions/mean_length": 506.9375, + "completions/mean_terminated_length": 506.9375, + "completions/min_length": 452.0, + "completions/min_terminated_length": 452.0, + "entropy": 1.2944460734724998, + "epoch": 0.326, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13530702888965607, + "kl": 0.03180140187032521, + "learning_rate": 3e-05, + "loss": 0.016086462885141373, + "num_tokens": 1674423.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.1556015014648438, + "sampling/importance_sampling_ratio/mean": 0.583191990852356, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.1959445476531982, + "sampling/sampling_logp_difference/mean": 0.028113799169659615, + "step": 163, + "step_time": 21.659780140966177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 525.0, + "completions/max_terminated_length": 525.0, + "completions/mean_length": 490.5625, + "completions/mean_terminated_length": 490.5625, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.289131723344326, + "epoch": 0.328, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5087841153144836, + "kl": 0.026518055819906294, + "learning_rate": 3e-05, + "loss": 0.2851857542991638, + "num_tokens": 1683864.0, + "reward": 6.5, + "reward_std": 1.0327955484390259, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.0327955484390259, + "sampling/importance_sampling_ratio/max": 2.531486988067627, + "sampling/importance_sampling_ratio/mean": 0.8939677476882935, + "sampling/importance_sampling_ratio/min": 0.09362189471721649, + "sampling/sampling_logp_difference/max": 0.38115930557250977, + "sampling/sampling_logp_difference/mean": 0.028977636247873306, + "step": 164, + "step_time": 22.67840038659051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 570.0, + "completions/max_terminated_length": 570.0, + "completions/mean_length": 508.5, + "completions/mean_terminated_length": 508.5, + "completions/min_length": 465.0, + "completions/min_terminated_length": 465.0, + "entropy": 1.2583990097045898, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5895075798034668, + "kl": 0.03340678755193949, + "learning_rate": 3e-05, + "loss": 0.5024052858352661, + "num_tokens": 1693592.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.5420279502868652, + "sampling/importance_sampling_ratio/mean": 0.8359720706939697, + "sampling/importance_sampling_ratio/min": 0.13951139152050018, + "sampling/sampling_logp_difference/max": 0.25212621688842773, + "sampling/sampling_logp_difference/mean": 0.027791393920779228, + "step": 165, + "step_time": 23.054075544700027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 522.625, + "completions/mean_terminated_length": 522.625, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.2723611742258072, + "epoch": 0.332, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4107528030872345, + "kl": 0.028830039431340992, + "learning_rate": 3e-05, + "loss": 0.43730953335762024, + "num_tokens": 1703722.0, + "reward": 7.25, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 1.8613929748535156, + "sampling/importance_sampling_ratio/mean": 0.5325981378555298, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33945512771606445, + "sampling/sampling_logp_difference/mean": 0.028407979756593704, + "step": 166, + "step_time": 14.466566514223814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 485.4375, + "completions/mean_terminated_length": 485.4375, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.159641794860363, + "epoch": 0.334, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23576849699020386, + "kl": 0.03369407169520855, + "learning_rate": 3e-05, + "loss": -0.17192532122135162, + "num_tokens": 1713129.0, + "reward": 6.9375, + "reward_std": 0.8539125919342041, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.8539125919342041, + "sampling/importance_sampling_ratio/max": 2.364237070083618, + "sampling/importance_sampling_ratio/mean": 0.835480809211731, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4001603126525879, + "sampling/sampling_logp_difference/mean": 0.027760744094848633, + "step": 167, + "step_time": 15.427942908834666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 531.125, + "completions/mean_terminated_length": 531.125, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.1074568964540958, + "epoch": 0.336, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14523518085479736, + "kl": 0.032465216936543584, + "learning_rate": 3e-05, + "loss": -0.10911832749843597, + "num_tokens": 1723371.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.233501672744751, + "sampling/importance_sampling_ratio/mean": 0.5449534058570862, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42547130584716797, + "sampling/sampling_logp_difference/mean": 0.026400625705718994, + "step": 168, + "step_time": 14.74156094249338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 629.0, + "completions/max_terminated_length": 629.0, + "completions/mean_length": 518.25, + "completions/mean_terminated_length": 518.25, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.2571639120578766, + "epoch": 0.338, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3119359314441681, + "kl": 0.03448521322570741, + "learning_rate": 3e-05, + "loss": 0.14656513929367065, + "num_tokens": 1733279.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.882887601852417, + "sampling/importance_sampling_ratio/mean": 0.8524343967437744, + "sampling/importance_sampling_ratio/min": 0.061056625097990036, + "sampling/sampling_logp_difference/max": 0.34301328659057617, + "sampling/sampling_logp_difference/mean": 0.027896685525774956, + "step": 169, + "step_time": 22.03652939805761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 552.8125, + "completions/mean_terminated_length": 552.8125, + "completions/min_length": 454.0, + "completions/min_terminated_length": 454.0, + "entropy": 1.2771715074777603, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23340070247650146, + "kl": 0.03460722556337714, + "learning_rate": 3e-05, + "loss": -0.11407067626714706, + "num_tokens": 1743740.0, + "reward": 6.8125, + "reward_std": 0.6551081538200378, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.6551081538200378, + "sampling/importance_sampling_ratio/max": 1.6628351211547852, + "sampling/importance_sampling_ratio/mean": 0.5497220754623413, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.314577579498291, + "sampling/sampling_logp_difference/mean": 0.03021315485239029, + "step": 170, + "step_time": 18.62061845092103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 564.0, + "completions/max_terminated_length": 564.0, + "completions/mean_length": 495.1875, + "completions/mean_terminated_length": 495.1875, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "entropy": 1.1425480283796787, + "epoch": 0.342, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07674646377563477, + "kl": 0.0316173325991258, + "learning_rate": 3e-05, + "loss": -0.012545892037451267, + "num_tokens": 1753231.0, + "reward": 6.4375, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.37643563747406, + "sampling/importance_sampling_ratio/mean": 0.4176323413848877, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5454483032226562, + "sampling/sampling_logp_difference/mean": 0.027837729081511497, + "step": 171, + "step_time": 35.69278695946559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 536.5, + "completions/mean_terminated_length": 536.5, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "entropy": 1.3311709240078926, + "epoch": 0.344, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5406383275985718, + "kl": 0.03873496490996331, + "learning_rate": 3e-05, + "loss": 0.2167063057422638, + "num_tokens": 1763511.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.74458646774292, + "sampling/importance_sampling_ratio/mean": 1.1003804206848145, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5733523368835449, + "sampling/sampling_logp_difference/mean": 0.028938323259353638, + "step": 172, + "step_time": 19.092736863531172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 508.0625, + "completions/mean_terminated_length": 508.0625, + "completions/min_length": 437.0, + "completions/min_terminated_length": 437.0, + "entropy": 1.22428647428751, + "epoch": 0.346, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11082098633050919, + "kl": 0.03200511261820793, + "learning_rate": 3e-05, + "loss": 0.07203174382448196, + "num_tokens": 1773304.0, + "reward": 7.125, + "reward_std": 0.5, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8160909414291382, + "sampling/importance_sampling_ratio/mean": 0.5755537748336792, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.595893383026123, + "sampling/sampling_logp_difference/mean": 0.028249088674783707, + "step": 173, + "step_time": 36.1855756030418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 720.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 556.9375, + "completions/mean_terminated_length": 556.9375, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.1718761064112186, + "epoch": 0.348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3865050971508026, + "kl": 0.03591357555706054, + "learning_rate": 3e-05, + "loss": 0.2996499538421631, + "num_tokens": 1784039.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.0816619396209717, + "sampling/importance_sampling_ratio/mean": 0.5800511240959167, + "sampling/importance_sampling_ratio/min": 0.09306051582098007, + "sampling/sampling_logp_difference/max": 0.41143274307250977, + "sampling/sampling_logp_difference/mean": 0.027585921809077263, + "step": 174, + "step_time": 20.269209342077374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 494.6875, + "completions/mean_terminated_length": 494.6875, + "completions/min_length": 442.0, + "completions/min_terminated_length": 442.0, + "entropy": 1.3337711468338966, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11888998746871948, + "kl": 0.03842530632391572, + "learning_rate": 3e-05, + "loss": 0.07849398255348206, + "num_tokens": 1793682.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.3145290613174438, + "sampling/importance_sampling_ratio/mean": 0.4274219870567322, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4447822570800781, + "sampling/sampling_logp_difference/mean": 0.031242625787854195, + "step": 175, + "step_time": 17.293509372044355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 544.8125, + "completions/mean_terminated_length": 544.8125, + "completions/min_length": 462.0, + "completions/min_terminated_length": 462.0, + "entropy": 1.19626072794199, + "epoch": 0.352, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2991865277290344, + "kl": 0.036185722798109055, + "learning_rate": 3e-05, + "loss": 0.11461115628480911, + "num_tokens": 1804039.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5985729694366455, + "sampling/importance_sampling_ratio/mean": 0.5072454810142517, + "sampling/importance_sampling_ratio/min": 0.07339605689048767, + "sampling/sampling_logp_difference/max": 0.3649592399597168, + "sampling/sampling_logp_difference/mean": 0.026338135823607445, + "step": 176, + "step_time": 40.05168053135276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 488.0625, + "completions/mean_terminated_length": 488.0625, + "completions/min_length": 419.0, + "completions/min_terminated_length": 419.0, + "entropy": 1.2391329184174538, + "epoch": 0.354, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21207627654075623, + "kl": 0.036609378294087946, + "learning_rate": 3e-05, + "loss": 0.11777876317501068, + "num_tokens": 1813440.0, + "reward": 7.0, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.3352530002593994, + "sampling/importance_sampling_ratio/mean": 0.6533275246620178, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35143423080444336, + "sampling/sampling_logp_difference/mean": 0.027743803337216377, + "step": 177, + "step_time": 33.59382761735469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 570.0, + "completions/max_terminated_length": 570.0, + "completions/mean_length": 495.125, + "completions/mean_terminated_length": 495.125, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "entropy": 1.3244052603840828, + "epoch": 0.356, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2076549232006073, + "kl": 0.03601150680333376, + "learning_rate": 3e-05, + "loss": -0.20823253691196442, + "num_tokens": 1823018.0, + "reward": 7.0, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.9768388271331787, + "sampling/importance_sampling_ratio/mean": 0.8220031261444092, + "sampling/importance_sampling_ratio/min": 0.08218635618686676, + "sampling/sampling_logp_difference/max": 0.3826625347137451, + "sampling/sampling_logp_difference/mean": 0.02992408722639084, + "step": 178, + "step_time": 19.288791641127318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 497.875, + "completions/mean_terminated_length": 497.875, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.2238815650343895, + "epoch": 0.358, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5341953039169312, + "kl": 0.0400471081957221, + "learning_rate": 3e-05, + "loss": 0.06042468547821045, + "num_tokens": 1832736.0, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.1878349781036377, + "sampling/importance_sampling_ratio/mean": 1.0708422660827637, + "sampling/importance_sampling_ratio/min": 0.037978146225214005, + "sampling/sampling_logp_difference/max": 0.5151598453521729, + "sampling/sampling_logp_difference/mean": 0.029129499569535255, + "step": 179, + "step_time": 17.113372664432973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 541.6875, + "completions/mean_terminated_length": 541.6875, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.4256544262170792, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21081708371639252, + "kl": 0.03922082995995879, + "learning_rate": 3e-05, + "loss": -0.10235662013292313, + "num_tokens": 1843203.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 1.1632962226867676, + "sampling/importance_sampling_ratio/mean": 0.4484874904155731, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31549549102783203, + "sampling/sampling_logp_difference/mean": 0.02854372188448906, + "step": 180, + "step_time": 38.8069160906598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 541.0, + "completions/mean_terminated_length": 541.0, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.2127383947372437, + "epoch": 0.362, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28472819924354553, + "kl": 0.039078159374184906, + "learning_rate": 3e-05, + "loss": -0.1838480830192566, + "num_tokens": 1853555.0, + "reward": 6.1875, + "reward_std": 0.6551081538200378, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.6551081538200378, + "sampling/importance_sampling_ratio/max": 1.6971478462219238, + "sampling/importance_sampling_ratio/mean": 0.5436820983886719, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4628629684448242, + "sampling/sampling_logp_difference/mean": 0.028904814273118973, + "step": 181, + "step_time": 22.927347922697663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 521.1875, + "completions/mean_terminated_length": 521.1875, + "completions/min_length": 481.0, + "completions/min_terminated_length": 481.0, + "entropy": 1.205168604850769, + "epoch": 0.364, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40580859780311584, + "kl": 0.03652556415181607, + "learning_rate": 3e-05, + "loss": 0.11645574867725372, + "num_tokens": 1863670.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.6977338790893555, + "sampling/importance_sampling_ratio/mean": 0.7627977132797241, + "sampling/importance_sampling_ratio/min": 0.0672435387969017, + "sampling/sampling_logp_difference/max": 0.42972230911254883, + "sampling/sampling_logp_difference/mean": 0.02727590501308441, + "step": 182, + "step_time": 17.11851307330653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 501.6875, + "completions/mean_terminated_length": 501.6875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2342532575130463, + "epoch": 0.366, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30436721444129944, + "kl": 0.04035243031103164, + "learning_rate": 3e-05, + "loss": 0.07254824787378311, + "num_tokens": 1873353.0, + "reward": 7.0, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 1.7811214923858643, + "sampling/importance_sampling_ratio/mean": 0.6635246276855469, + "sampling/importance_sampling_ratio/min": 0.05660989508032799, + "sampling/sampling_logp_difference/max": 0.3192565441131592, + "sampling/sampling_logp_difference/mean": 0.028735067695379257, + "step": 183, + "step_time": 15.842315018642694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 540.625, + "completions/mean_terminated_length": 540.625, + "completions/min_length": 488.0, + "completions/min_terminated_length": 488.0, + "entropy": 1.3501724749803543, + "epoch": 0.368, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15960462391376495, + "kl": 0.04095173126552254, + "learning_rate": 3e-05, + "loss": -0.020260833203792572, + "num_tokens": 1883907.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 1.9560747146606445, + "sampling/importance_sampling_ratio/mean": 0.6561183929443359, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.9810755252838135, + "sampling/sampling_logp_difference/mean": 0.029474109411239624, + "step": 184, + "step_time": 27.30614952277392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 532.6875, + "completions/mean_terminated_length": 532.6875, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "entropy": 1.3397118523716927, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19153976440429688, + "kl": 0.045232257107272744, + "learning_rate": 3e-05, + "loss": 0.07327698916196823, + "num_tokens": 1894262.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 2.3196122646331787, + "sampling/importance_sampling_ratio/mean": 0.7404133677482605, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.378201961517334, + "sampling/sampling_logp_difference/mean": 0.02841799519956112, + "step": 185, + "step_time": 17.66303364513442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 626.0, + "completions/max_terminated_length": 626.0, + "completions/mean_length": 547.875, + "completions/mean_terminated_length": 547.875, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.4480287805199623, + "epoch": 0.372, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1921214759349823, + "kl": 0.04523745132610202, + "learning_rate": 3e-05, + "loss": -0.1904258131980896, + "num_tokens": 1904748.0, + "reward": 6.1875, + "reward_std": 1.0468206405639648, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.0468206405639648, + "sampling/importance_sampling_ratio/max": 2.1734814643859863, + "sampling/importance_sampling_ratio/mean": 0.8759132027626038, + "sampling/importance_sampling_ratio/min": 0.1473371982574463, + "sampling/sampling_logp_difference/max": 0.43239259719848633, + "sampling/sampling_logp_difference/mean": 0.030228231102228165, + "step": 186, + "step_time": 15.321936973370612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 522.5, + "completions/mean_terminated_length": 522.5, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.1799098625779152, + "epoch": 0.374, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1446281522512436, + "kl": 0.04069687286391854, + "learning_rate": 3e-05, + "loss": -0.005613957531750202, + "num_tokens": 1915044.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1428344249725342, + "sampling/importance_sampling_ratio/mean": 0.49870407581329346, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3580789566040039, + "sampling/sampling_logp_difference/mean": 0.028958076611161232, + "step": 187, + "step_time": 20.772348938975483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 526.6875, + "completions/mean_terminated_length": 526.6875, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.1814791783690453, + "epoch": 0.376, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09093533456325531, + "kl": 0.048393386183306575, + "learning_rate": 3e-05, + "loss": -0.09858276695013046, + "num_tokens": 1925055.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.5353080034255981, + "sampling/importance_sampling_ratio/mean": 0.47519102692604065, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.455765962600708, + "sampling/sampling_logp_difference/mean": 0.02906900830566883, + "step": 188, + "step_time": 19.42380119021982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 663.0, + "completions/max_terminated_length": 663.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 538.5, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.2316770479083061, + "epoch": 0.378, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20003275573253632, + "kl": 0.04744360945187509, + "learning_rate": 3e-05, + "loss": -0.16722381114959717, + "num_tokens": 1935415.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.9816064834594727, + "sampling/importance_sampling_ratio/mean": 0.680183470249176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4760274887084961, + "sampling/sampling_logp_difference/mean": 0.028748787939548492, + "step": 189, + "step_time": 19.73181858472526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 555.125, + "completions/mean_terminated_length": 555.125, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.198552466928959, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13213400542736053, + "kl": 0.04299823543988168, + "learning_rate": 3e-05, + "loss": 0.04805057868361473, + "num_tokens": 1945985.0, + "reward": 6.625, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 1.563953161239624, + "sampling/importance_sampling_ratio/mean": 0.41058292984962463, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3399984836578369, + "sampling/sampling_logp_difference/mean": 0.0274125337600708, + "step": 190, + "step_time": 17.826407154556364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 684.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 539.25, + "completions/mean_terminated_length": 539.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.179109387099743, + "epoch": 0.382, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2910110354423523, + "kl": 0.046097030164673924, + "learning_rate": 3e-05, + "loss": -0.3787975311279297, + "num_tokens": 1956445.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.4134271144866943, + "sampling/importance_sampling_ratio/mean": 0.8701735734939575, + "sampling/importance_sampling_ratio/min": 0.1316290944814682, + "sampling/sampling_logp_difference/max": 0.3755226135253906, + "sampling/sampling_logp_difference/mean": 0.029267774894833565, + "step": 191, + "step_time": 24.135659996420145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 540.0, + "completions/mean_terminated_length": 540.0, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.2047618329524994, + "epoch": 0.384, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20911987125873566, + "kl": 0.04525213362649083, + "learning_rate": 3e-05, + "loss": 0.013828473165631294, + "num_tokens": 1966693.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 1.9282022714614868, + "sampling/importance_sampling_ratio/mean": 0.5034524202346802, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4313013553619385, + "sampling/sampling_logp_difference/mean": 0.02878478728234768, + "step": 192, + "step_time": 14.677200393751264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 597.0, + "completions/max_terminated_length": 597.0, + "completions/mean_length": 553.125, + "completions/mean_terminated_length": 553.125, + "completions/min_length": 501.0, + "completions/min_terminated_length": 501.0, + "entropy": 1.1857876032590866, + "epoch": 0.386, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3986797034740448, + "kl": 0.04699963750317693, + "learning_rate": 3e-05, + "loss": -0.06190801039338112, + "num_tokens": 1977311.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.877324104309082, + "sampling/importance_sampling_ratio/mean": 0.9780403971672058, + "sampling/importance_sampling_ratio/min": 0.02624017745256424, + "sampling/sampling_logp_difference/max": 0.7719376087188721, + "sampling/sampling_logp_difference/mean": 0.02950127050280571, + "step": 193, + "step_time": 22.189579842612147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 531.5625, + "completions/mean_terminated_length": 531.5625, + "completions/min_length": 465.0, + "completions/min_terminated_length": 465.0, + "entropy": 1.3334204703569412, + "epoch": 0.388, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22821271419525146, + "kl": 0.04149021068587899, + "learning_rate": 3e-05, + "loss": -0.15963155031204224, + "num_tokens": 1987680.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 2.790942907333374, + "sampling/importance_sampling_ratio/mean": 0.7382668256759644, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45777153968811035, + "sampling/sampling_logp_difference/mean": 0.029790451750159264, + "step": 194, + "step_time": 21.228061076253653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 505.0625, + "completions/mean_terminated_length": 505.0625, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.2570307329297066, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3617555797100067, + "kl": 0.042452862951904535, + "learning_rate": 3e-05, + "loss": -0.2362610548734665, + "num_tokens": 1997481.0, + "reward": 6.625, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.3454530239105225, + "sampling/importance_sampling_ratio/mean": 0.9099248647689819, + "sampling/importance_sampling_ratio/min": 0.09889467060565948, + "sampling/sampling_logp_difference/max": 0.6245463490486145, + "sampling/sampling_logp_difference/mean": 0.029685894027352333, + "step": 195, + "step_time": 17.880568680819124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 544.6875, + "completions/mean_terminated_length": 544.6875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.208221659064293, + "epoch": 0.392, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35804808139801025, + "kl": 0.03823408088646829, + "learning_rate": 3e-05, + "loss": 0.026877164840698242, + "num_tokens": 2007860.0, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 2.670585870742798, + "sampling/importance_sampling_ratio/mean": 0.6493271589279175, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4234185218811035, + "sampling/sampling_logp_difference/mean": 0.02899312786757946, + "step": 196, + "step_time": 20.456977635622025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 582.875, + "completions/mean_terminated_length": 582.875, + "completions/min_length": 517.0, + "completions/min_terminated_length": 517.0, + "entropy": 1.3732100576162338, + "epoch": 0.394, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.05589874088764191, + "kl": 0.03486072865780443, + "learning_rate": 3e-05, + "loss": -0.019679736346006393, + "num_tokens": 2018946.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1570472717285156, + "sampling/importance_sampling_ratio/mean": 0.35759687423706055, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33884429931640625, + "sampling/sampling_logp_difference/mean": 0.029303058981895447, + "step": 197, + "step_time": 16.522779263556004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 533.5, + "completions/mean_terminated_length": 533.5, + "completions/min_length": 479.0, + "completions/min_terminated_length": 479.0, + "entropy": 1.1664377599954605, + "epoch": 0.396, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23638494312763214, + "kl": 0.037777561345137656, + "learning_rate": 3e-05, + "loss": 0.0447416789829731, + "num_tokens": 2029178.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.484117865562439, + "sampling/importance_sampling_ratio/mean": 0.539449155330658, + "sampling/importance_sampling_ratio/min": 0.09831889718770981, + "sampling/sampling_logp_difference/max": 0.37561988830566406, + "sampling/sampling_logp_difference/mean": 0.029459550976753235, + "step": 198, + "step_time": 30.065524043980986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 536.0625, + "completions/mean_terminated_length": 536.0625, + "completions/min_length": 472.0, + "completions/min_terminated_length": 472.0, + "entropy": 1.254080481827259, + "epoch": 0.398, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1923029124736786, + "kl": 0.03572200902272016, + "learning_rate": 3e-05, + "loss": -0.09766081720590591, + "num_tokens": 2039347.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.6362762451171875, + "sampling/importance_sampling_ratio/mean": 0.5547572374343872, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35316991806030273, + "sampling/sampling_logp_difference/mean": 0.027488065883517265, + "step": 199, + "step_time": 30.96936017088592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 551.5, + "completions/mean_terminated_length": 551.5, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.2724409252405167, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22953365743160248, + "kl": 0.036497756373137236, + "learning_rate": 3e-05, + "loss": 0.014221633784472942, + "num_tokens": 2049795.0, + "reward": 6.5625, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.4539027214050293, + "sampling/importance_sampling_ratio/mean": 0.6678089499473572, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7396450042724609, + "sampling/sampling_logp_difference/mean": 0.0300765261054039, + "step": 200, + "step_time": 16.165886579081416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 582.0, + "completions/max_terminated_length": 582.0, + "completions/mean_length": 528.0, + "completions/mean_terminated_length": 528.0, + "completions/min_length": 474.0, + "completions/min_terminated_length": 474.0, + "entropy": 1.120336215943098, + "epoch": 0.402, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0991974025964737, + "kl": 0.03168696933425963, + "learning_rate": 3e-05, + "loss": -0.13335926830768585, + "num_tokens": 2059963.0, + "reward": 6.375, + "reward_std": 1.7841898202896118, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.7841898202896118, + "sampling/importance_sampling_ratio/max": 2.3520584106445312, + "sampling/importance_sampling_ratio/mean": 0.6451135277748108, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5111579895019531, + "sampling/sampling_logp_difference/mean": 0.027531839907169342, + "step": 201, + "step_time": 38.76227374607697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 635.0, + "completions/max_terminated_length": 635.0, + "completions/mean_length": 549.9375, + "completions/mean_terminated_length": 549.9375, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.2026560828089714, + "epoch": 0.404, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07962016016244888, + "kl": 0.040914483717642725, + "learning_rate": 3e-05, + "loss": 0.04318992793560028, + "num_tokens": 2070626.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.161137580871582, + "sampling/importance_sampling_ratio/mean": 0.265199214220047, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42658185958862305, + "sampling/sampling_logp_difference/mean": 0.0287276990711689, + "step": 202, + "step_time": 20.940971142146736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 566.0, + "completions/max_terminated_length": 566.0, + "completions/mean_length": 511.1875, + "completions/mean_terminated_length": 511.1875, + "completions/min_length": 460.0, + "completions/min_terminated_length": 460.0, + "entropy": 1.2832268327474594, + "epoch": 0.406, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1376999020576477, + "kl": 0.03862898051738739, + "learning_rate": 3e-05, + "loss": -0.14832699298858643, + "num_tokens": 2080861.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.544222831726074, + "sampling/importance_sampling_ratio/mean": 0.7174543142318726, + "sampling/importance_sampling_ratio/min": 0.18541352450847626, + "sampling/sampling_logp_difference/max": 0.3160414695739746, + "sampling/sampling_logp_difference/mean": 0.02949603646993637, + "step": 203, + "step_time": 20.54771270370111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 605.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 524.25, + "completions/mean_terminated_length": 524.25, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.2392274662852287, + "epoch": 0.408, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12836101651191711, + "kl": 0.032396848779171705, + "learning_rate": 3e-05, + "loss": -0.010008644312620163, + "num_tokens": 2090953.0, + "reward": 7.125, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.204956531524658, + "sampling/importance_sampling_ratio/mean": 0.5054515600204468, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.29064249992370605, + "sampling/sampling_logp_difference/mean": 0.028712285682559013, + "step": 204, + "step_time": 18.479188771452755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 708.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 566.625, + "completions/mean_terminated_length": 566.625, + "completions/min_length": 482.0, + "completions/min_terminated_length": 482.0, + "entropy": 1.3167504370212555, + "epoch": 0.41, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2566336691379547, + "kl": 0.03499211696907878, + "learning_rate": 3e-05, + "loss": 0.053824737668037415, + "num_tokens": 2101795.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2913551330566406, + "sampling/importance_sampling_ratio/mean": 0.6769458651542664, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4542531967163086, + "sampling/sampling_logp_difference/mean": 0.027578134089708328, + "step": 205, + "step_time": 17.109014553949237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 506.25, + "completions/mean_terminated_length": 506.25, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "entropy": 1.244155466556549, + "epoch": 0.412, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3224847912788391, + "kl": 0.03367950115352869, + "learning_rate": 3e-05, + "loss": -0.2610609829425812, + "num_tokens": 2111719.0, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.0974948406219482, + "sampling/importance_sampling_ratio/mean": 0.5962464809417725, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5845310688018799, + "sampling/sampling_logp_difference/mean": 0.02760414592921734, + "step": 206, + "step_time": 12.972559538204223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 623.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 515.125, + "completions/mean_terminated_length": 515.125, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.2021623700857162, + "epoch": 0.414, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22504572570323944, + "kl": 0.039928025915287435, + "learning_rate": 3e-05, + "loss": 0.08153954148292542, + "num_tokens": 2121913.0, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 1.6359574794769287, + "sampling/importance_sampling_ratio/mean": 0.4843714237213135, + "sampling/importance_sampling_ratio/min": 0.16781684756278992, + "sampling/sampling_logp_difference/max": 0.3475990295410156, + "sampling/sampling_logp_difference/mean": 0.029437636956572533, + "step": 207, + "step_time": 43.64637131197378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 613.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 512.3125, + "completions/mean_terminated_length": 512.3125, + "completions/min_length": 421.0, + "completions/min_terminated_length": 421.0, + "entropy": 1.2001312859356403, + "epoch": 0.416, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3592093884944916, + "kl": 0.036205250886268914, + "learning_rate": 3e-05, + "loss": 0.34191110730171204, + "num_tokens": 2131806.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.192653179168701, + "sampling/importance_sampling_ratio/mean": 0.6557403206825256, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3398885726928711, + "sampling/sampling_logp_difference/mean": 0.027714602649211884, + "step": 208, + "step_time": 14.901265816297382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 513.1875, + "completions/mean_terminated_length": 513.1875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2848069965839386, + "epoch": 0.418, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.47909337282180786, + "kl": 0.02966410096269101, + "learning_rate": 3e-05, + "loss": 0.2796367406845093, + "num_tokens": 2141849.0, + "reward": 6.1875, + "reward_std": 1.0468206405639648, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.0468206405639648, + "sampling/importance_sampling_ratio/max": 2.964437484741211, + "sampling/importance_sampling_ratio/mean": 0.48602545261383057, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3722946047782898, + "sampling/sampling_logp_difference/mean": 0.029922205954790115, + "step": 209, + "step_time": 26.673682311549783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 507.9375, + "completions/mean_terminated_length": 507.9375, + "completions/min_length": 422.0, + "completions/min_terminated_length": 422.0, + "entropy": 1.2064250856637955, + "epoch": 0.42, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3124609887599945, + "kl": 0.038097753771580756, + "learning_rate": 3e-05, + "loss": 0.07772707939147949, + "num_tokens": 2151864.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.737311840057373, + "sampling/importance_sampling_ratio/mean": 0.9012110233306885, + "sampling/importance_sampling_ratio/min": 0.052471309900283813, + "sampling/sampling_logp_difference/max": 0.36536455154418945, + "sampling/sampling_logp_difference/mean": 0.027899259701371193, + "step": 210, + "step_time": 23.180102336220443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 525.0, + "completions/max_terminated_length": 525.0, + "completions/mean_length": 480.9375, + "completions/mean_terminated_length": 480.9375, + "completions/min_length": 436.0, + "completions/min_terminated_length": 436.0, + "entropy": 1.2063737213611603, + "epoch": 0.422, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.44815266132354736, + "kl": 0.027290108264423907, + "learning_rate": 3e-05, + "loss": 0.28776273131370544, + "num_tokens": 2161039.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.063380479812622, + "sampling/importance_sampling_ratio/mean": 0.8898900151252747, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.32479190826416016, + "sampling/sampling_logp_difference/mean": 0.02677021361887455, + "step": 211, + "step_time": 22.177836938295513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 599.0, + "completions/max_terminated_length": 599.0, + "completions/mean_length": 504.625, + "completions/mean_terminated_length": 504.625, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.3087149783968925, + "epoch": 0.424, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35702478885650635, + "kl": 0.031013125786557794, + "learning_rate": 3e-05, + "loss": 0.17483392357826233, + "num_tokens": 2170961.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.4756224155426025, + "sampling/importance_sampling_ratio/mean": 0.7680925130844116, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35749173164367676, + "sampling/sampling_logp_difference/mean": 0.027991417795419693, + "step": 212, + "step_time": 17.408967671450227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 797.0, + "completions/max_terminated_length": 797.0, + "completions/mean_length": 640.8125, + "completions/mean_terminated_length": 640.8125, + "completions/min_length": 457.0, + "completions/min_terminated_length": 457.0, + "entropy": 1.3978670835494995, + "epoch": 0.426, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29086264967918396, + "kl": 0.024261576938442886, + "learning_rate": 3e-05, + "loss": 0.09740053862333298, + "num_tokens": 2182910.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 1.893927812576294, + "sampling/importance_sampling_ratio/mean": 0.7117372155189514, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3868746757507324, + "sampling/sampling_logp_difference/mean": 0.028743820264935493, + "step": 213, + "step_time": 23.464720248244703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 486.75, + "completions/mean_terminated_length": 486.75, + "completions/min_length": 404.0, + "completions/min_terminated_length": 404.0, + "entropy": 1.2355968467891216, + "epoch": 0.428, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2767268419265747, + "kl": 0.02811512805055827, + "learning_rate": 3e-05, + "loss": 0.18954241275787354, + "num_tokens": 2192242.0, + "reward": 7.0625, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.0872268676757812, + "sampling/importance_sampling_ratio/mean": 0.8847656846046448, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3631856441497803, + "sampling/sampling_logp_difference/mean": 0.02797355316579342, + "step": 214, + "step_time": 15.314252337440848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 503.1875, + "completions/mean_terminated_length": 503.1875, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.2383001297712326, + "epoch": 0.43, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1873309463262558, + "kl": 0.03220478829462081, + "learning_rate": 3e-05, + "loss": -0.01179824024438858, + "num_tokens": 2202045.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 2.573594808578491, + "sampling/importance_sampling_ratio/mean": 0.7044014930725098, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6166388988494873, + "sampling/sampling_logp_difference/mean": 0.028071925044059753, + "step": 215, + "step_time": 43.65747703285888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 565.0, + "completions/max_terminated_length": 565.0, + "completions/mean_length": 503.9375, + "completions/mean_terminated_length": 503.9375, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.1677803546190262, + "epoch": 0.432, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09634320437908173, + "kl": 0.03325538453646004, + "learning_rate": 3e-05, + "loss": 0.03834616392850876, + "num_tokens": 2212188.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.9774657487869263, + "sampling/importance_sampling_ratio/mean": 0.5414069890975952, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35487866401672363, + "sampling/sampling_logp_difference/mean": 0.027339771389961243, + "step": 216, + "step_time": 17.274593455251306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 650.0, + "completions/max_terminated_length": 650.0, + "completions/mean_length": 343.75, + "completions/mean_terminated_length": 343.75, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.0582842603325844, + "epoch": 0.434, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3425586521625519, + "kl": 0.03517636051401496, + "learning_rate": 3e-05, + "loss": 0.11844412982463837, + "num_tokens": 2219440.0, + "reward": 3.5, + "reward_std": 3.265986442565918, + "rewards/quality_reward/mean": 3.5, + "rewards/quality_reward/std": 3.265986442565918, + "sampling/importance_sampling_ratio/max": 1.4673620462417603, + "sampling/importance_sampling_ratio/mean": 0.781898021697998, + "sampling/importance_sampling_ratio/min": 0.08477991074323654, + "sampling/sampling_logp_difference/max": 0.3881380558013916, + "sampling/sampling_logp_difference/mean": 0.02829045243561268, + "step": 217, + "step_time": 13.36990327714011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 471.4375, + "completions/mean_terminated_length": 471.4375, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "entropy": 1.1179756224155426, + "epoch": 0.436, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33612945675849915, + "kl": 0.030886436812579632, + "learning_rate": 3e-05, + "loss": 0.006120521575212479, + "num_tokens": 2228975.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.1587650775909424, + "sampling/importance_sampling_ratio/mean": 0.7267376780509949, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.32509779930114746, + "sampling/sampling_logp_difference/mean": 0.02783289924263954, + "step": 218, + "step_time": 17.99441510764882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 520.125, + "completions/mean_terminated_length": 520.125, + "completions/min_length": 462.0, + "completions/min_terminated_length": 462.0, + "entropy": 1.2453451082110405, + "epoch": 0.438, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2976357340812683, + "kl": 0.03020153450779617, + "learning_rate": 3e-05, + "loss": 0.1648026406764984, + "num_tokens": 2238953.0, + "reward": 5.3125, + "reward_std": 1.078192949295044, + "rewards/quality_reward/mean": 5.3125, + "rewards/quality_reward/std": 1.078192949295044, + "sampling/importance_sampling_ratio/max": 1.6686924695968628, + "sampling/importance_sampling_ratio/mean": 0.7682108879089355, + "sampling/importance_sampling_ratio/min": 0.06810324639081955, + "sampling/sampling_logp_difference/max": 0.3546750545501709, + "sampling/sampling_logp_difference/mean": 0.028889676555991173, + "step": 219, + "step_time": 20.669593635946512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 552.0, + "completions/max_terminated_length": 552.0, + "completions/mean_length": 465.375, + "completions/mean_terminated_length": 465.375, + "completions/min_length": 356.0, + "completions/min_terminated_length": 356.0, + "entropy": 1.3331433907151222, + "epoch": 0.44, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13799847662448883, + "kl": 0.03191920532844961, + "learning_rate": 3e-05, + "loss": -0.133737251162529, + "num_tokens": 2248199.0, + "reward": 6.0, + "reward_std": 0.9660918116569519, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.9660918116569519, + "sampling/importance_sampling_ratio/max": 2.8963325023651123, + "sampling/importance_sampling_ratio/mean": 0.5178577899932861, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35057830810546875, + "sampling/sampling_logp_difference/mean": 0.027748603373765945, + "step": 220, + "step_time": 21.38788841944188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 507.5, + "completions/mean_terminated_length": 507.5, + "completions/min_length": 474.0, + "completions/min_terminated_length": 474.0, + "entropy": 1.241542100906372, + "epoch": 0.442, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3648691773414612, + "kl": 0.03521239408291876, + "learning_rate": 3e-05, + "loss": 0.2634640634059906, + "num_tokens": 2258327.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.1063547134399414, + "sampling/importance_sampling_ratio/mean": 0.6709499359130859, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3389136791229248, + "sampling/sampling_logp_difference/mean": 0.02875763736665249, + "step": 221, + "step_time": 30.28709344472736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 459.375, + "completions/mean_terminated_length": 459.375, + "completions/min_length": 405.0, + "completions/min_terminated_length": 405.0, + "entropy": 1.141789611428976, + "epoch": 0.444, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1954081803560257, + "kl": 0.03689368430059403, + "learning_rate": 3e-05, + "loss": 0.13035088777542114, + "num_tokens": 2267381.0, + "reward": 6.5625, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.681690216064453, + "sampling/importance_sampling_ratio/mean": 0.6949984431266785, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31617891788482666, + "sampling/sampling_logp_difference/mean": 0.02665363810956478, + "step": 222, + "step_time": 37.694539529737085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 571.0, + "completions/max_terminated_length": 571.0, + "completions/mean_length": 515.5625, + "completions/mean_terminated_length": 515.5625, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2330311760306358, + "epoch": 0.446, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27568548917770386, + "kl": 0.032993816188536584, + "learning_rate": 3e-05, + "loss": 0.017291374504566193, + "num_tokens": 2277446.0, + "reward": 7.125, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.847236156463623, + "sampling/importance_sampling_ratio/mean": 0.7807494401931763, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.49445104598999023, + "sampling/sampling_logp_difference/mean": 0.027980424463748932, + "step": 223, + "step_time": 18.57380611775443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 634.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 523.1875, + "completions/mean_terminated_length": 523.1875, + "completions/min_length": 451.0, + "completions/min_terminated_length": 451.0, + "entropy": 1.0975877195596695, + "epoch": 0.448, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10739865154027939, + "kl": 0.02909247507341206, + "learning_rate": 3e-05, + "loss": 0.05642539635300636, + "num_tokens": 2287345.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.9240283966064453, + "sampling/importance_sampling_ratio/mean": 0.6011937856674194, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40827369689941406, + "sampling/sampling_logp_difference/mean": 0.027437299489974976, + "step": 224, + "step_time": 39.087660535704345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 533.0, + "completions/max_terminated_length": 533.0, + "completions/mean_length": 478.25, + "completions/mean_terminated_length": 478.25, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.3173525258898735, + "epoch": 0.45, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22121335566043854, + "kl": 0.03540586424060166, + "learning_rate": 3e-05, + "loss": -0.02710402011871338, + "num_tokens": 2296741.0, + "reward": 6.4375, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.8824238777160645, + "sampling/importance_sampling_ratio/mean": 0.6062434315681458, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4316587448120117, + "sampling/sampling_logp_difference/mean": 0.02776467800140381, + "step": 225, + "step_time": 26.200199087150395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 499.9375, + "completions/mean_terminated_length": 499.9375, + "completions/min_length": 448.0, + "completions/min_terminated_length": 448.0, + "entropy": 1.1501530036330223, + "epoch": 0.452, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3865528702735901, + "kl": 0.03646970179397613, + "learning_rate": 3e-05, + "loss": 0.19160562753677368, + "num_tokens": 2306676.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 2.6609690189361572, + "sampling/importance_sampling_ratio/mean": 0.9618603587150574, + "sampling/importance_sampling_ratio/min": 0.0957244485616684, + "sampling/sampling_logp_difference/max": 0.35040283203125, + "sampling/sampling_logp_difference/mean": 0.028557972982525826, + "step": 226, + "step_time": 17.752630488947034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 510.75, + "completions/mean_terminated_length": 510.75, + "completions/min_length": 419.0, + "completions/min_terminated_length": 419.0, + "entropy": 1.190872348845005, + "epoch": 0.454, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1407802700996399, + "kl": 0.030792692443355918, + "learning_rate": 3e-05, + "loss": 0.04028765484690666, + "num_tokens": 2316696.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 1.2131233215332031, + "sampling/importance_sampling_ratio/mean": 0.6032290458679199, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3711676597595215, + "sampling/sampling_logp_difference/mean": 0.027024609968066216, + "step": 227, + "step_time": 27.143527323380113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 495.625, + "completions/mean_terminated_length": 495.625, + "completions/min_length": 419.0, + "completions/min_terminated_length": 419.0, + "entropy": 1.2335442155599594, + "epoch": 0.456, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18564435839653015, + "kl": 0.025341857108287513, + "learning_rate": 3e-05, + "loss": -0.26227492094039917, + "num_tokens": 2326322.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.640001058578491, + "sampling/importance_sampling_ratio/mean": 0.6159635782241821, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.34752869606018066, + "sampling/sampling_logp_difference/mean": 0.027370886877179146, + "step": 228, + "step_time": 22.369792928919196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 528.0, + "completions/max_terminated_length": 528.0, + "completions/mean_length": 487.6875, + "completions/mean_terminated_length": 487.6875, + "completions/min_length": 449.0, + "completions/min_terminated_length": 449.0, + "entropy": 1.2502131089568138, + "epoch": 0.458, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2313818782567978, + "kl": 0.03635518567170948, + "learning_rate": 3e-05, + "loss": -0.031097467988729477, + "num_tokens": 2335973.0, + "reward": 6.9375, + "reward_std": 0.8539125919342041, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.8539125919342041, + "sampling/importance_sampling_ratio/max": 2.227858543395996, + "sampling/importance_sampling_ratio/mean": 0.7488094568252563, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3627934455871582, + "sampling/sampling_logp_difference/mean": 0.02758944220840931, + "step": 229, + "step_time": 30.443659604527056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 476.375, + "completions/mean_terminated_length": 476.375, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 1.2348124124109745, + "epoch": 0.46, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5150085687637329, + "kl": 0.032692725653760135, + "learning_rate": 3e-05, + "loss": 0.5464498996734619, + "num_tokens": 2345579.0, + "reward": 6.8125, + "reward_std": 0.6551081538200378, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.6551081538200378, + "sampling/importance_sampling_ratio/max": 2.376497268676758, + "sampling/importance_sampling_ratio/mean": 0.7721551060676575, + "sampling/importance_sampling_ratio/min": 0.1553211510181427, + "sampling/sampling_logp_difference/max": 0.42708492279052734, + "sampling/sampling_logp_difference/mean": 0.026741618290543556, + "step": 230, + "step_time": 24.71390812145546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 517.5, + "completions/mean_terminated_length": 517.5, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "entropy": 1.1868174076080322, + "epoch": 0.462, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11509153991937637, + "kl": 0.03900455019902438, + "learning_rate": 3e-05, + "loss": -0.1646902710199356, + "num_tokens": 2355499.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2603800296783447, + "sampling/importance_sampling_ratio/mean": 0.6012319326400757, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4672989845275879, + "sampling/sampling_logp_difference/mean": 0.027583574876189232, + "step": 231, + "step_time": 23.276649605948478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 691.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 560.875, + "completions/mean_terminated_length": 560.875, + "completions/min_length": 494.0, + "completions/min_terminated_length": 494.0, + "entropy": 1.1533633023500443, + "epoch": 0.464, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17417575418949127, + "kl": 0.031763071776367724, + "learning_rate": 3e-05, + "loss": 0.00344286416657269, + "num_tokens": 2366049.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.206153154373169, + "sampling/importance_sampling_ratio/mean": 0.740157961845398, + "sampling/importance_sampling_ratio/min": 0.13027621805667877, + "sampling/sampling_logp_difference/max": 0.4202132225036621, + "sampling/sampling_logp_difference/mean": 0.028019659221172333, + "step": 232, + "step_time": 20.686087283305824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 500.8125, + "completions/mean_terminated_length": 500.8125, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.2008480802178383, + "epoch": 0.466, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22023369371891022, + "kl": 0.03284288931172341, + "learning_rate": 3e-05, + "loss": 0.03899282589554787, + "num_tokens": 2375582.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.0323734283447266, + "sampling/importance_sampling_ratio/mean": 0.6224058270454407, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4571092128753662, + "sampling/sampling_logp_difference/mean": 0.027327092364430428, + "step": 233, + "step_time": 26.555491346865892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 845.0, + "completions/max_terminated_length": 845.0, + "completions/mean_length": 587.5, + "completions/mean_terminated_length": 587.5, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.2540696933865547, + "epoch": 0.468, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13053849339485168, + "kl": 0.03163444856181741, + "learning_rate": 3e-05, + "loss": -0.06800927221775055, + "num_tokens": 2386702.0, + "reward": 6.0, + "reward_std": 1.7511900663375854, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.7511900663375854, + "sampling/importance_sampling_ratio/max": 1.4020758867263794, + "sampling/importance_sampling_ratio/mean": 0.4879741370677948, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5877337455749512, + "sampling/sampling_logp_difference/mean": 0.02704041451215744, + "step": 234, + "step_time": 34.3772664074786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 506.375, + "completions/mean_terminated_length": 506.375, + "completions/min_length": 422.0, + "completions/min_terminated_length": 422.0, + "entropy": 1.3376594334840775, + "epoch": 0.47, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1838671714067459, + "kl": 0.03747367626056075, + "learning_rate": 3e-05, + "loss": -0.1597842425107956, + "num_tokens": 2396564.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.9093716144561768, + "sampling/importance_sampling_ratio/mean": 0.7693536281585693, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37027931213378906, + "sampling/sampling_logp_difference/mean": 0.029480738565325737, + "step": 235, + "step_time": 14.542957273777574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 605.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 509.6875, + "completions/mean_terminated_length": 509.6875, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "entropy": 1.3714017421007156, + "epoch": 0.472, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.44443222880363464, + "kl": 0.03565627557691187, + "learning_rate": 3e-05, + "loss": 0.24737706780433655, + "num_tokens": 2406551.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.7392280101776123, + "sampling/importance_sampling_ratio/mean": 0.7888213396072388, + "sampling/importance_sampling_ratio/min": 0.04889443516731262, + "sampling/sampling_logp_difference/max": 0.29999852180480957, + "sampling/sampling_logp_difference/mean": 0.028956102207303047, + "step": 236, + "step_time": 19.966124589089304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 541.1875, + "completions/mean_terminated_length": 541.1875, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.199029415845871, + "epoch": 0.474, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1285122185945511, + "kl": 0.03105375764425844, + "learning_rate": 3e-05, + "loss": -0.006456274073570967, + "num_tokens": 2416754.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 1.395982265472412, + "sampling/importance_sampling_ratio/mean": 0.5822510719299316, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4087851047515869, + "sampling/sampling_logp_difference/mean": 0.027147701010107994, + "step": 237, + "step_time": 31.801921805832535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 772.0, + "completions/max_terminated_length": 772.0, + "completions/mean_length": 598.1875, + "completions/mean_terminated_length": 598.1875, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.2089053243398666, + "epoch": 0.476, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3568721413612366, + "kl": 0.03257777914404869, + "learning_rate": 3e-05, + "loss": 0.3960018455982208, + "num_tokens": 2428005.0, + "reward": 6.25, + "reward_std": 1.1254628896713257, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.1254628896713257, + "sampling/importance_sampling_ratio/max": 1.9572224617004395, + "sampling/importance_sampling_ratio/mean": 0.5545582175254822, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42554450035095215, + "sampling/sampling_logp_difference/mean": 0.027511969208717346, + "step": 238, + "step_time": 25.979049060028046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 580.0, + "completions/max_terminated_length": 580.0, + "completions/mean_length": 508.875, + "completions/mean_terminated_length": 508.875, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.4715757966041565, + "epoch": 0.478, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07262199372053146, + "kl": 0.03327966062352061, + "learning_rate": 3e-05, + "loss": 0.025357680395245552, + "num_tokens": 2437779.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.7364466190338135, + "sampling/importance_sampling_ratio/mean": 0.8696970343589783, + "sampling/importance_sampling_ratio/min": 0.07304152101278305, + "sampling/sampling_logp_difference/max": 0.37055206298828125, + "sampling/sampling_logp_difference/mean": 0.029029743745923042, + "step": 239, + "step_time": 25.994311626069248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 580.0, + "completions/max_terminated_length": 580.0, + "completions/mean_length": 527.9375, + "completions/mean_terminated_length": 527.9375, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "entropy": 1.311545416712761, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28445860743522644, + "kl": 0.03817834367509931, + "learning_rate": 3e-05, + "loss": 0.12445695698261261, + "num_tokens": 2448202.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.700259804725647, + "sampling/importance_sampling_ratio/mean": 0.5472592711448669, + "sampling/importance_sampling_ratio/min": 0.052414167672395706, + "sampling/sampling_logp_difference/max": 0.5368318557739258, + "sampling/sampling_logp_difference/mean": 0.02976268343627453, + "step": 240, + "step_time": 22.84421144844964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 598.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 543.3125, + "completions/mean_terminated_length": 543.3125, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.180833749473095, + "epoch": 0.482, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16590596735477448, + "kl": 0.03646332467906177, + "learning_rate": 3e-05, + "loss": 0.04229091852903366, + "num_tokens": 2458743.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.728398323059082, + "sampling/importance_sampling_ratio/mean": 0.9457916021347046, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40506887435913086, + "sampling/sampling_logp_difference/mean": 0.02782438136637211, + "step": 241, + "step_time": 25.389156353194267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 579.0, + "completions/max_terminated_length": 579.0, + "completions/mean_length": 502.3125, + "completions/mean_terminated_length": 502.3125, + "completions/min_length": 454.0, + "completions/min_terminated_length": 454.0, + "entropy": 1.1694707348942757, + "epoch": 0.484, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25544801354408264, + "kl": 0.03445987973827869, + "learning_rate": 3e-05, + "loss": -0.017443601042032242, + "num_tokens": 2468492.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 1.8124310970306396, + "sampling/importance_sampling_ratio/mean": 0.8106446266174316, + "sampling/importance_sampling_ratio/min": 0.08132059127092361, + "sampling/sampling_logp_difference/max": 0.6171557903289795, + "sampling/sampling_logp_difference/mean": 0.027156969532370567, + "step": 242, + "step_time": 24.112746777944267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 654.0, + "completions/max_terminated_length": 654.0, + "completions/mean_length": 538.125, + "completions/mean_terminated_length": 538.125, + "completions/min_length": 448.0, + "completions/min_terminated_length": 448.0, + "entropy": 1.297831729054451, + "epoch": 0.486, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17728154361248016, + "kl": 0.03608768491540104, + "learning_rate": 3e-05, + "loss": -0.030910439789295197, + "num_tokens": 2478878.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.7654427289962769, + "sampling/importance_sampling_ratio/mean": 0.5417827367782593, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3394327163696289, + "sampling/sampling_logp_difference/mean": 0.02802959457039833, + "step": 243, + "step_time": 39.02764433948323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 703.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 564.75, + "completions/mean_terminated_length": 564.75, + "completions/min_length": 449.0, + "completions/min_terminated_length": 449.0, + "entropy": 1.2894479781389236, + "epoch": 0.488, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1783561259508133, + "kl": 0.04178670607507229, + "learning_rate": 3e-05, + "loss": 0.010581104084849358, + "num_tokens": 2489938.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 1.9127941131591797, + "sampling/importance_sampling_ratio/mean": 0.6222037672996521, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.8026522397994995, + "sampling/sampling_logp_difference/mean": 0.028042398393154144, + "step": 244, + "step_time": 22.52857542503625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 599.0, + "completions/max_terminated_length": 599.0, + "completions/mean_length": 511.875, + "completions/mean_terminated_length": 511.875, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.1273594908416271, + "epoch": 0.49, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34433481097221375, + "kl": 0.04671380412764847, + "learning_rate": 3e-05, + "loss": 0.06864061206579208, + "num_tokens": 2499760.0, + "reward": 7.1875, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.993313789367676, + "sampling/importance_sampling_ratio/mean": 0.9394024014472961, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3724935054779053, + "sampling/sampling_logp_difference/mean": 0.027506975457072258, + "step": 245, + "step_time": 24.86254589399323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 684.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 533.0, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 467.0, + "completions/min_terminated_length": 467.0, + "entropy": 1.2337471172213554, + "epoch": 0.492, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2073342353105545, + "kl": 0.03919998917263001, + "learning_rate": 3e-05, + "loss": -0.00970650464296341, + "num_tokens": 2510176.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.6556100845336914, + "sampling/importance_sampling_ratio/mean": 0.6703793406486511, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5199446678161621, + "sampling/sampling_logp_difference/mean": 0.028828633949160576, + "step": 246, + "step_time": 27.641962222289294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 636.0, + "completions/max_terminated_length": 636.0, + "completions/mean_length": 555.8125, + "completions/mean_terminated_length": 555.8125, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.1313174478709698, + "epoch": 0.494, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10448901355266571, + "kl": 0.043822019128128886, + "learning_rate": 3e-05, + "loss": -0.060149889439344406, + "num_tokens": 2520981.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 1.087956428527832, + "sampling/importance_sampling_ratio/mean": 0.4310106039047241, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36969757080078125, + "sampling/sampling_logp_difference/mean": 0.028163518756628036, + "step": 247, + "step_time": 27.198071955237538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 533.1875, + "completions/mean_terminated_length": 533.1875, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.2226731404662132, + "epoch": 0.496, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21327541768550873, + "kl": 0.0436656444799155, + "learning_rate": 3e-05, + "loss": -0.15169084072113037, + "num_tokens": 2531056.0, + "reward": 6.4375, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.145639657974243, + "sampling/importance_sampling_ratio/mean": 0.924071729183197, + "sampling/importance_sampling_ratio/min": 0.042581744492053986, + "sampling/sampling_logp_difference/max": 0.5983643531799316, + "sampling/sampling_logp_difference/mean": 0.028493624180555344, + "step": 248, + "step_time": 46.37140509998426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 675.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 580.5625, + "completions/mean_terminated_length": 580.5625, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "entropy": 1.2647478878498077, + "epoch": 0.498, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1687636375427246, + "kl": 0.03778462728951126, + "learning_rate": 3e-05, + "loss": 0.024922871962189674, + "num_tokens": 2542129.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.1956820487976074, + "sampling/importance_sampling_ratio/mean": 0.6349776983261108, + "sampling/importance_sampling_ratio/min": 0.11655592173337936, + "sampling/sampling_logp_difference/max": 0.35921406745910645, + "sampling/sampling_logp_difference/mean": 0.029055560007691383, + "step": 249, + "step_time": 34.322586783673614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 694.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 543.6875, + "completions/mean_terminated_length": 543.6875, + "completions/min_length": 451.0, + "completions/min_terminated_length": 451.0, + "entropy": 1.2015386000275612, + "epoch": 0.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3467749357223511, + "kl": 0.03919053066056222, + "learning_rate": 3e-05, + "loss": -0.24095430970191956, + "num_tokens": 2552412.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.5754284858703613, + "sampling/importance_sampling_ratio/mean": 0.9581880569458008, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3446807861328125, + "sampling/sampling_logp_difference/mean": 0.028353629633784294, + "step": 250, + "step_time": 26.97987106954679 + } + ], + "logging_steps": 1, + "max_steps": 300, + "num_input_tokens_seen": 2552412, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/outputs/E0-baseline/checkpoint-250/training_args.bin b/outputs/E0-baseline/checkpoint-250/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c3a2693682d14c331dbd12e8872552fc28d54b49 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-250/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6 +size 7697 diff --git a/outputs/E0-baseline/checkpoint-300/README.md b/outputs/E0-baseline/checkpoint-300/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88990785a4695219513364eafae195070ddd3182 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-300/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507 +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.20.0 \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-300/adapter_config.json b/outputs/E0-baseline/checkpoint-300/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3b383bef044f277e3ff79f0fc5addabd3c441593 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-300/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.20.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-300/adapter_model.safetensors b/outputs/E0-baseline/checkpoint-300/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eef8d99a1c73429a073cd00c931eb0288438ca32 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-300/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f546b7b6fd17327ebb9554c32f1730033ed34378b532ee5fe945489c3a60734 +size 264308896 diff --git a/outputs/E0-baseline/checkpoint-300/chat_template.jinja b/outputs/E0-baseline/checkpoint-300/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-300/chat_template.jinja @@ -0,0 +1,61 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-300/tokenizer.json b/outputs/E0-baseline/checkpoint-300/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-300/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/outputs/E0-baseline/checkpoint-300/tokenizer_config.json b/outputs/E0-baseline/checkpoint-300/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..86d67326516569b4f1cec3ac7b51b8903062c648 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-300/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/outputs/E0-baseline/checkpoint-300/trainer_state.json b/outputs/E0-baseline/checkpoint-300/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c67dd5dfa74a2dedaef720df00460409562e589d --- /dev/null +++ b/outputs/E0-baseline/checkpoint-300/trainer_state.json @@ -0,0 +1,9934 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6, + "eval_steps": 500, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 529.0, + "completions/mean_terminated_length": 529.0, + "completions/min_length": 482.0, + "completions/min_terminated_length": 482.0, + "entropy": 1.2025159299373627, + "epoch": 0.002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22668755054473877, + "kl": 0.0, + "learning_rate": 0.0, + "loss": 0.2398601919412613, + "num_tokens": 10400.0, + "reward": 6.375, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6477458477020264, + "sampling/importance_sampling_ratio/mean": 0.5064857602119446, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40894174575805664, + "sampling/sampling_logp_difference/mean": 0.026567919179797173, + "step": 1, + "step_time": 12.760562099982053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 644.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 562.25, + "completions/mean_terminated_length": 562.25, + "completions/min_length": 497.0, + "completions/min_terminated_length": 497.0, + "entropy": 1.1930748969316483, + "epoch": 0.004, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12967805564403534, + "kl": 0.0, + "learning_rate": 3e-06, + "loss": -0.08571265637874603, + "num_tokens": 20924.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 1.0037232637405396, + "sampling/importance_sampling_ratio/mean": 0.41275694966316223, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45699238777160645, + "sampling/sampling_logp_difference/mean": 0.025086138397455215, + "step": 2, + "step_time": 11.406366533134133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 483.625, + "completions/mean_terminated_length": 483.625, + "completions/min_length": 407.0, + "completions/min_terminated_length": 407.0, + "entropy": 1.1096689626574516, + "epoch": 0.006, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22966289520263672, + "kl": 0.0008355328354809899, + "learning_rate": 6e-06, + "loss": 0.020913563668727875, + "num_tokens": 30222.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.927120566368103, + "sampling/importance_sampling_ratio/mean": 0.6114993095397949, + "sampling/importance_sampling_ratio/min": 0.11067161709070206, + "sampling/sampling_logp_difference/max": 0.4620504379272461, + "sampling/sampling_logp_difference/mean": 0.024864500388503075, + "step": 3, + "step_time": 26.480680393986404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 608.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 524.75, + "completions/mean_terminated_length": 524.75, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.211122527718544, + "epoch": 0.008, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30378466844558716, + "kl": 0.0008403196770814247, + "learning_rate": 9e-06, + "loss": -0.12959149479866028, + "num_tokens": 40410.0, + "reward": 6.25, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.406888961791992, + "sampling/importance_sampling_ratio/mean": 0.5457419753074646, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3680229187011719, + "sampling/sampling_logp_difference/mean": 0.02656388282775879, + "step": 4, + "step_time": 22.95301443943754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 487.5625, + "completions/mean_terminated_length": 487.5625, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "entropy": 1.247180238366127, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5199307799339294, + "kl": 0.0008723233368073124, + "learning_rate": 1.2e-05, + "loss": 0.11524428427219391, + "num_tokens": 49915.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.1544158458709717, + "sampling/importance_sampling_ratio/mean": 0.963020920753479, + "sampling/importance_sampling_ratio/min": 0.04948288947343826, + "sampling/sampling_logp_difference/max": 0.4774587154388428, + "sampling/sampling_logp_difference/mean": 0.02625642716884613, + "step": 5, + "step_time": 22.37928077671677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 533.0, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 394.0, + "completions/min_terminated_length": 394.0, + "entropy": 1.1693861335515976, + "epoch": 0.012, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27643197774887085, + "kl": 0.0009261858467652928, + "learning_rate": 1.5e-05, + "loss": 0.15093231201171875, + "num_tokens": 60219.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 2.125091791152954, + "sampling/importance_sampling_ratio/mean": 0.7733402252197266, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7049552202224731, + "sampling/sampling_logp_difference/mean": 0.025986071676015854, + "step": 6, + "step_time": 21.00841654697433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 505.6875, + "completions/mean_terminated_length": 505.6875, + "completions/min_length": 425.0, + "completions/min_terminated_length": 425.0, + "entropy": 1.2473183795809746, + "epoch": 0.014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2166663259267807, + "kl": 0.0009701631606731098, + "learning_rate": 1.8e-05, + "loss": -0.08582288026809692, + "num_tokens": 69902.0, + "reward": 7.1875, + "reward_std": 0.75, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.8287386894226074, + "sampling/importance_sampling_ratio/mean": 0.5286832451820374, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.38030898571014404, + "sampling/sampling_logp_difference/mean": 0.0264718160033226, + "step": 7, + "step_time": 46.596127359196544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.1040107272565365, + "epoch": 0.016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18874908983707428, + "kl": 0.0010721117541834246, + "learning_rate": 2.1e-05, + "loss": -0.1946130096912384, + "num_tokens": 79501.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.3561792373657227, + "sampling/importance_sampling_ratio/mean": 0.6918502449989319, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.26114892959594727, + "sampling/sampling_logp_difference/mean": 0.02554757334291935, + "step": 8, + "step_time": 25.433595282491297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 554.0, + "completions/max_terminated_length": 554.0, + "completions/mean_length": 490.1875, + "completions/mean_terminated_length": 490.1875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1377170644700527, + "epoch": 0.018, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.204215869307518, + "kl": 0.002002388624532614, + "learning_rate": 2.4e-05, + "loss": -0.08130021393299103, + "num_tokens": 88976.0, + "reward": 6.5, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.0982444286346436, + "sampling/importance_sampling_ratio/mean": 0.5873216986656189, + "sampling/importance_sampling_ratio/min": 0.04890051856637001, + "sampling/sampling_logp_difference/max": 0.8512144088745117, + "sampling/sampling_logp_difference/mean": 0.02638406865298748, + "step": 9, + "step_time": 18.427699581719935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 500.1875, + "completions/mean_terminated_length": 500.1875, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.1230391450226307, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1985069215297699, + "kl": 0.0026735300780273974, + "learning_rate": 2.7000000000000002e-05, + "loss": -0.12387816607952118, + "num_tokens": 98603.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.983199954032898, + "sampling/importance_sampling_ratio/mean": 0.5639468431472778, + "sampling/importance_sampling_ratio/min": 0.012905921787023544, + "sampling/sampling_logp_difference/max": 0.3653905391693115, + "sampling/sampling_logp_difference/mean": 0.025383003056049347, + "step": 10, + "step_time": 14.99981931829825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 459.6875, + "completions/mean_terminated_length": 459.6875, + "completions/min_length": 379.0, + "completions/min_terminated_length": 379.0, + "entropy": 1.213625729084015, + "epoch": 0.022, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3695620596408844, + "kl": 0.00424640768324025, + "learning_rate": 3e-05, + "loss": -0.06913074851036072, + "num_tokens": 107734.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.000136375427246, + "sampling/importance_sampling_ratio/mean": 0.8144867420196533, + "sampling/importance_sampling_ratio/min": 0.06302778422832489, + "sampling/sampling_logp_difference/max": 0.3647327423095703, + "sampling/sampling_logp_difference/mean": 0.026674197986721992, + "step": 11, + "step_time": 30.46549107739702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.191277615725994, + "epoch": 0.024, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35457733273506165, + "kl": 0.007200263120466843, + "learning_rate": 3e-05, + "loss": 0.22097699344158173, + "num_tokens": 117199.0, + "reward": 5.9375, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 1.5660414695739746, + "sampling/importance_sampling_ratio/mean": 0.649204432964325, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3921785354614258, + "sampling/sampling_logp_difference/mean": 0.02726689912378788, + "step": 12, + "step_time": 15.719243939965963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 464.3125, + "completions/mean_terminated_length": 464.3125, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "entropy": 1.24251464381814, + "epoch": 0.026, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30047014355659485, + "kl": 0.0058551667898427695, + "learning_rate": 3e-05, + "loss": -0.07746122777462006, + "num_tokens": 126556.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6028401851654053, + "sampling/importance_sampling_ratio/mean": 0.7561360597610474, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4946432113647461, + "sampling/sampling_logp_difference/mean": 0.02639186754822731, + "step": 13, + "step_time": 18.08984712138772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 444.8125, + "completions/mean_terminated_length": 444.8125, + "completions/min_length": 389.0, + "completions/min_terminated_length": 389.0, + "entropy": 1.1501125395298004, + "epoch": 0.028, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24105942249298096, + "kl": 0.012796793889719993, + "learning_rate": 3e-05, + "loss": 0.10855278372764587, + "num_tokens": 135417.0, + "reward": 3.1875, + "reward_std": 3.310966730117798, + "rewards/quality_reward/mean": 3.1875, + "rewards/quality_reward/std": 3.310966730117798, + "sampling/importance_sampling_ratio/max": 2.541518211364746, + "sampling/importance_sampling_ratio/mean": 0.5903321504592896, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.8390979766845703, + "sampling/sampling_logp_difference/mean": 0.02838001400232315, + "step": 14, + "step_time": 20.055794408079237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 497.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 442.25, + "completions/mean_terminated_length": 442.25, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 1.1262825280427933, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19040776789188385, + "kl": 0.010701361010433175, + "learning_rate": 3e-05, + "loss": -0.007966680452227592, + "num_tokens": 144205.0, + "reward": 5.875, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.0754293203353882, + "sampling/importance_sampling_ratio/mean": 0.41446638107299805, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3982200622558594, + "sampling/sampling_logp_difference/mean": 0.027210108935832977, + "step": 15, + "step_time": 14.176074750721455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 478.125, + "completions/mean_terminated_length": 478.125, + "completions/min_length": 433.0, + "completions/min_terminated_length": 433.0, + "entropy": 1.2985924184322357, + "epoch": 0.032, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23625652492046356, + "kl": 0.0213887135614641, + "learning_rate": 3e-05, + "loss": -0.21546132862567902, + "num_tokens": 153543.0, + "reward": 6.5, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 2.0074336528778076, + "sampling/importance_sampling_ratio/mean": 0.49076417088508606, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5095968246459961, + "sampling/sampling_logp_difference/mean": 0.028833162039518356, + "step": 16, + "step_time": 15.848205763846636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 469.25, + "completions/mean_terminated_length": 469.25, + "completions/min_length": 423.0, + "completions/min_terminated_length": 423.0, + "entropy": 1.3148910626769066, + "epoch": 0.034, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17765119671821594, + "kl": 0.02128879475640133, + "learning_rate": 3e-05, + "loss": -0.0828079879283905, + "num_tokens": 163043.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 1.5988941192626953, + "sampling/importance_sampling_ratio/mean": 0.5021570324897766, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.591062068939209, + "sampling/sampling_logp_difference/mean": 0.030804751440882683, + "step": 17, + "step_time": 28.313011147081852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 447.0, + "completions/mean_terminated_length": 447.0, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 1.390664003789425, + "epoch": 0.036, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37244123220443726, + "kl": 0.019650122558232397, + "learning_rate": 3e-05, + "loss": -0.01184748113155365, + "num_tokens": 172379.0, + "reward": 6.0, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.675238847732544, + "sampling/importance_sampling_ratio/mean": 0.9581233263015747, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5762512683868408, + "sampling/sampling_logp_difference/mean": 0.03126702085137367, + "step": 18, + "step_time": 30.991567107848823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 498.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 447.75, + "completions/mean_terminated_length": 447.75, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 1.3287223279476166, + "epoch": 0.038, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23734751343727112, + "kl": 0.022738213243428618, + "learning_rate": 3e-05, + "loss": 0.040024783462285995, + "num_tokens": 181239.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.095722198486328, + "sampling/importance_sampling_ratio/mean": 0.6408629417419434, + "sampling/importance_sampling_ratio/min": 0.1203346848487854, + "sampling/sampling_logp_difference/max": 0.4722297191619873, + "sampling/sampling_logp_difference/mean": 0.030378391966223717, + "step": 19, + "step_time": 18.615950418636203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 452.3125, + "completions/mean_terminated_length": 452.3125, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 1.1001618690788746, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34637194871902466, + "kl": 0.015380491153337061, + "learning_rate": 3e-05, + "loss": 0.1691148728132248, + "num_tokens": 190068.0, + "reward": 6.125, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.6566128730773926, + "sampling/importance_sampling_ratio/mean": 0.7767290472984314, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.6531033515930176, + "sampling/sampling_logp_difference/mean": 0.030463142320513725, + "step": 20, + "step_time": 15.741292077116668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 457.5625, + "completions/mean_terminated_length": 457.5625, + "completions/min_length": 390.0, + "completions/min_terminated_length": 390.0, + "entropy": 1.3708399310708046, + "epoch": 0.042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2514922618865967, + "kl": 0.018277494702488184, + "learning_rate": 3e-05, + "loss": -0.13425321877002716, + "num_tokens": 198941.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.5612297058105469, + "sampling/importance_sampling_ratio/mean": 0.601022481918335, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4818992614746094, + "sampling/sampling_logp_difference/mean": 0.0315740592777729, + "step": 21, + "step_time": 17.24192419089377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 421.4375, + "completions/mean_terminated_length": 421.4375, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 1.136269599199295, + "epoch": 0.044, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2268020063638687, + "kl": 0.017973962530959398, + "learning_rate": 3e-05, + "loss": 0.010622119531035423, + "num_tokens": 207300.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.7986358404159546, + "sampling/importance_sampling_ratio/mean": 0.46136727929115295, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5335149765014648, + "sampling/sampling_logp_difference/mean": 0.02804401144385338, + "step": 22, + "step_time": 19.37282825494185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 452.6875, + "completions/mean_terminated_length": 452.6875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3299130946397781, + "epoch": 0.046, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33994877338409424, + "kl": 0.021804221265483648, + "learning_rate": 3e-05, + "loss": -0.24404363334178925, + "num_tokens": 216343.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.452249526977539, + "sampling/importance_sampling_ratio/mean": 0.747193455696106, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6328549385070801, + "sampling/sampling_logp_difference/mean": 0.02918298915028572, + "step": 23, + "step_time": 15.356728344224393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 462.4375, + "completions/mean_terminated_length": 462.4375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.379701942205429, + "epoch": 0.048, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3076202869415283, + "kl": 0.020299295720178634, + "learning_rate": 3e-05, + "loss": -0.09123071283102036, + "num_tokens": 225550.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.5845099687576294, + "sampling/importance_sampling_ratio/mean": 0.6625345945358276, + "sampling/importance_sampling_ratio/min": 0.04495502635836601, + "sampling/sampling_logp_difference/max": 0.36547183990478516, + "sampling/sampling_logp_difference/mean": 0.028946854174137115, + "step": 24, + "step_time": 14.95462113339454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.2471638694405556, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22180576622486115, + "kl": 0.018309170845896006, + "learning_rate": 3e-05, + "loss": -0.1412944793701172, + "num_tokens": 235005.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.6407876014709473, + "sampling/importance_sampling_ratio/mean": 0.7712795734405518, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4363563060760498, + "sampling/sampling_logp_difference/mean": 0.02898716926574707, + "step": 25, + "step_time": 16.46686205593869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 455.75, + "completions/mean_terminated_length": 455.75, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 1.28530602902174, + "epoch": 0.052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37535253167152405, + "kl": 0.020504546992015094, + "learning_rate": 3e-05, + "loss": 0.10839397460222244, + "num_tokens": 243953.0, + "reward": 6.25, + "reward_std": 1.0645813941955566, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.0645813941955566, + "sampling/importance_sampling_ratio/max": 2.0567266941070557, + "sampling/importance_sampling_ratio/mean": 0.5800145864486694, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4523625373840332, + "sampling/sampling_logp_difference/mean": 0.027610119432210922, + "step": 26, + "step_time": 22.121026155073196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.0, + "completions/max_terminated_length": 514.0, + "completions/mean_length": 462.75, + "completions/mean_terminated_length": 462.75, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.2253629937767982, + "epoch": 0.054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2340388149023056, + "kl": 0.020236384589225054, + "learning_rate": 3e-05, + "loss": 0.04823978245258331, + "num_tokens": 253237.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.1560932397842407, + "sampling/importance_sampling_ratio/mean": 0.40753045678138733, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35022830963134766, + "sampling/sampling_logp_difference/mean": 0.028367744758725166, + "step": 27, + "step_time": 15.10967448912561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 538.0, + "completions/max_terminated_length": 538.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "entropy": 1.3779077678918839, + "epoch": 0.056, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5531017184257507, + "kl": 0.01706669357372448, + "learning_rate": 3e-05, + "loss": 0.0933581069111824, + "num_tokens": 262454.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.5711586475372314, + "sampling/importance_sampling_ratio/mean": 0.7730721831321716, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.343827486038208, + "sampling/sampling_logp_difference/mean": 0.028883326798677444, + "step": 28, + "step_time": 38.59647103771567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 477.0, + "completions/mean_terminated_length": 477.0, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.3791070505976677, + "epoch": 0.058, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33771538734436035, + "kl": 0.02141271048458293, + "learning_rate": 3e-05, + "loss": 0.010216176509857178, + "num_tokens": 271918.0, + "reward": 5.75, + "reward_std": 1.2909945249557495, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.2909945249557495, + "sampling/importance_sampling_ratio/max": 2.4999797344207764, + "sampling/importance_sampling_ratio/mean": 1.0250636339187622, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3258817195892334, + "sampling/sampling_logp_difference/mean": 0.029029540717601776, + "step": 29, + "step_time": 23.610272648278624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 471.1875, + "completions/mean_terminated_length": 471.1875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.4652926102280617, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21955685317516327, + "kl": 0.019562674744520336, + "learning_rate": 3e-05, + "loss": -0.014814459718763828, + "num_tokens": 281305.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 2.802098274230957, + "sampling/importance_sampling_ratio/mean": 0.866391658782959, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6354451179504395, + "sampling/sampling_logp_difference/mean": 0.03177585452795029, + "step": 30, + "step_time": 16.961607525125146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 490.5, + "completions/mean_terminated_length": 490.5, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 1.1957123205065727, + "epoch": 0.062, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12176825106143951, + "kl": 0.026934220048133284, + "learning_rate": 3e-05, + "loss": -0.08307373523712158, + "num_tokens": 291409.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.610858678817749, + "sampling/importance_sampling_ratio/mean": 0.6937527656555176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4253416061401367, + "sampling/sampling_logp_difference/mean": 0.02871524728834629, + "step": 31, + "step_time": 15.012207658961415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 462.0, + "completions/mean_terminated_length": 462.0, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3018206283450127, + "epoch": 0.064, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4979296624660492, + "kl": 0.03539742121938616, + "learning_rate": 3e-05, + "loss": 0.0017175457905977964, + "num_tokens": 300649.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8527640104293823, + "sampling/importance_sampling_ratio/mean": 0.7824680209159851, + "sampling/importance_sampling_ratio/min": 0.07447884231805801, + "sampling/sampling_logp_difference/max": 0.5221483707427979, + "sampling/sampling_logp_difference/mean": 0.029107660055160522, + "step": 32, + "step_time": 22.78309725271538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 474.0, + "completions/mean_terminated_length": 474.0, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.321175642311573, + "epoch": 0.066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22016967833042145, + "kl": 0.029592803912237287, + "learning_rate": 3e-05, + "loss": 0.05625719577074051, + "num_tokens": 309889.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.9875097274780273, + "sampling/importance_sampling_ratio/mean": 0.7832435369491577, + "sampling/importance_sampling_ratio/min": 0.18006731569766998, + "sampling/sampling_logp_difference/max": 0.38585615158081055, + "sampling/sampling_logp_difference/mean": 0.027828343212604523, + "step": 33, + "step_time": 44.51360382232815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 459.9375, + "completions/mean_terminated_length": 459.9375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.2222414836287498, + "epoch": 0.068, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2873040437698364, + "kl": 0.02840927499346435, + "learning_rate": 3e-05, + "loss": -0.037432070821523666, + "num_tokens": 318904.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 2.6647069454193115, + "sampling/importance_sampling_ratio/mean": 0.5744763016700745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.41987133026123047, + "sampling/sampling_logp_difference/mean": 0.029470665380358696, + "step": 34, + "step_time": 133.03877451224253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 522.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 462.1875, + "completions/mean_terminated_length": 462.1875, + "completions/min_length": 413.0, + "completions/min_terminated_length": 413.0, + "entropy": 1.1665974482893944, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2490653246641159, + "kl": 0.025841041060630232, + "learning_rate": 3e-05, + "loss": -0.20422951877117157, + "num_tokens": 328011.0, + "reward": 6.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 2.510730266571045, + "sampling/importance_sampling_ratio/mean": 0.7954420447349548, + "sampling/importance_sampling_ratio/min": 0.020566223189234734, + "sampling/sampling_logp_difference/max": 0.36430132389068604, + "sampling/sampling_logp_difference/mean": 0.026844775304198265, + "step": 35, + "step_time": 21.5843787365593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 549.0, + "completions/max_terminated_length": 549.0, + "completions/mean_length": 492.0, + "completions/mean_terminated_length": 492.0, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3467887043952942, + "epoch": 0.072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15268851816654205, + "kl": 0.023660800594370812, + "learning_rate": 3e-05, + "loss": -0.11188814043998718, + "num_tokens": 337731.0, + "reward": 6.5625, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.6270908117294312, + "sampling/importance_sampling_ratio/mean": 0.614537239074707, + "sampling/importance_sampling_ratio/min": 0.10853960365056992, + "sampling/sampling_logp_difference/max": 0.4023854732513428, + "sampling/sampling_logp_difference/mean": 0.028961554169654846, + "step": 36, + "step_time": 18.843947287183255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.0, + "completions/max_terminated_length": 513.0, + "completions/mean_length": 460.75, + "completions/mean_terminated_length": 460.75, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "entropy": 1.2476315572857857, + "epoch": 0.074, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42548227310180664, + "kl": 0.022181478852871805, + "learning_rate": 3e-05, + "loss": 0.37223517894744873, + "num_tokens": 346815.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.6624510288238525, + "sampling/importance_sampling_ratio/mean": 0.7942180633544922, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4037543535232544, + "sampling/sampling_logp_difference/mean": 0.028780322521924973, + "step": 37, + "step_time": 42.04662919091061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 452.0, + "completions/mean_terminated_length": 452.0, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "entropy": 1.1745503433048725, + "epoch": 0.076, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16980381309986115, + "kl": 0.017483733594417572, + "learning_rate": 3e-05, + "loss": -0.09029137343168259, + "num_tokens": 355711.0, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "sampling/importance_sampling_ratio/max": 2.3201518058776855, + "sampling/importance_sampling_ratio/mean": 0.721072793006897, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.30433225631713867, + "sampling/sampling_logp_difference/mean": 0.026646045967936516, + "step": 38, + "step_time": 38.63075139513239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 466.8125, + "completions/mean_terminated_length": 466.8125, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2664988860487938, + "epoch": 0.078, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21305795013904572, + "kl": 0.021121050289366394, + "learning_rate": 3e-05, + "loss": -0.03154226019978523, + "num_tokens": 364860.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.0297553539276123, + "sampling/importance_sampling_ratio/mean": 0.674609363079071, + "sampling/importance_sampling_ratio/min": 0.11245065927505493, + "sampling/sampling_logp_difference/max": 0.37443917989730835, + "sampling/sampling_logp_difference/mean": 0.028257746249437332, + "step": 39, + "step_time": 30.028073193039745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 485.1875, + "completions/mean_terminated_length": 485.1875, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.3458357080817223, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12235487997531891, + "kl": 0.018535695446189493, + "learning_rate": 3e-05, + "loss": -0.035816628485918045, + "num_tokens": 374607.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.242042303085327, + "sampling/importance_sampling_ratio/mean": 0.5344723463058472, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36985254287719727, + "sampling/sampling_logp_difference/mean": 0.029600802809000015, + "step": 40, + "step_time": 15.446288945619017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 498.875, + "completions/mean_terminated_length": 498.875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3402792811393738, + "epoch": 0.082, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15642686188220978, + "kl": 0.013967045990284532, + "learning_rate": 3e-05, + "loss": 0.0011727213859558105, + "num_tokens": 384317.0, + "reward": 6.375, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5691092014312744, + "sampling/importance_sampling_ratio/mean": 0.3696203827857971, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6487679481506348, + "sampling/sampling_logp_difference/mean": 0.03018251620233059, + "step": 41, + "step_time": 18.15720977121964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.3317564576864243, + "epoch": 0.084, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3236794173717499, + "kl": 0.01707366103073582, + "learning_rate": 3e-05, + "loss": 0.10363321751356125, + "num_tokens": 393934.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.0568668842315674, + "sampling/importance_sampling_ratio/mean": 0.6415404677391052, + "sampling/importance_sampling_ratio/min": 0.07682990282773972, + "sampling/sampling_logp_difference/max": 0.7769032716751099, + "sampling/sampling_logp_difference/mean": 0.02936164103448391, + "step": 42, + "step_time": 15.066733688581735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 510.9375, + "completions/mean_terminated_length": 510.9375, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2099780030548573, + "epoch": 0.086, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1771102100610733, + "kl": 0.01784718461567536, + "learning_rate": 3e-05, + "loss": -0.027566466480493546, + "num_tokens": 403893.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.3430161476135254, + "sampling/importance_sampling_ratio/mean": 0.7769261598587036, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36153650283813477, + "sampling/sampling_logp_difference/mean": 0.028799494728446007, + "step": 43, + "step_time": 16.90863296529278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 494.1875, + "completions/mean_terminated_length": 494.1875, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "entropy": 1.2924985438585281, + "epoch": 0.088, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3432070314884186, + "kl": 0.014529847539961338, + "learning_rate": 3e-05, + "loss": -0.04157561436295509, + "num_tokens": 413312.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.726816177368164, + "sampling/importance_sampling_ratio/mean": 0.8989821672439575, + "sampling/importance_sampling_ratio/min": 0.07410597801208496, + "sampling/sampling_logp_difference/max": 0.31444358825683594, + "sampling/sampling_logp_difference/mean": 0.028122060000896454, + "step": 44, + "step_time": 16.7880685236305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 543.5, + "completions/mean_terminated_length": 543.5, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.206408604979515, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36233776807785034, + "kl": 0.015796773659531027, + "learning_rate": 3e-05, + "loss": 0.029176680371165276, + "num_tokens": 423624.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.677195429801941, + "sampling/importance_sampling_ratio/mean": 0.6537867188453674, + "sampling/importance_sampling_ratio/min": 0.09822077304124832, + "sampling/sampling_logp_difference/max": 0.31381869316101074, + "sampling/sampling_logp_difference/mean": 0.02717999368906021, + "step": 45, + "step_time": 23.35960763087496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 643.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 534.5625, + "completions/mean_terminated_length": 534.5625, + "completions/min_length": 453.0, + "completions/min_terminated_length": 453.0, + "entropy": 1.2577891275286674, + "epoch": 0.092, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20219561457633972, + "kl": 0.014481160265859216, + "learning_rate": 3e-05, + "loss": 0.18850615620613098, + "num_tokens": 433817.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.203894853591919, + "sampling/importance_sampling_ratio/mean": 0.697495698928833, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35237741470336914, + "sampling/sampling_logp_difference/mean": 0.02723248302936554, + "step": 46, + "step_time": 19.56032704282552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 538.25, + "completions/mean_terminated_length": 538.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.3271892964839935, + "epoch": 0.094, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22262753546237946, + "kl": 0.012554377142805606, + "learning_rate": 3e-05, + "loss": 0.06984467804431915, + "num_tokens": 444045.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 2.2261502742767334, + "sampling/importance_sampling_ratio/mean": 0.6712950468063354, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4496123790740967, + "sampling/sampling_logp_difference/mean": 0.028838323429226875, + "step": 47, + "step_time": 21.226045075803995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 514.25, + "completions/mean_terminated_length": 514.25, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "entropy": 1.1054812744259834, + "epoch": 0.096, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3244606554508209, + "kl": 0.0157591889728792, + "learning_rate": 3e-05, + "loss": 0.09024985134601593, + "num_tokens": 453945.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.6745388507843018, + "sampling/importance_sampling_ratio/mean": 0.7607913017272949, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650559186935425, + "sampling/sampling_logp_difference/mean": 0.026116060093045235, + "step": 48, + "step_time": 17.565261672716588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 585.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 502.4375, + "completions/mean_terminated_length": 502.4375, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.2281213253736496, + "epoch": 0.098, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4033137857913971, + "kl": 0.015613102470524609, + "learning_rate": 3e-05, + "loss": -0.2898017466068268, + "num_tokens": 463576.0, + "reward": 6.4375, + "reward_std": 1.2632629871368408, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.2632629871368408, + "sampling/importance_sampling_ratio/max": 2.6974196434020996, + "sampling/importance_sampling_ratio/mean": 0.7124314308166504, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3675417900085449, + "sampling/sampling_logp_difference/mean": 0.028341906145215034, + "step": 49, + "step_time": 29.838082558009773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 530.1875, + "completions/mean_terminated_length": 530.1875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.1955150067806244, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23144562542438507, + "kl": 0.01374751579714939, + "learning_rate": 3e-05, + "loss": -0.19065965712070465, + "num_tokens": 473915.0, + "reward": 6.3125, + "reward_std": 1.1954776048660278, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.1954776048660278, + "sampling/importance_sampling_ratio/max": 2.54063081741333, + "sampling/importance_sampling_ratio/mean": 0.8162041902542114, + "sampling/importance_sampling_ratio/min": 0.1628064066171646, + "sampling/sampling_logp_difference/max": 0.2777421474456787, + "sampling/sampling_logp_difference/mean": 0.02853373810648918, + "step": 50, + "step_time": 16.047010839451104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 549.875, + "completions/mean_terminated_length": 549.875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1870752647519112, + "epoch": 0.102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4241364896297455, + "kl": 0.013923745544161648, + "learning_rate": 3e-05, + "loss": -0.1761355698108673, + "num_tokens": 484577.0, + "reward": 6.5625, + "reward_std": 1.0935417413711548, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 1.0935417413711548, + "sampling/importance_sampling_ratio/max": 2.929507255554199, + "sampling/importance_sampling_ratio/mean": 0.6905896663665771, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5649824142456055, + "sampling/sampling_logp_difference/mean": 0.028025619685649872, + "step": 51, + "step_time": 44.488836522214115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 552.375, + "completions/mean_terminated_length": 552.375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.098166175186634, + "epoch": 0.104, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.32851356267929077, + "kl": 0.01297539210645482, + "learning_rate": 3e-05, + "loss": -0.06503897905349731, + "num_tokens": 495015.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.9652340412139893, + "sampling/importance_sampling_ratio/mean": 0.9612224102020264, + "sampling/importance_sampling_ratio/min": 0.040177375078201294, + "sampling/sampling_logp_difference/max": 0.3454720973968506, + "sampling/sampling_logp_difference/mean": 0.02687419019639492, + "step": 52, + "step_time": 20.21497478755191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 578.375, + "completions/mean_terminated_length": 578.375, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2311968132853508, + "epoch": 0.106, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21921825408935547, + "kl": 0.017025968758389354, + "learning_rate": 3e-05, + "loss": -0.18975484371185303, + "num_tokens": 505909.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.0094237327575684, + "sampling/importance_sampling_ratio/mean": 0.5587533116340637, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.39138758182525635, + "sampling/sampling_logp_difference/mean": 0.028095029294490814, + "step": 53, + "step_time": 31.140278964303434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 473.25, + "completions/mean_terminated_length": 473.25, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 1.2682743221521378, + "epoch": 0.108, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27410373091697693, + "kl": 0.018500705540645868, + "learning_rate": 3e-05, + "loss": 0.14847250282764435, + "num_tokens": 515073.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.875750780105591, + "sampling/importance_sampling_ratio/mean": 0.7429271936416626, + "sampling/importance_sampling_ratio/min": 0.09779425710439682, + "sampling/sampling_logp_difference/max": 0.5433444976806641, + "sampling/sampling_logp_difference/mean": 0.02810005284845829, + "step": 54, + "step_time": 18.365382733754814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 549.0, + "completions/mean_terminated_length": 549.0, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.3073157891631126, + "epoch": 0.11, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29323700070381165, + "kl": 0.016703857632819563, + "learning_rate": 3e-05, + "loss": 0.05967015400528908, + "num_tokens": 525377.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 1.4832638502120972, + "sampling/importance_sampling_ratio/mean": 0.6094669103622437, + "sampling/importance_sampling_ratio/min": 0.08072065562009811, + "sampling/sampling_logp_difference/max": 0.39328718185424805, + "sampling/sampling_logp_difference/mean": 0.02906947024166584, + "step": 55, + "step_time": 30.47015379369259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 576.625, + "completions/mean_terminated_length": 576.625, + "completions/min_length": 500.0, + "completions/min_terminated_length": 500.0, + "entropy": 1.2820357605814934, + "epoch": 0.112, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.200644388794899, + "kl": 0.018819268501829356, + "learning_rate": 3e-05, + "loss": -0.04828515648841858, + "num_tokens": 536163.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.3941831588745117, + "sampling/importance_sampling_ratio/mean": 0.5633801221847534, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3664684295654297, + "sampling/sampling_logp_difference/mean": 0.02962428703904152, + "step": 56, + "step_time": 16.172011949121952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 640.25, + "completions/mean_terminated_length": 640.25, + "completions/min_length": 556.0, + "completions/min_terminated_length": 556.0, + "entropy": 1.4191219061613083, + "epoch": 0.114, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19633841514587402, + "kl": 0.02060725452611223, + "learning_rate": 3e-05, + "loss": -0.12029920518398285, + "num_tokens": 548143.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.3877830505371094, + "sampling/importance_sampling_ratio/mean": 0.6956661343574524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33841991424560547, + "sampling/sampling_logp_difference/mean": 0.028747636824846268, + "step": 57, + "step_time": 26.892430102452636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 646.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 560.4375, + "completions/mean_terminated_length": 560.4375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.3337246850132942, + "epoch": 0.116, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24178451299667358, + "kl": 0.023009511292912066, + "learning_rate": 3e-05, + "loss": -0.02738652005791664, + "num_tokens": 558710.0, + "reward": 6.0625, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.4787031412124634, + "sampling/importance_sampling_ratio/mean": 0.4813012480735779, + "sampling/importance_sampling_ratio/min": 0.05691095441579819, + "sampling/sampling_logp_difference/max": 0.3029825687408447, + "sampling/sampling_logp_difference/mean": 0.029777564108371735, + "step": 58, + "step_time": 17.878377372398973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 697.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 595.25, + "completions/mean_terminated_length": 595.25, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.3107040077447891, + "epoch": 0.118, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1262614130973816, + "kl": 0.022026430582627654, + "learning_rate": 3e-05, + "loss": 0.04775794595479965, + "num_tokens": 569826.0, + "reward": 6.4375, + "reward_std": 1.0307763814926147, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.0307763814926147, + "sampling/importance_sampling_ratio/max": 2.1841602325439453, + "sampling/importance_sampling_ratio/mean": 0.5139275193214417, + "sampling/importance_sampling_ratio/min": 0.026369251310825348, + "sampling/sampling_logp_difference/max": 0.42272377014160156, + "sampling/sampling_logp_difference/mean": 0.028494788333773613, + "step": 59, + "step_time": 24.42572767334059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 549.1875, + "completions/mean_terminated_length": 549.1875, + "completions/min_length": 489.0, + "completions/min_terminated_length": 489.0, + "entropy": 1.1738965958356857, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.588756799697876, + "kl": 0.025482238037511706, + "learning_rate": 3e-05, + "loss": 0.2925710678100586, + "num_tokens": 580229.0, + "reward": 6.0625, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 2.8465583324432373, + "sampling/importance_sampling_ratio/mean": 1.1227651834487915, + "sampling/importance_sampling_ratio/min": 0.1096419170498848, + "sampling/sampling_logp_difference/max": 0.4628920555114746, + "sampling/sampling_logp_difference/mean": 0.02885228767991066, + "step": 60, + "step_time": 21.57787229306996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 692.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 577.3125, + "completions/mean_terminated_length": 577.3125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.1913195550441742, + "epoch": 0.122, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2779391407966614, + "kl": 0.02629381464794278, + "learning_rate": 3e-05, + "loss": 0.12304374575614929, + "num_tokens": 591178.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.1122686862945557, + "sampling/importance_sampling_ratio/mean": 0.650765061378479, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.47726941108703613, + "sampling/sampling_logp_difference/mean": 0.027112768962979317, + "step": 61, + "step_time": 23.71764762001112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 628.3125, + "completions/mean_terminated_length": 628.3125, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3224291801452637, + "epoch": 0.124, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1645125448703766, + "kl": 0.025764177553355694, + "learning_rate": 3e-05, + "loss": 0.17419062554836273, + "num_tokens": 603055.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1662254333496094, + "sampling/importance_sampling_ratio/mean": 0.5809424519538879, + "sampling/importance_sampling_ratio/min": 0.03490918502211571, + "sampling/sampling_logp_difference/max": 0.4525270462036133, + "sampling/sampling_logp_difference/mean": 0.028948483988642693, + "step": 62, + "step_time": 35.74759274255484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 597.375, + "completions/mean_terminated_length": 597.375, + "completions/min_length": 478.0, + "completions/min_terminated_length": 478.0, + "entropy": 1.1552416533231735, + "epoch": 0.126, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20707836747169495, + "kl": 0.026473846402950585, + "learning_rate": 3e-05, + "loss": -0.019145065918564796, + "num_tokens": 614341.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.0268709659576416, + "sampling/importance_sampling_ratio/mean": 0.7066210508346558, + "sampling/importance_sampling_ratio/min": 0.1595209240913391, + "sampling/sampling_logp_difference/max": 0.42907285690307617, + "sampling/sampling_logp_difference/mean": 0.028000790625810623, + "step": 63, + "step_time": 39.37250621803105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 568.5, + "completions/mean_terminated_length": 568.5, + "completions/min_length": 509.0, + "completions/min_terminated_length": 509.0, + "entropy": 1.2810933291912079, + "epoch": 0.128, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21552008390426636, + "kl": 0.029041914734989405, + "learning_rate": 3e-05, + "loss": 0.037037670612335205, + "num_tokens": 625165.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.1896748542785645, + "sampling/importance_sampling_ratio/mean": 0.6408567428588867, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.374176025390625, + "sampling/sampling_logp_difference/mean": 0.02858484350144863, + "step": 64, + "step_time": 18.69576471252367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 622.75, + "completions/mean_terminated_length": 622.75, + "completions/min_length": 545.0, + "completions/min_terminated_length": 545.0, + "entropy": 1.32467532902956, + "epoch": 0.13, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2287176102399826, + "kl": 0.023987084976397455, + "learning_rate": 3e-05, + "loss": 0.0731797143816948, + "num_tokens": 636633.0, + "reward": 6.375, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 1.4288272857666016, + "sampling/importance_sampling_ratio/mean": 0.5977773666381836, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4503474235534668, + "sampling/sampling_logp_difference/mean": 0.02755960263311863, + "step": 65, + "step_time": 27.502957582939416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 767.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 639.25, + "completions/mean_terminated_length": 639.25, + "completions/min_length": 554.0, + "completions/min_terminated_length": 554.0, + "entropy": 1.400998167693615, + "epoch": 0.132, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27425676584243774, + "kl": 0.028104660217650235, + "learning_rate": 3e-05, + "loss": 0.10324293375015259, + "num_tokens": 648597.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.971261501312256, + "sampling/importance_sampling_ratio/mean": 0.6433250904083252, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4819211959838867, + "sampling/sampling_logp_difference/mean": 0.029852069914340973, + "step": 66, + "step_time": 26.79405551031232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 541.375, + "completions/mean_terminated_length": 541.375, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.2635268718004227, + "epoch": 0.134, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13181555271148682, + "kl": 0.03373931790702045, + "learning_rate": 3e-05, + "loss": -0.11447598040103912, + "num_tokens": 658875.0, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 1.1883972883224487, + "sampling/importance_sampling_ratio/mean": 0.4192371368408203, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.44372403621673584, + "sampling/sampling_logp_difference/mean": 0.02911720983684063, + "step": 67, + "step_time": 27.6137525443919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 842.0, + "completions/max_terminated_length": 842.0, + "completions/mean_length": 689.4375, + "completions/mean_terminated_length": 689.4375, + "completions/min_length": 544.0, + "completions/min_terminated_length": 544.0, + "entropy": 1.2496536895632744, + "epoch": 0.136, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20560002326965332, + "kl": 0.026702777307946235, + "learning_rate": 3e-05, + "loss": -0.10130438208580017, + "num_tokens": 671690.0, + "reward": 5.875, + "reward_std": 2.0289571285247803, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 2.0289571285247803, + "sampling/importance_sampling_ratio/max": 2.8383262157440186, + "sampling/importance_sampling_ratio/mean": 0.9476768374443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35082435607910156, + "sampling/sampling_logp_difference/mean": 0.028364315629005432, + "step": 68, + "step_time": 29.35345455724746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 701.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 614.25, + "completions/mean_terminated_length": 614.25, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.1948458775877953, + "epoch": 0.138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20431844890117645, + "kl": 0.03377161466050893, + "learning_rate": 3e-05, + "loss": -0.0560678169131279, + "num_tokens": 683046.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.162766933441162, + "sampling/importance_sampling_ratio/mean": 0.5678162574768066, + "sampling/importance_sampling_ratio/min": 0.05678822472691536, + "sampling/sampling_logp_difference/max": 0.5758893489837646, + "sampling/sampling_logp_difference/mean": 0.028125843033194542, + "step": 69, + "step_time": 27.574916049838066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 588.3125, + "completions/mean_terminated_length": 588.3125, + "completions/min_length": 532.0, + "completions/min_terminated_length": 532.0, + "entropy": 1.2782762721180916, + "epoch": 0.14, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26451998949050903, + "kl": 0.03907236340455711, + "learning_rate": 3e-05, + "loss": 0.17035090923309326, + "num_tokens": 694323.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.950176477432251, + "sampling/importance_sampling_ratio/mean": 0.45171743631362915, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46894216537475586, + "sampling/sampling_logp_difference/mean": 0.02863166108727455, + "step": 70, + "step_time": 19.817490340210497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 597.125, + "completions/mean_terminated_length": 597.125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2051330730319023, + "epoch": 0.142, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5549097657203674, + "kl": 0.03670362115371972, + "learning_rate": 3e-05, + "loss": 0.4998631179332733, + "num_tokens": 705773.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.9404170513153076, + "sampling/importance_sampling_ratio/mean": 0.7522475123405457, + "sampling/importance_sampling_ratio/min": 0.05456363409757614, + "sampling/sampling_logp_difference/max": 0.4324514865875244, + "sampling/sampling_logp_difference/mean": 0.028340937569737434, + "step": 71, + "step_time": 41.66373607888818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 691.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 604.125, + "completions/mean_terminated_length": 604.125, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.18794547021389, + "epoch": 0.144, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10015435516834259, + "kl": 0.03911226138006896, + "learning_rate": 3e-05, + "loss": -0.02419177070260048, + "num_tokens": 717159.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1231037378311157, + "sampling/importance_sampling_ratio/mean": 0.4037884473800659, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6400742530822754, + "sampling/sampling_logp_difference/mean": 0.028367817401885986, + "step": 72, + "step_time": 23.86539705330506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 721.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 597.1875, + "completions/mean_terminated_length": 597.1875, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.2641174346208572, + "epoch": 0.146, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07918722927570343, + "kl": 0.03177848691120744, + "learning_rate": 3e-05, + "loss": -0.027635926380753517, + "num_tokens": 728402.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.5195796489715576, + "sampling/importance_sampling_ratio/mean": 0.4324396848678589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6819734573364258, + "sampling/sampling_logp_difference/mean": 0.030029678717255592, + "step": 73, + "step_time": 29.29490938829258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 640.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 574.375, + "completions/mean_terminated_length": 574.375, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.3364054411649704, + "epoch": 0.148, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3853444755077362, + "kl": 0.03433372196741402, + "learning_rate": 3e-05, + "loss": -0.031142480671405792, + "num_tokens": 739632.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6853926181793213, + "sampling/importance_sampling_ratio/mean": 0.9200767874717712, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42021608352661133, + "sampling/sampling_logp_difference/mean": 0.030795859172940254, + "step": 74, + "step_time": 34.52008486771956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 618.25, + "completions/mean_terminated_length": 618.25, + "completions/min_length": 539.0, + "completions/min_terminated_length": 539.0, + "entropy": 1.365300178527832, + "epoch": 0.15, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18101376295089722, + "kl": 0.02779634529724717, + "learning_rate": 3e-05, + "loss": -0.2718795835971832, + "num_tokens": 751164.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.8397568464279175, + "sampling/importance_sampling_ratio/mean": 0.5582400560379028, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42606019973754883, + "sampling/sampling_logp_difference/mean": 0.028895940631628036, + "step": 75, + "step_time": 18.76476171752438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 799.0, + "completions/max_terminated_length": 799.0, + "completions/mean_length": 603.375, + "completions/mean_terminated_length": 603.375, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2181015871465206, + "epoch": 0.152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1852622777223587, + "kl": 0.03176840906962752, + "learning_rate": 3e-05, + "loss": -0.10660596191883087, + "num_tokens": 762370.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.317336082458496, + "sampling/importance_sampling_ratio/mean": 0.550554633140564, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.481325626373291, + "sampling/sampling_logp_difference/mean": 0.028557566925883293, + "step": 76, + "step_time": 27.090129756834358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 572.9375, + "completions/mean_terminated_length": 572.9375, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2986655682325363, + "epoch": 0.154, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1531844586133957, + "kl": 0.03523328877054155, + "learning_rate": 3e-05, + "loss": -0.20856647193431854, + "num_tokens": 773169.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 2.855010986328125, + "sampling/importance_sampling_ratio/mean": 0.8239375352859497, + "sampling/importance_sampling_ratio/min": 0.1521405428647995, + "sampling/sampling_logp_difference/max": 0.4692528247833252, + "sampling/sampling_logp_difference/mean": 0.029906686395406723, + "step": 77, + "step_time": 25.004970545414835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 607.1875, + "completions/mean_terminated_length": 607.1875, + "completions/min_length": 540.0, + "completions/min_terminated_length": 540.0, + "entropy": 1.1003647185862064, + "epoch": 0.156, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14113759994506836, + "kl": 0.025135049945674837, + "learning_rate": 3e-05, + "loss": -0.10802068561315536, + "num_tokens": 784724.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.0096027851104736, + "sampling/importance_sampling_ratio/mean": 0.613497257232666, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4082927703857422, + "sampling/sampling_logp_difference/mean": 0.027884263545274734, + "step": 78, + "step_time": 29.614154959097505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 568.0625, + "completions/mean_terminated_length": 568.0625, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.266264721751213, + "epoch": 0.158, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18073000013828278, + "kl": 0.028119354974478483, + "learning_rate": 3e-05, + "loss": -0.0687609314918518, + "num_tokens": 795517.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.277299404144287, + "sampling/importance_sampling_ratio/mean": 0.3485238552093506, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4658241271972656, + "sampling/sampling_logp_difference/mean": 0.029626762494444847, + "step": 79, + "step_time": 23.5287338164635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 565.8125, + "completions/mean_terminated_length": 565.8125, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.2773499339818954, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.31362995505332947, + "kl": 0.028471783734858036, + "learning_rate": 3e-05, + "loss": 0.059164684265851974, + "num_tokens": 806258.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.5347814559936523, + "sampling/importance_sampling_ratio/mean": 0.7027873396873474, + "sampling/importance_sampling_ratio/min": 0.06356429308652878, + "sampling/sampling_logp_difference/max": 0.4123659133911133, + "sampling/sampling_logp_difference/mean": 0.02946357987821102, + "step": 80, + "step_time": 24.251087154261768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 600.0, + "completions/max_terminated_length": 600.0, + "completions/mean_length": 536.1875, + "completions/mean_terminated_length": 536.1875, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.211024284362793, + "epoch": 0.162, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1959177404642105, + "kl": 0.02152467134874314, + "learning_rate": 3e-05, + "loss": 0.14755703508853912, + "num_tokens": 816717.0, + "reward": 6.0, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.764387369155884, + "sampling/importance_sampling_ratio/mean": 0.8167816400527954, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.28648805618286133, + "sampling/sampling_logp_difference/mean": 0.02814806066453457, + "step": 81, + "step_time": 22.752198832575232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 568.625, + "completions/mean_terminated_length": 568.625, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "entropy": 1.2584010139107704, + "epoch": 0.164, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16224367916584015, + "kl": 0.02812566957436502, + "learning_rate": 3e-05, + "loss": -0.1586945354938507, + "num_tokens": 827591.0, + "reward": 6.5, + "reward_std": 1.26491117477417, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.26491117477417, + "sampling/importance_sampling_ratio/max": 2.4172537326812744, + "sampling/importance_sampling_ratio/mean": 0.7026975154876709, + "sampling/importance_sampling_ratio/min": 0.1125984862446785, + "sampling/sampling_logp_difference/max": 0.3966444730758667, + "sampling/sampling_logp_difference/mean": 0.02937215007841587, + "step": 82, + "step_time": 22.57465209439397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 783.0, + "completions/max_terminated_length": 783.0, + "completions/mean_length": 583.5625, + "completions/mean_terminated_length": 583.5625, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.2498536258935928, + "epoch": 0.166, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28551891446113586, + "kl": 0.023335880250670016, + "learning_rate": 3e-05, + "loss": 0.09868354350328445, + "num_tokens": 838760.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 1.9430233240127563, + "sampling/importance_sampling_ratio/mean": 0.7391272783279419, + "sampling/importance_sampling_ratio/min": 0.2032935619354248, + "sampling/sampling_logp_difference/max": 0.3390723466873169, + "sampling/sampling_logp_difference/mean": 0.02833949774503708, + "step": 83, + "step_time": 24.9633763525635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 581.0625, + "completions/mean_terminated_length": 581.0625, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.1582137942314148, + "epoch": 0.168, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1089889332652092, + "kl": 0.02546319324756041, + "learning_rate": 3e-05, + "loss": -0.04368923604488373, + "num_tokens": 849945.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.6800583600997925, + "sampling/importance_sampling_ratio/mean": 0.4864083528518677, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48818397521972656, + "sampling/sampling_logp_difference/mean": 0.02942320704460144, + "step": 84, + "step_time": 22.7196712391451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 553.3125, + "completions/mean_terminated_length": 553.3125, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.1762890554964542, + "epoch": 0.17, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18219847977161407, + "kl": 0.023275951389223337, + "learning_rate": 3e-05, + "loss": 0.055040232837200165, + "num_tokens": 860734.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.2940757274627686, + "sampling/importance_sampling_ratio/mean": 0.6381184458732605, + "sampling/importance_sampling_ratio/min": 0.08803392946720123, + "sampling/sampling_logp_difference/max": 0.3728243112564087, + "sampling/sampling_logp_difference/mean": 0.028103860095143318, + "step": 85, + "step_time": 28.569310082122684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 538.5, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.1561524420976639, + "epoch": 0.172, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2631295323371887, + "kl": 0.027657793601974845, + "learning_rate": 3e-05, + "loss": 0.019699495285749435, + "num_tokens": 871182.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.950981378555298, + "sampling/importance_sampling_ratio/mean": 0.5496660470962524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.358644962310791, + "sampling/sampling_logp_difference/mean": 0.02922222763299942, + "step": 86, + "step_time": 19.95468496438116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 688.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 528.625, + "completions/mean_terminated_length": 528.625, + "completions/min_length": 418.0, + "completions/min_terminated_length": 418.0, + "entropy": 1.382395550608635, + "epoch": 0.174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25233277678489685, + "kl": 0.025461523793637753, + "learning_rate": 3e-05, + "loss": -0.0012568621896207333, + "num_tokens": 881272.0, + "reward": 6.3125, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 1.940340280532837, + "sampling/importance_sampling_ratio/mean": 0.44232380390167236, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3433331251144409, + "sampling/sampling_logp_difference/mean": 0.030555889010429382, + "step": 87, + "step_time": 29.044239778537303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 537.0, + "completions/mean_terminated_length": 537.0, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.1740282103419304, + "epoch": 0.176, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1609290987253189, + "kl": 0.023582924506627023, + "learning_rate": 3e-05, + "loss": -0.0040507810190320015, + "num_tokens": 891608.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.8592076301574707, + "sampling/importance_sampling_ratio/mean": 0.4413543939590454, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3994784355163574, + "sampling/sampling_logp_difference/mean": 0.028627343475818634, + "step": 88, + "step_time": 24.642031190916896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 562.0, + "completions/mean_terminated_length": 562.0, + "completions/min_length": 490.0, + "completions/min_terminated_length": 490.0, + "entropy": 1.3354259580373764, + "epoch": 0.178, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25136521458625793, + "kl": 0.03104234568309039, + "learning_rate": 3e-05, + "loss": -0.10014888644218445, + "num_tokens": 902472.0, + "reward": 6.5, + "reward_std": 1.154700517654419, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.154700517654419, + "sampling/importance_sampling_ratio/max": 2.0172529220581055, + "sampling/importance_sampling_ratio/mean": 0.5528109073638916, + "sampling/importance_sampling_ratio/min": 0.031755149364471436, + "sampling/sampling_logp_difference/max": 0.48168420791625977, + "sampling/sampling_logp_difference/mean": 0.029525987803936005, + "step": 89, + "step_time": 23.934129936154932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 543.6875, + "completions/mean_terminated_length": 543.6875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.3072879239916801, + "epoch": 0.18, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2416294664144516, + "kl": 0.02474795945454389, + "learning_rate": 3e-05, + "loss": 0.02994484454393387, + "num_tokens": 913003.0, + "reward": 6.125, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4122473001480103, + "sampling/importance_sampling_ratio/mean": 0.5672672390937805, + "sampling/importance_sampling_ratio/min": 0.1312582641839981, + "sampling/sampling_logp_difference/max": 0.36547136306762695, + "sampling/sampling_logp_difference/mean": 0.030115650966763496, + "step": 90, + "step_time": 16.719651044346392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 587.5, + "completions/mean_terminated_length": 587.5, + "completions/min_length": 491.0, + "completions/min_terminated_length": 491.0, + "entropy": 1.2642723061144352, + "epoch": 0.182, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11401186883449554, + "kl": 0.018764875654596835, + "learning_rate": 3e-05, + "loss": 0.17740829288959503, + "num_tokens": 923971.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.223915934562683, + "sampling/importance_sampling_ratio/mean": 0.4373893141746521, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.43628501892089844, + "sampling/sampling_logp_difference/mean": 0.027218280360102654, + "step": 91, + "step_time": 21.256958127953112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 519.3125, + "completions/mean_terminated_length": 519.3125, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.2509336844086647, + "epoch": 0.184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14000695943832397, + "kl": 0.017409664229489863, + "learning_rate": 3e-05, + "loss": -0.1092228814959526, + "num_tokens": 933880.0, + "reward": 7.125, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 2.4079525470733643, + "sampling/importance_sampling_ratio/mean": 0.6406391263008118, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3843420743942261, + "sampling/sampling_logp_difference/mean": 0.02841360680758953, + "step": 92, + "step_time": 20.99564675288275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 681.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 587.8125, + "completions/mean_terminated_length": 587.8125, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.1215453520417213, + "epoch": 0.186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16520461440086365, + "kl": 0.028165725176222622, + "learning_rate": 3e-05, + "loss": -0.15029624104499817, + "num_tokens": 945269.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.7825064659118652, + "sampling/importance_sampling_ratio/mean": 0.5902000069618225, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3571600914001465, + "sampling/sampling_logp_difference/mean": 0.02762974239885807, + "step": 93, + "step_time": 17.922352592926472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 566.0, + "completions/mean_terminated_length": 566.0, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.2864234894514084, + "epoch": 0.188, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24750091135501862, + "kl": 0.02070689742686227, + "learning_rate": 3e-05, + "loss": 0.2850193381309509, + "num_tokens": 956021.0, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "sampling/importance_sampling_ratio/max": 1.9146449565887451, + "sampling/importance_sampling_ratio/mean": 0.6849822402000427, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4927506446838379, + "sampling/sampling_logp_difference/mean": 0.029227914288640022, + "step": 94, + "step_time": 23.034203104209155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 516.6875, + "completions/mean_terminated_length": 516.6875, + "completions/min_length": 486.0, + "completions/min_terminated_length": 486.0, + "entropy": 1.2404684573411942, + "epoch": 0.19, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11928244680166245, + "kl": 0.023086783126927912, + "learning_rate": 3e-05, + "loss": -0.032361116260290146, + "num_tokens": 965920.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.216193437576294, + "sampling/importance_sampling_ratio/mean": 0.32463955879211426, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37001991271972656, + "sampling/sampling_logp_difference/mean": 0.02843114361166954, + "step": 95, + "step_time": 15.103232022374868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 650.0, + "completions/max_terminated_length": 650.0, + "completions/mean_length": 548.4375, + "completions/mean_terminated_length": 548.4375, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.2861761301755905, + "epoch": 0.192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2920665442943573, + "kl": 0.017841250344645232, + "learning_rate": 3e-05, + "loss": 0.1640928089618683, + "num_tokens": 976695.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.8954812288284302, + "sampling/importance_sampling_ratio/mean": 0.754618763923645, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31093156337738037, + "sampling/sampling_logp_difference/mean": 0.02842729538679123, + "step": 96, + "step_time": 40.74571412149817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 553.5625, + "completions/mean_terminated_length": 553.5625, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.1928813084959984, + "epoch": 0.194, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29983460903167725, + "kl": 0.020173200638964772, + "learning_rate": 3e-05, + "loss": 0.05926981568336487, + "num_tokens": 987120.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.0362496376037598, + "sampling/importance_sampling_ratio/mean": 0.6645779609680176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40996313095092773, + "sampling/sampling_logp_difference/mean": 0.02854262851178646, + "step": 97, + "step_time": 17.38945102225989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 546.1875, + "completions/mean_terminated_length": 546.1875, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "entropy": 1.365786962211132, + "epoch": 0.196, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12385546416044235, + "kl": 0.02262102661188692, + "learning_rate": 3e-05, + "loss": -0.09927551448345184, + "num_tokens": 997395.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2717533111572266, + "sampling/importance_sampling_ratio/mean": 0.5988417267799377, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35561084747314453, + "sampling/sampling_logp_difference/mean": 0.029298899695277214, + "step": 98, + "step_time": 23.35145698580891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 593.4375, + "completions/mean_terminated_length": 593.4375, + "completions/min_length": 508.0, + "completions/min_terminated_length": 508.0, + "entropy": 1.1754724085330963, + "epoch": 0.198, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2662176787853241, + "kl": 0.02589411090593785, + "learning_rate": 3e-05, + "loss": 0.2574020326137543, + "num_tokens": 1008458.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1308085918426514, + "sampling/importance_sampling_ratio/mean": 0.6420408487319946, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42550981044769287, + "sampling/sampling_logp_difference/mean": 0.02820964716374874, + "step": 99, + "step_time": 21.02622760878876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 538.4375, + "completions/mean_terminated_length": 538.4375, + "completions/min_length": 483.0, + "completions/min_terminated_length": 483.0, + "entropy": 1.3136962801218033, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2549664378166199, + "kl": 0.024644543533213437, + "learning_rate": 3e-05, + "loss": 0.06747792661190033, + "num_tokens": 1018793.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.3327062129974365, + "sampling/importance_sampling_ratio/mean": 0.7165549993515015, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4304838180541992, + "sampling/sampling_logp_difference/mean": 0.0299112256616354, + "step": 100, + "step_time": 16.768271412234753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 567.875, + "completions/mean_terminated_length": 567.875, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.15415108948946, + "epoch": 0.202, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34537598490715027, + "kl": 0.025688456720672548, + "learning_rate": 3e-05, + "loss": -0.21041882038116455, + "num_tokens": 1029751.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.282634973526001, + "sampling/importance_sampling_ratio/mean": 0.5392330288887024, + "sampling/importance_sampling_ratio/min": 0.026465320959687233, + "sampling/sampling_logp_difference/max": 0.3903813362121582, + "sampling/sampling_logp_difference/mean": 0.02962569333612919, + "step": 101, + "step_time": 16.715499105863273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 574.125, + "completions/mean_terminated_length": 574.125, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.3766441270709038, + "epoch": 0.204, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27241969108581543, + "kl": 0.025680337683297694, + "learning_rate": 3e-05, + "loss": 0.24403473734855652, + "num_tokens": 1040601.0, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 2.3365371227264404, + "sampling/importance_sampling_ratio/mean": 0.6375491619110107, + "sampling/importance_sampling_ratio/min": 0.07846305519342422, + "sampling/sampling_logp_difference/max": 0.4158613681793213, + "sampling/sampling_logp_difference/mean": 0.030232973396778107, + "step": 102, + "step_time": 19.942134194541723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 557.75, + "completions/mean_terminated_length": 557.75, + "completions/min_length": 510.0, + "completions/min_terminated_length": 510.0, + "entropy": 1.3887510225176811, + "epoch": 0.206, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2711203992366791, + "kl": 0.022622586810030043, + "learning_rate": 3e-05, + "loss": -0.07210355252027512, + "num_tokens": 1051229.0, + "reward": 6.3125, + "reward_std": 1.0144785642623901, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.0144785642623901, + "sampling/importance_sampling_ratio/max": 1.8279200792312622, + "sampling/importance_sampling_ratio/mean": 0.5626887083053589, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3629894256591797, + "sampling/sampling_logp_difference/mean": 0.030201904475688934, + "step": 103, + "step_time": 30.44108156999573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 544.75, + "completions/mean_terminated_length": 544.75, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.1656717136502266, + "epoch": 0.208, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27351662516593933, + "kl": 0.02198210428468883, + "learning_rate": 3e-05, + "loss": 0.06065649166703224, + "num_tokens": 1061745.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.500911235809326, + "sampling/importance_sampling_ratio/mean": 0.8908482789993286, + "sampling/importance_sampling_ratio/min": 0.05167346075177193, + "sampling/sampling_logp_difference/max": 0.3885481357574463, + "sampling/sampling_logp_difference/mean": 0.027608510106801987, + "step": 104, + "step_time": 16.46229960815981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 616.0, + "completions/max_terminated_length": 616.0, + "completions/mean_length": 562.875, + "completions/mean_terminated_length": 562.875, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.3012276738882065, + "epoch": 0.21, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21975310146808624, + "kl": 0.023721053614281118, + "learning_rate": 3e-05, + "loss": 0.06463687866926193, + "num_tokens": 1072231.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 1.7785983085632324, + "sampling/importance_sampling_ratio/mean": 0.5429776906967163, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.9359657764434814, + "sampling/sampling_logp_difference/mean": 0.030092012137174606, + "step": 105, + "step_time": 24.80119998846203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 580.0625, + "completions/mean_terminated_length": 580.0625, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.3428374752402306, + "epoch": 0.212, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23242872953414917, + "kl": 0.025716557982377708, + "learning_rate": 3e-05, + "loss": -0.004262822680175304, + "num_tokens": 1083184.0, + "reward": 6.375, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.0552361011505127, + "sampling/importance_sampling_ratio/mean": 0.6959555745124817, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7246572971343994, + "sampling/sampling_logp_difference/mean": 0.02952728420495987, + "step": 106, + "step_time": 17.27699494967237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 655.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 589.75, + "completions/mean_terminated_length": 589.75, + "completions/min_length": 547.0, + "completions/min_terminated_length": 547.0, + "entropy": 1.471379242837429, + "epoch": 0.214, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23287689685821533, + "kl": 0.025674917036667466, + "learning_rate": 3e-05, + "loss": 0.08491670340299606, + "num_tokens": 1094204.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 2.1368408203125, + "sampling/importance_sampling_ratio/mean": 0.5767678022384644, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36995506286621094, + "sampling/sampling_logp_difference/mean": 0.02880111336708069, + "step": 107, + "step_time": 40.17427978478372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 595.6875, + "completions/mean_terminated_length": 595.6875, + "completions/min_length": 524.0, + "completions/min_terminated_length": 524.0, + "entropy": 1.2550728917121887, + "epoch": 0.216, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09948146343231201, + "kl": 0.022876911563798785, + "learning_rate": 3e-05, + "loss": 0.04861483350396156, + "num_tokens": 1105303.0, + "reward": 6.125, + "reward_std": 1.0878112316131592, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0878112316131592, + "sampling/importance_sampling_ratio/max": 1.1923820972442627, + "sampling/importance_sampling_ratio/mean": 0.3343955874443054, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3477153778076172, + "sampling/sampling_logp_difference/mean": 0.029913678765296936, + "step": 108, + "step_time": 18.59066634438932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 583.5, + "completions/mean_terminated_length": 583.5, + "completions/min_length": 467.0, + "completions/min_terminated_length": 467.0, + "entropy": 1.2097205966711044, + "epoch": 0.218, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1392563134431839, + "kl": 0.0263087993953377, + "learning_rate": 3e-05, + "loss": 0.10488568246364594, + "num_tokens": 1116591.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.0463244915008545, + "sampling/importance_sampling_ratio/mean": 0.4996475875377655, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31003570556640625, + "sampling/sampling_logp_difference/mean": 0.0288787130266428, + "step": 109, + "step_time": 17.845282280817628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 571.375, + "completions/mean_terminated_length": 571.375, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.28146480768919, + "epoch": 0.22, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5394540429115295, + "kl": 0.024339908617548645, + "learning_rate": 3e-05, + "loss": -0.23892748355865479, + "num_tokens": 1127493.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 2.671478509902954, + "sampling/importance_sampling_ratio/mean": 1.223832130432129, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650541305541992, + "sampling/sampling_logp_difference/mean": 0.028643080964684486, + "step": 110, + "step_time": 22.93386760680005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 493.875, + "completions/mean_terminated_length": 493.875, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 1.12203798443079, + "epoch": 0.222, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4297163188457489, + "kl": 0.024493444827385247, + "learning_rate": 3e-05, + "loss": -0.21332937479019165, + "num_tokens": 1136979.0, + "reward": 6.9375, + "reward_std": 0.6800735592842102, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.6800735592842102, + "sampling/importance_sampling_ratio/max": 2.889394760131836, + "sampling/importance_sampling_ratio/mean": 0.7321407794952393, + "sampling/importance_sampling_ratio/min": 0.02116413414478302, + "sampling/sampling_logp_difference/max": 0.49600934982299805, + "sampling/sampling_logp_difference/mean": 0.028577474877238274, + "step": 111, + "step_time": 21.056686570402235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 672.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 583.8125, + "completions/mean_terminated_length": 583.8125, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.4159239530563354, + "epoch": 0.224, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.008436380885541439, + "kl": 0.024869016953743994, + "learning_rate": 3e-05, + "loss": 0.0004943995736539364, + "num_tokens": 1148176.0, + "reward": 7.0, + "reward_std": 0.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.642366886138916, + "sampling/importance_sampling_ratio/mean": 0.7060814499855042, + "sampling/importance_sampling_ratio/min": 0.006825839169323444, + "sampling/sampling_logp_difference/max": 0.572547435760498, + "sampling/sampling_logp_difference/mean": 0.03075096383690834, + "step": 112, + "step_time": 20.1555822850205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 638.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 582.75, + "completions/mean_terminated_length": 582.75, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.2932439520955086, + "epoch": 0.226, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2439681738615036, + "kl": 0.025248280493542552, + "learning_rate": 3e-05, + "loss": -0.22745110094547272, + "num_tokens": 1159380.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.5284109115600586, + "sampling/importance_sampling_ratio/mean": 1.0150926113128662, + "sampling/importance_sampling_ratio/min": 0.021104907616972923, + "sampling/sampling_logp_difference/max": 0.2513730525970459, + "sampling/sampling_logp_difference/mean": 0.02884429693222046, + "step": 113, + "step_time": 18.09852197067812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 580.0, + "completions/mean_terminated_length": 580.0, + "completions/min_length": 535.0, + "completions/min_terminated_length": 535.0, + "entropy": 1.246352232992649, + "epoch": 0.228, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23755788803100586, + "kl": 0.02679906424600631, + "learning_rate": 3e-05, + "loss": 0.3550976812839508, + "num_tokens": 1170628.0, + "reward": 7.25, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.750225305557251, + "sampling/importance_sampling_ratio/mean": 1.0749173164367676, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5576918125152588, + "sampling/sampling_logp_difference/mean": 0.03159492090344429, + "step": 114, + "step_time": 24.629896679893136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 657.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 562.6875, + "completions/mean_terminated_length": 562.6875, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.253239594399929, + "epoch": 0.23, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3102787733078003, + "kl": 0.027133187628351152, + "learning_rate": 3e-05, + "loss": -0.05118201673030853, + "num_tokens": 1181295.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.634671926498413, + "sampling/importance_sampling_ratio/mean": 0.8949281573295593, + "sampling/importance_sampling_ratio/min": 0.09920533001422882, + "sampling/sampling_logp_difference/max": 0.6224374771118164, + "sampling/sampling_logp_difference/mean": 0.030200565233826637, + "step": 115, + "step_time": 38.715506959706545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 610.9375, + "completions/mean_terminated_length": 610.9375, + "completions/min_length": 538.0, + "completions/min_terminated_length": 538.0, + "entropy": 1.2940760999917984, + "epoch": 0.232, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14933602511882782, + "kl": 0.031228074803948402, + "learning_rate": 3e-05, + "loss": -0.05550215765833855, + "num_tokens": 1192750.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6116093397140503, + "sampling/importance_sampling_ratio/mean": 0.41488125920295715, + "sampling/importance_sampling_ratio/min": 0.009796842001378536, + "sampling/sampling_logp_difference/max": 0.5450258255004883, + "sampling/sampling_logp_difference/mean": 0.03152918070554733, + "step": 116, + "step_time": 17.821606623008847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 560.8125, + "completions/mean_terminated_length": 560.8125, + "completions/min_length": 501.0, + "completions/min_terminated_length": 501.0, + "entropy": 1.21239273250103, + "epoch": 0.234, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23778750002384186, + "kl": 0.03231424337718636, + "learning_rate": 3e-05, + "loss": -0.09421571344137192, + "num_tokens": 1203219.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.1054162979125977, + "sampling/importance_sampling_ratio/mean": 0.7292319536209106, + "sampling/importance_sampling_ratio/min": 0.010288448072969913, + "sampling/sampling_logp_difference/max": 0.8687701225280762, + "sampling/sampling_logp_difference/mean": 0.030015992000699043, + "step": 117, + "step_time": 16.80020274501294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 712.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 583.4375, + "completions/mean_terminated_length": 583.4375, + "completions/min_length": 503.0, + "completions/min_terminated_length": 503.0, + "entropy": 1.0914121232926846, + "epoch": 0.236, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.177206888794899, + "kl": 0.027808396029286087, + "learning_rate": 3e-05, + "loss": 0.010135526768863201, + "num_tokens": 1214562.0, + "reward": 6.375, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.6413226127624512, + "sampling/importance_sampling_ratio/mean": 0.5455202460289001, + "sampling/importance_sampling_ratio/min": 0.14393718540668488, + "sampling/sampling_logp_difference/max": 0.37839651107788086, + "sampling/sampling_logp_difference/mean": 0.02755727432668209, + "step": 118, + "step_time": 32.04508572584018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 802.0, + "completions/max_terminated_length": 802.0, + "completions/mean_length": 623.875, + "completions/mean_terminated_length": 623.875, + "completions/min_length": 555.0, + "completions/min_terminated_length": 555.0, + "entropy": 1.2099597677588463, + "epoch": 0.238, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10887355357408524, + "kl": 0.029803494922816753, + "learning_rate": 3e-05, + "loss": -0.05460066720843315, + "num_tokens": 1226136.0, + "reward": 6.25, + "reward_std": 1.983263373374939, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.983263373374939, + "sampling/importance_sampling_ratio/max": 1.5456031560897827, + "sampling/importance_sampling_ratio/mean": 0.4625241756439209, + "sampling/importance_sampling_ratio/min": 0.06713607162237167, + "sampling/sampling_logp_difference/max": 0.5650186538696289, + "sampling/sampling_logp_difference/mean": 0.03079008124768734, + "step": 119, + "step_time": 20.80143166380003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 621.0, + "completions/mean_terminated_length": 621.0, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "entropy": 1.3042216151952744, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1758948117494583, + "kl": 0.03042414935771376, + "learning_rate": 3e-05, + "loss": -0.1489834189414978, + "num_tokens": 1237856.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.4328413009643555, + "sampling/importance_sampling_ratio/mean": 0.7058912515640259, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40829265117645264, + "sampling/sampling_logp_difference/mean": 0.029741039499640465, + "step": 120, + "step_time": 17.90572352707386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 593.8125, + "completions/mean_terminated_length": 593.8125, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 1.132676463574171, + "epoch": 0.242, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42553070187568665, + "kl": 0.04428348131477833, + "learning_rate": 3e-05, + "loss": 0.22666211426258087, + "num_tokens": 1249173.0, + "reward": 5.625, + "reward_std": 1.8574175834655762, + "rewards/quality_reward/mean": 5.625, + "rewards/quality_reward/std": 1.8574175834655762, + "sampling/importance_sampling_ratio/max": 2.5172836780548096, + "sampling/importance_sampling_ratio/mean": 0.6919515132904053, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6260476112365723, + "sampling/sampling_logp_difference/mean": 0.030399736016988754, + "step": 121, + "step_time": 38.02521731564775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 587.625, + "completions/mean_terminated_length": 587.625, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.2395975515246391, + "epoch": 0.244, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.248436838388443, + "kl": 0.03361149993725121, + "learning_rate": 3e-05, + "loss": 0.024570390582084656, + "num_tokens": 1260463.0, + "reward": 6.0625, + "reward_std": 1.236594796180725, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.236594796180725, + "sampling/importance_sampling_ratio/max": 2.833437442779541, + "sampling/importance_sampling_ratio/mean": 0.8825340867042542, + "sampling/importance_sampling_ratio/min": 0.06326956301927567, + "sampling/sampling_logp_difference/max": 0.540553092956543, + "sampling/sampling_logp_difference/mean": 0.030399201437830925, + "step": 122, + "step_time": 17.796182619407773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 585.3125, + "completions/mean_terminated_length": 585.3125, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.375852882862091, + "epoch": 0.246, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16029156744480133, + "kl": 0.03495740657672286, + "learning_rate": 3e-05, + "loss": -0.053390923887491226, + "num_tokens": 1271644.0, + "reward": 6.4375, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 2.6618704795837402, + "sampling/importance_sampling_ratio/mean": 0.5607253909111023, + "sampling/importance_sampling_ratio/min": 0.08802779763936996, + "sampling/sampling_logp_difference/max": 0.7028732299804688, + "sampling/sampling_logp_difference/mean": 0.031593482941389084, + "step": 123, + "step_time": 14.86260191956535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 630.1875, + "completions/mean_terminated_length": 630.1875, + "completions/min_length": 571.0, + "completions/min_terminated_length": 571.0, + "entropy": 1.3353190049529076, + "epoch": 0.248, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19288285076618195, + "kl": 0.033586084260605276, + "learning_rate": 3e-05, + "loss": -0.010514559224247932, + "num_tokens": 1283351.0, + "reward": 6.625, + "reward_std": 1.0246951580047607, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 1.0246951580047607, + "sampling/importance_sampling_ratio/max": 1.4858638048171997, + "sampling/importance_sampling_ratio/mean": 0.5407211184501648, + "sampling/importance_sampling_ratio/min": 0.007580960635095835, + "sampling/sampling_logp_difference/max": 0.6886825561523438, + "sampling/sampling_logp_difference/mean": 0.030591927468776703, + "step": 124, + "step_time": 19.809663326013833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 599.875, + "completions/mean_terminated_length": 599.875, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.168940719217062, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15600983798503876, + "kl": 0.03213575447443873, + "learning_rate": 3e-05, + "loss": 0.10287950932979584, + "num_tokens": 1294997.0, + "reward": 7.0, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 1.832617163658142, + "sampling/importance_sampling_ratio/mean": 0.6814589500427246, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5988303422927856, + "sampling/sampling_logp_difference/mean": 0.030487919226288795, + "step": 125, + "step_time": 36.94939920771867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 634.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 582.25, + "completions/mean_terminated_length": 582.25, + "completions/min_length": 527.0, + "completions/min_terminated_length": 527.0, + "entropy": 1.0670793130993843, + "epoch": 0.252, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3560245931148529, + "kl": 0.025753034162335098, + "learning_rate": 3e-05, + "loss": 0.16848862171173096, + "num_tokens": 1305993.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.17899489402771, + "sampling/importance_sampling_ratio/mean": 0.7458471059799194, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7731144428253174, + "sampling/sampling_logp_difference/mean": 0.029648227617144585, + "step": 126, + "step_time": 16.284966757521033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 717.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 650.8125, + "completions/mean_terminated_length": 650.8125, + "completions/min_length": 571.0, + "completions/min_terminated_length": 571.0, + "entropy": 1.2732752412557602, + "epoch": 0.254, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18122251331806183, + "kl": 0.03214431612286717, + "learning_rate": 3e-05, + "loss": 0.16780534386634827, + "num_tokens": 1318054.0, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 1.937699556350708, + "sampling/importance_sampling_ratio/mean": 0.5485143065452576, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.0368115901947021, + "sampling/sampling_logp_difference/mean": 0.03218457102775574, + "step": 127, + "step_time": 19.58785921242088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 743.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 617.375, + "completions/mean_terminated_length": 617.375, + "completions/min_length": 530.0, + "completions/min_terminated_length": 530.0, + "entropy": 1.306506209075451, + "epoch": 0.256, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1356409788131714, + "kl": 0.026579287368804216, + "learning_rate": 3e-05, + "loss": 0.06286599487066269, + "num_tokens": 1329500.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.106013298034668, + "sampling/importance_sampling_ratio/mean": 0.4681059420108795, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.676828145980835, + "sampling/sampling_logp_difference/mean": 0.032271042466163635, + "step": 128, + "step_time": 17.268729500938207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 599.0, + "completions/mean_terminated_length": 599.0, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "entropy": 1.3493447452783585, + "epoch": 0.258, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2950517237186432, + "kl": 0.02614310395438224, + "learning_rate": 3e-05, + "loss": -0.059055861085653305, + "num_tokens": 1341020.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.7381844520568848, + "sampling/importance_sampling_ratio/mean": 0.6402126550674438, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5858409404754639, + "sampling/sampling_logp_difference/mean": 0.031067587435245514, + "step": 129, + "step_time": 28.869210730306804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 710.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 585.75, + "completions/mean_terminated_length": 585.75, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "entropy": 1.2956265732645988, + "epoch": 0.26, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42696425318717957, + "kl": 0.02683100081048906, + "learning_rate": 3e-05, + "loss": 0.003650778206065297, + "num_tokens": 1351928.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.9060652256011963, + "sampling/importance_sampling_ratio/mean": 0.8535536527633667, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4930081367492676, + "sampling/sampling_logp_difference/mean": 0.031333789229393005, + "step": 130, + "step_time": 18.11609491892159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 568.5625, + "completions/mean_terminated_length": 568.5625, + "completions/min_length": 470.0, + "completions/min_terminated_length": 470.0, + "entropy": 1.2097233161330223, + "epoch": 0.262, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20076203346252441, + "kl": 0.024291134322993457, + "learning_rate": 3e-05, + "loss": -0.01662549562752247, + "num_tokens": 1362825.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.6266331672668457, + "sampling/importance_sampling_ratio/mean": 0.6302506327629089, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.963031530380249, + "sampling/sampling_logp_difference/mean": 0.0319429412484169, + "step": 131, + "step_time": 24.060474771540612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 741.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 605.9375, + "completions/mean_terminated_length": 605.9375, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.2823583781719208, + "epoch": 0.264, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.184434711933136, + "kl": 0.02043789450544864, + "learning_rate": 3e-05, + "loss": -0.0666906088590622, + "num_tokens": 1374296.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.831458568572998, + "sampling/importance_sampling_ratio/mean": 0.5425443649291992, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.671515941619873, + "sampling/sampling_logp_difference/mean": 0.030500290915369987, + "step": 132, + "step_time": 17.10482985060662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 665.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 591.375, + "completions/mean_terminated_length": 591.375, + "completions/min_length": 513.0, + "completions/min_terminated_length": 513.0, + "entropy": 1.1699804589152336, + "epoch": 0.266, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4572683274745941, + "kl": 0.025262096198275685, + "learning_rate": 3e-05, + "loss": 0.6353421211242676, + "num_tokens": 1385318.0, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.3584084510803223, + "sampling/importance_sampling_ratio/mean": 0.7995439767837524, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3869748115539551, + "sampling/sampling_logp_difference/mean": 0.028536029160022736, + "step": 133, + "step_time": 36.004622367210686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 652.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 554.25, + "completions/mean_terminated_length": 554.25, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1336797252297401, + "epoch": 0.268, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.256655752658844, + "kl": 0.022568197746295482, + "learning_rate": 3e-05, + "loss": 0.026529084891080856, + "num_tokens": 1396234.0, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "sampling/importance_sampling_ratio/max": 1.9517148733139038, + "sampling/importance_sampling_ratio/mean": 0.8202446699142456, + "sampling/importance_sampling_ratio/min": 0.022647401317954063, + "sampling/sampling_logp_difference/max": 1.145315170288086, + "sampling/sampling_logp_difference/mean": 0.02867746911942959, + "step": 134, + "step_time": 20.057327402289957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 641.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 573.5, + "completions/mean_terminated_length": 573.5, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.1878332123160362, + "epoch": 0.27, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30302342772483826, + "kl": 0.02298387698829174, + "learning_rate": 3e-05, + "loss": 0.2014756053686142, + "num_tokens": 1407322.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.9701545238494873, + "sampling/importance_sampling_ratio/mean": 0.6178270578384399, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4479391574859619, + "sampling/sampling_logp_difference/mean": 0.03068450093269348, + "step": 135, + "step_time": 36.01238542608917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 534.0, + "completions/mean_terminated_length": 534.0, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2640416622161865, + "epoch": 0.272, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39115583896636963, + "kl": 0.020954113570041955, + "learning_rate": 3e-05, + "loss": 0.3642374873161316, + "num_tokens": 1418010.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.0828306674957275, + "sampling/importance_sampling_ratio/mean": 0.7149391770362854, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5613884925842285, + "sampling/sampling_logp_difference/mean": 0.0298798568546772, + "step": 136, + "step_time": 20.20259432308376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 538.8125, + "completions/mean_terminated_length": 538.8125, + "completions/min_length": 451.0, + "completions/min_terminated_length": 451.0, + "entropy": 1.2438515722751617, + "epoch": 0.274, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4104433059692383, + "kl": 0.02069689182098955, + "learning_rate": 3e-05, + "loss": -0.15938539803028107, + "num_tokens": 1428335.0, + "reward": 6.875, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.013305425643921, + "sampling/importance_sampling_ratio/mean": 0.5245123505592346, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.46263813972473145, + "sampling/sampling_logp_difference/mean": 0.030781995505094528, + "step": 137, + "step_time": 18.362532567232847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 542.4375, + "completions/mean_terminated_length": 542.4375, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2741251289844513, + "epoch": 0.276, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41347458958625793, + "kl": 0.020388772361911833, + "learning_rate": 3e-05, + "loss": 0.3297041654586792, + "num_tokens": 1438734.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.6671712398529053, + "sampling/importance_sampling_ratio/mean": 0.822363555431366, + "sampling/importance_sampling_ratio/min": 0.016625043004751205, + "sampling/sampling_logp_difference/max": 0.40987062454223633, + "sampling/sampling_logp_difference/mean": 0.02844768762588501, + "step": 138, + "step_time": 34.91616539563984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 516.3125, + "completions/mean_terminated_length": 516.3125, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.2030403539538383, + "epoch": 0.278, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3896540701389313, + "kl": 0.022598200594075024, + "learning_rate": 3e-05, + "loss": -0.25778308510780334, + "num_tokens": 1448611.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.9001679420471191, + "sampling/importance_sampling_ratio/mean": 0.6815162897109985, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.2820701599121094, + "sampling/sampling_logp_difference/mean": 0.027655858546495438, + "step": 139, + "step_time": 20.81112790806219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 512.8125, + "completions/mean_terminated_length": 512.8125, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "entropy": 1.124063789844513, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2338583916425705, + "kl": 0.022081921808421612, + "learning_rate": 3e-05, + "loss": 0.09288515895605087, + "num_tokens": 1458456.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.7245709896087646, + "sampling/importance_sampling_ratio/mean": 0.7086957693099976, + "sampling/importance_sampling_ratio/min": 0.13776090741157532, + "sampling/sampling_logp_difference/max": 0.4399615526199341, + "sampling/sampling_logp_difference/mean": 0.02669401653110981, + "step": 140, + "step_time": 22.541061893571168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 521.9375, + "completions/mean_terminated_length": 521.9375, + "completions/min_length": 460.0, + "completions/min_terminated_length": 460.0, + "entropy": 1.3581550270318985, + "epoch": 0.282, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1792808622121811, + "kl": 0.02498150523751974, + "learning_rate": 3e-05, + "loss": 0.08992868661880493, + "num_tokens": 1468623.0, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.6533762216567993, + "sampling/importance_sampling_ratio/mean": 0.5165826082229614, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42975759506225586, + "sampling/sampling_logp_difference/mean": 0.028398238122463226, + "step": 141, + "step_time": 20.935550182592124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 454.625, + "completions/mean_terminated_length": 454.625, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 1.2165675312280655, + "epoch": 0.284, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2593871057033539, + "kl": 0.024267837987281382, + "learning_rate": 3e-05, + "loss": 0.24750135838985443, + "num_tokens": 1477449.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 1.5698224306106567, + "sampling/importance_sampling_ratio/mean": 0.6540807485580444, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3056577444076538, + "sampling/sampling_logp_difference/mean": 0.029166901484131813, + "step": 142, + "step_time": 17.03540184069425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 490.625, + "completions/mean_terminated_length": 490.625, + "completions/min_length": 446.0, + "completions/min_terminated_length": 446.0, + "entropy": 1.1259984448552132, + "epoch": 0.286, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37285444140434265, + "kl": 0.019997276307549328, + "learning_rate": 3e-05, + "loss": 0.330167293548584, + "num_tokens": 1486931.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.156266927719116, + "sampling/importance_sampling_ratio/mean": 0.7264441251754761, + "sampling/importance_sampling_ratio/min": 0.07088703662157059, + "sampling/sampling_logp_difference/max": 0.42168426513671875, + "sampling/sampling_logp_difference/mean": 0.02719968557357788, + "step": 143, + "step_time": 15.82226228993386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 529.0625, + "completions/mean_terminated_length": 496.0666809082031, + "completions/min_length": 421.0, + "completions/min_terminated_length": 421.0, + "entropy": 1.0426596216857433, + "epoch": 0.288, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11882677674293518, + "kl": 0.027060870255809277, + "learning_rate": 3e-05, + "loss": 0.07405021786689758, + "num_tokens": 1496916.0, + "reward": 5.75, + "reward_std": 1.732050895690918, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.732050895690918, + "sampling/importance_sampling_ratio/max": 0.8768613934516907, + "sampling/importance_sampling_ratio/mean": 0.39511895179748535, + "sampling/importance_sampling_ratio/min": 0.11731626838445663, + "sampling/sampling_logp_difference/max": 0.6632819175720215, + "sampling/sampling_logp_difference/mean": 0.02569635957479477, + "step": 144, + "step_time": 21.100794151891023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 502.5625, + "completions/mean_terminated_length": 502.5625, + "completions/min_length": 452.0, + "completions/min_terminated_length": 452.0, + "entropy": 1.2609772458672523, + "epoch": 0.29, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25876912474632263, + "kl": 0.028287828317843378, + "learning_rate": 3e-05, + "loss": -0.18078479170799255, + "num_tokens": 1507157.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.90565824508667, + "sampling/importance_sampling_ratio/mean": 0.7513852119445801, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3683091402053833, + "sampling/sampling_logp_difference/mean": 0.02768835797905922, + "step": 145, + "step_time": 19.17377450503409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 586.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 478.25, + "completions/mean_terminated_length": 478.25, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "entropy": 1.2076954543590546, + "epoch": 0.292, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4150021970272064, + "kl": 0.027647150680422783, + "learning_rate": 3e-05, + "loss": -0.07925756275653839, + "num_tokens": 1516833.0, + "reward": 6.5, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.0522961616516113, + "sampling/importance_sampling_ratio/mean": 0.6499220132827759, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40656137466430664, + "sampling/sampling_logp_difference/mean": 0.028425993397831917, + "step": 146, + "step_time": 19.426104408223182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 512.0, + "completions/mean_length": 467.1875, + "completions/mean_terminated_length": 467.1875, + "completions/min_length": 396.0, + "completions/min_terminated_length": 396.0, + "entropy": 1.1012553870677948, + "epoch": 0.294, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33721745014190674, + "kl": 0.02999569766689092, + "learning_rate": 3e-05, + "loss": 0.04165569692850113, + "num_tokens": 1526028.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.507693290710449, + "sampling/importance_sampling_ratio/mean": 0.8091086149215698, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4148428440093994, + "sampling/sampling_logp_difference/mean": 0.028098180890083313, + "step": 147, + "step_time": 32.705999514088035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 467.0, + "completions/mean_terminated_length": 467.0, + "completions/min_length": 385.0, + "completions/min_terminated_length": 385.0, + "entropy": 1.2554677203297615, + "epoch": 0.296, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.26061347126960754, + "kl": 0.0399768726201728, + "learning_rate": 3e-05, + "loss": -0.018139198422431946, + "num_tokens": 1535348.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.5010173320770264, + "sampling/importance_sampling_ratio/mean": 0.6952720880508423, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35097575187683105, + "sampling/sampling_logp_difference/mean": 0.028018539771437645, + "step": 148, + "step_time": 34.92355508869514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 486.75, + "completions/mean_terminated_length": 486.75, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "entropy": 1.2563373371958733, + "epoch": 0.298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2081325203180313, + "kl": 0.03278218396008015, + "learning_rate": 3e-05, + "loss": -0.01242863480001688, + "num_tokens": 1544912.0, + "reward": 5.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 1.0925099849700928, + "sampling/importance_sampling_ratio/mean": 0.4945816695690155, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45126640796661377, + "sampling/sampling_logp_difference/mean": 0.030079778283834457, + "step": 149, + "step_time": 26.784944237209857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 571.0, + "completions/max_terminated_length": 571.0, + "completions/mean_length": 483.0625, + "completions/mean_terminated_length": 483.0625, + "completions/min_length": 366.0, + "completions/min_terminated_length": 366.0, + "entropy": 1.1289120316505432, + "epoch": 0.3, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40016695857048035, + "kl": 0.032314015785232186, + "learning_rate": 3e-05, + "loss": -0.1471974402666092, + "num_tokens": 1554417.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.876359701156616, + "sampling/importance_sampling_ratio/mean": 0.8288668394088745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.438828706741333, + "sampling/sampling_logp_difference/mean": 0.027187082916498184, + "step": 150, + "step_time": 25.687996607273817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 504.25, + "completions/mean_terminated_length": 504.25, + "completions/min_length": 436.0, + "completions/min_terminated_length": 436.0, + "entropy": 1.1718142479658127, + "epoch": 0.302, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3958420157432556, + "kl": 0.02723738143686205, + "learning_rate": 3e-05, + "loss": -0.3493230938911438, + "num_tokens": 1564101.0, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "sampling/importance_sampling_ratio/max": 2.8077433109283447, + "sampling/importance_sampling_ratio/mean": 0.7897872924804688, + "sampling/importance_sampling_ratio/min": 0.06443088501691818, + "sampling/sampling_logp_difference/max": 0.48229074478149414, + "sampling/sampling_logp_difference/mean": 0.02742597460746765, + "step": 151, + "step_time": 34.508475522045046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 519.0, + "completions/max_terminated_length": 519.0, + "completions/mean_length": 296.0, + "completions/mean_terminated_length": 296.0, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 1.0032543204724789, + "epoch": 0.304, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1944788247346878, + "kl": 0.030508540105074644, + "learning_rate": 3e-05, + "loss": -0.2918842136859894, + "num_tokens": 1570429.0, + "reward": 3.4375, + "reward_std": 3.558440685272217, + "rewards/quality_reward/mean": 3.4375, + "rewards/quality_reward/std": 3.558440685272217, + "sampling/importance_sampling_ratio/max": 1.85885751247406, + "sampling/importance_sampling_ratio/mean": 1.0381181240081787, + "sampling/importance_sampling_ratio/min": 0.1504185050725937, + "sampling/sampling_logp_difference/max": 0.4975461959838867, + "sampling/sampling_logp_difference/mean": 0.02710951678454876, + "step": 152, + "step_time": 22.075861463323236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 514.4375, + "completions/mean_terminated_length": 514.4375, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2780758067965508, + "epoch": 0.306, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4307408928871155, + "kl": 0.02628148818621412, + "learning_rate": 3e-05, + "loss": 0.33224302530288696, + "num_tokens": 1580372.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.5702013969421387, + "sampling/importance_sampling_ratio/mean": 0.8224437832832336, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48117589950561523, + "sampling/sampling_logp_difference/mean": 0.027627088129520416, + "step": 153, + "step_time": 18.07258096849546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 529.0, + "completions/max_terminated_length": 529.0, + "completions/mean_length": 467.125, + "completions/mean_terminated_length": 467.125, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.161174800246954, + "epoch": 0.308, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1959461271762848, + "kl": 0.026262085768394172, + "learning_rate": 3e-05, + "loss": 0.05762449651956558, + "num_tokens": 1589438.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.2318003177642822, + "sampling/importance_sampling_ratio/mean": 0.7094592452049255, + "sampling/importance_sampling_ratio/min": 0.11490790545940399, + "sampling/sampling_logp_difference/max": 0.43965983390808105, + "sampling/sampling_logp_difference/mean": 0.02732112817466259, + "step": 154, + "step_time": 22.3326059714891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 396.875, + "completions/mean_terminated_length": 396.875, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 1.1771309785544872, + "epoch": 0.31, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4554482400417328, + "kl": 0.03402461623772979, + "learning_rate": 3e-05, + "loss": -0.19043315947055817, + "num_tokens": 1597588.0, + "reward": 5.5625, + "reward_std": 1.3149778842926025, + "rewards/quality_reward/mean": 5.5625, + "rewards/quality_reward/std": 1.3149778842926025, + "sampling/importance_sampling_ratio/max": 2.1980347633361816, + "sampling/importance_sampling_ratio/mean": 0.7672010064125061, + "sampling/importance_sampling_ratio/min": 0.10123267024755478, + "sampling/sampling_logp_difference/max": 0.5363807678222656, + "sampling/sampling_logp_difference/mean": 0.029761571437120438, + "step": 155, + "step_time": 18.508908156771213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 520.5, + "completions/mean_terminated_length": 520.5, + "completions/min_length": 422.0, + "completions/min_terminated_length": 422.0, + "entropy": 1.123583823442459, + "epoch": 0.312, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20946863293647766, + "kl": 0.02810170315206051, + "learning_rate": 3e-05, + "loss": 0.07504862546920776, + "num_tokens": 1607508.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.429632306098938, + "sampling/importance_sampling_ratio/mean": 0.631747305393219, + "sampling/importance_sampling_ratio/min": 0.016474967822432518, + "sampling/sampling_logp_difference/max": 0.40722954273223877, + "sampling/sampling_logp_difference/mean": 0.028051164001226425, + "step": 156, + "step_time": 17.622006124351174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 623.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 471.8125, + "completions/mean_terminated_length": 471.8125, + "completions/min_length": 374.0, + "completions/min_terminated_length": 374.0, + "entropy": 1.0345656536519527, + "epoch": 0.314, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3838019073009491, + "kl": 0.029524097801186144, + "learning_rate": 3e-05, + "loss": 0.29842275381088257, + "num_tokens": 1617121.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.64119291305542, + "sampling/importance_sampling_ratio/mean": 0.5510131120681763, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.48647427558898926, + "sampling/sampling_logp_difference/mean": 0.02675374038517475, + "step": 157, + "step_time": 34.35223956173286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 479.3125, + "completions/mean_terminated_length": 479.3125, + "completions/min_length": 415.0, + "completions/min_terminated_length": 415.0, + "entropy": 1.197593629360199, + "epoch": 0.316, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14107273519039154, + "kl": 0.02758750319480896, + "learning_rate": 3e-05, + "loss": 0.07440078258514404, + "num_tokens": 1626462.0, + "reward": 6.1875, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.719329833984375, + "sampling/importance_sampling_ratio/mean": 0.515890896320343, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37659645080566406, + "sampling/sampling_logp_difference/mean": 0.02727513015270233, + "step": 158, + "step_time": 30.49356387415901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 465.4375, + "completions/mean_terminated_length": 465.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.182745985686779, + "epoch": 0.318, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22129040956497192, + "kl": 0.02928700065240264, + "learning_rate": 3e-05, + "loss": 0.0027256053872406483, + "num_tokens": 1635613.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.2551939487457275, + "sampling/importance_sampling_ratio/mean": 0.4655284583568573, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3532288074493408, + "sampling/sampling_logp_difference/mean": 0.027347734197974205, + "step": 159, + "step_time": 23.563114238902926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 480.9375, + "completions/mean_terminated_length": 480.9375, + "completions/min_length": 378.0, + "completions/min_terminated_length": 378.0, + "entropy": 1.0412059053778648, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22958238422870636, + "kl": 0.025641236337833107, + "learning_rate": 3e-05, + "loss": -0.1119004413485527, + "num_tokens": 1645060.0, + "reward": 5.8125, + "reward_std": 2.9033026695251465, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 2.9033026695251465, + "sampling/importance_sampling_ratio/max": 2.3590943813323975, + "sampling/importance_sampling_ratio/mean": 0.6600984334945679, + "sampling/importance_sampling_ratio/min": 0.16755303740501404, + "sampling/sampling_logp_difference/max": 0.3485531806945801, + "sampling/sampling_logp_difference/mean": 0.02425791323184967, + "step": 160, + "step_time": 17.564059282653034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 511.5625, + "completions/mean_terminated_length": 511.5625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.0806752033531666, + "epoch": 0.322, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2202390879392624, + "kl": 0.03178418125025928, + "learning_rate": 3e-05, + "loss": -0.24258574843406677, + "num_tokens": 1654901.0, + "reward": 6.5, + "reward_std": 0.9660918116569519, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.9660918116569519, + "sampling/importance_sampling_ratio/max": 2.616337776184082, + "sampling/importance_sampling_ratio/mean": 0.8924014568328857, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37401676177978516, + "sampling/sampling_logp_difference/mean": 0.02693682350218296, + "step": 161, + "step_time": 22.71096785319969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 484.1875, + "completions/mean_terminated_length": 484.1875, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.2182030156254768, + "epoch": 0.324, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2978271245956421, + "kl": 0.026724245748482645, + "learning_rate": 3e-05, + "loss": -0.0895138755440712, + "num_tokens": 1664568.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.0902533531188965, + "sampling/importance_sampling_ratio/mean": 0.9070306420326233, + "sampling/importance_sampling_ratio/min": 0.20267778635025024, + "sampling/sampling_logp_difference/max": 0.3564906120300293, + "sampling/sampling_logp_difference/mean": 0.02701719105243683, + "step": 162, + "step_time": 24.52080715773627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 543.0, + "completions/max_terminated_length": 543.0, + "completions/mean_length": 506.9375, + "completions/mean_terminated_length": 506.9375, + "completions/min_length": 452.0, + "completions/min_terminated_length": 452.0, + "entropy": 1.2944460734724998, + "epoch": 0.326, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13530702888965607, + "kl": 0.03180140187032521, + "learning_rate": 3e-05, + "loss": 0.016086462885141373, + "num_tokens": 1674423.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.1556015014648438, + "sampling/importance_sampling_ratio/mean": 0.583191990852356, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.1959445476531982, + "sampling/sampling_logp_difference/mean": 0.028113799169659615, + "step": 163, + "step_time": 21.659780140966177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 525.0, + "completions/max_terminated_length": 525.0, + "completions/mean_length": 490.5625, + "completions/mean_terminated_length": 490.5625, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.289131723344326, + "epoch": 0.328, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5087841153144836, + "kl": 0.026518055819906294, + "learning_rate": 3e-05, + "loss": 0.2851857542991638, + "num_tokens": 1683864.0, + "reward": 6.5, + "reward_std": 1.0327955484390259, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.0327955484390259, + "sampling/importance_sampling_ratio/max": 2.531486988067627, + "sampling/importance_sampling_ratio/mean": 0.8939677476882935, + "sampling/importance_sampling_ratio/min": 0.09362189471721649, + "sampling/sampling_logp_difference/max": 0.38115930557250977, + "sampling/sampling_logp_difference/mean": 0.028977636247873306, + "step": 164, + "step_time": 22.67840038659051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 570.0, + "completions/max_terminated_length": 570.0, + "completions/mean_length": 508.5, + "completions/mean_terminated_length": 508.5, + "completions/min_length": 465.0, + "completions/min_terminated_length": 465.0, + "entropy": 1.2583990097045898, + "epoch": 0.33, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5895075798034668, + "kl": 0.03340678755193949, + "learning_rate": 3e-05, + "loss": 0.5024052858352661, + "num_tokens": 1693592.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.5420279502868652, + "sampling/importance_sampling_ratio/mean": 0.8359720706939697, + "sampling/importance_sampling_ratio/min": 0.13951139152050018, + "sampling/sampling_logp_difference/max": 0.25212621688842773, + "sampling/sampling_logp_difference/mean": 0.027791393920779228, + "step": 165, + "step_time": 23.054075544700027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 522.625, + "completions/mean_terminated_length": 522.625, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.2723611742258072, + "epoch": 0.332, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4107528030872345, + "kl": 0.028830039431340992, + "learning_rate": 3e-05, + "loss": 0.43730953335762024, + "num_tokens": 1703722.0, + "reward": 7.25, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 1.8613929748535156, + "sampling/importance_sampling_ratio/mean": 0.5325981378555298, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33945512771606445, + "sampling/sampling_logp_difference/mean": 0.028407979756593704, + "step": 166, + "step_time": 14.466566514223814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 544.0, + "completions/max_terminated_length": 544.0, + "completions/mean_length": 485.4375, + "completions/mean_terminated_length": 485.4375, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.159641794860363, + "epoch": 0.334, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23576849699020386, + "kl": 0.03369407169520855, + "learning_rate": 3e-05, + "loss": -0.17192532122135162, + "num_tokens": 1713129.0, + "reward": 6.9375, + "reward_std": 0.8539125919342041, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.8539125919342041, + "sampling/importance_sampling_ratio/max": 2.364237070083618, + "sampling/importance_sampling_ratio/mean": 0.835480809211731, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4001603126525879, + "sampling/sampling_logp_difference/mean": 0.027760744094848633, + "step": 167, + "step_time": 15.427942908834666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 531.125, + "completions/mean_terminated_length": 531.125, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.1074568964540958, + "epoch": 0.336, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14523518085479736, + "kl": 0.032465216936543584, + "learning_rate": 3e-05, + "loss": -0.10911832749843597, + "num_tokens": 1723371.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.233501672744751, + "sampling/importance_sampling_ratio/mean": 0.5449534058570862, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42547130584716797, + "sampling/sampling_logp_difference/mean": 0.026400625705718994, + "step": 168, + "step_time": 14.74156094249338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 629.0, + "completions/max_terminated_length": 629.0, + "completions/mean_length": 518.25, + "completions/mean_terminated_length": 518.25, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.2571639120578766, + "epoch": 0.338, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3119359314441681, + "kl": 0.03448521322570741, + "learning_rate": 3e-05, + "loss": 0.14656513929367065, + "num_tokens": 1733279.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.882887601852417, + "sampling/importance_sampling_ratio/mean": 0.8524343967437744, + "sampling/importance_sampling_ratio/min": 0.061056625097990036, + "sampling/sampling_logp_difference/max": 0.34301328659057617, + "sampling/sampling_logp_difference/mean": 0.027896685525774956, + "step": 169, + "step_time": 22.03652939805761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 552.8125, + "completions/mean_terminated_length": 552.8125, + "completions/min_length": 454.0, + "completions/min_terminated_length": 454.0, + "entropy": 1.2771715074777603, + "epoch": 0.34, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23340070247650146, + "kl": 0.03460722556337714, + "learning_rate": 3e-05, + "loss": -0.11407067626714706, + "num_tokens": 1743740.0, + "reward": 6.8125, + "reward_std": 0.6551081538200378, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.6551081538200378, + "sampling/importance_sampling_ratio/max": 1.6628351211547852, + "sampling/importance_sampling_ratio/mean": 0.5497220754623413, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.314577579498291, + "sampling/sampling_logp_difference/mean": 0.03021315485239029, + "step": 170, + "step_time": 18.62061845092103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 564.0, + "completions/max_terminated_length": 564.0, + "completions/mean_length": 495.1875, + "completions/mean_terminated_length": 495.1875, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "entropy": 1.1425480283796787, + "epoch": 0.342, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07674646377563477, + "kl": 0.0316173325991258, + "learning_rate": 3e-05, + "loss": -0.012545892037451267, + "num_tokens": 1753231.0, + "reward": 6.4375, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.37643563747406, + "sampling/importance_sampling_ratio/mean": 0.4176323413848877, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5454483032226562, + "sampling/sampling_logp_difference/mean": 0.027837729081511497, + "step": 171, + "step_time": 35.69278695946559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 536.5, + "completions/mean_terminated_length": 536.5, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "entropy": 1.3311709240078926, + "epoch": 0.344, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5406383275985718, + "kl": 0.03873496490996331, + "learning_rate": 3e-05, + "loss": 0.2167063057422638, + "num_tokens": 1763511.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.74458646774292, + "sampling/importance_sampling_ratio/mean": 1.1003804206848145, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5733523368835449, + "sampling/sampling_logp_difference/mean": 0.028938323259353638, + "step": 172, + "step_time": 19.092736863531172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 508.0625, + "completions/mean_terminated_length": 508.0625, + "completions/min_length": 437.0, + "completions/min_terminated_length": 437.0, + "entropy": 1.22428647428751, + "epoch": 0.346, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11082098633050919, + "kl": 0.03200511261820793, + "learning_rate": 3e-05, + "loss": 0.07203174382448196, + "num_tokens": 1773304.0, + "reward": 7.125, + "reward_std": 0.5, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8160909414291382, + "sampling/importance_sampling_ratio/mean": 0.5755537748336792, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.595893383026123, + "sampling/sampling_logp_difference/mean": 0.028249088674783707, + "step": 173, + "step_time": 36.1855756030418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 720.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 556.9375, + "completions/mean_terminated_length": 556.9375, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.1718761064112186, + "epoch": 0.348, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3865050971508026, + "kl": 0.03591357555706054, + "learning_rate": 3e-05, + "loss": 0.2996499538421631, + "num_tokens": 1784039.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.0816619396209717, + "sampling/importance_sampling_ratio/mean": 0.5800511240959167, + "sampling/importance_sampling_ratio/min": 0.09306051582098007, + "sampling/sampling_logp_difference/max": 0.41143274307250977, + "sampling/sampling_logp_difference/mean": 0.027585921809077263, + "step": 174, + "step_time": 20.269209342077374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 494.6875, + "completions/mean_terminated_length": 494.6875, + "completions/min_length": 442.0, + "completions/min_terminated_length": 442.0, + "entropy": 1.3337711468338966, + "epoch": 0.35, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11888998746871948, + "kl": 0.03842530632391572, + "learning_rate": 3e-05, + "loss": 0.07849398255348206, + "num_tokens": 1793682.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.3145290613174438, + "sampling/importance_sampling_ratio/mean": 0.4274219870567322, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4447822570800781, + "sampling/sampling_logp_difference/mean": 0.031242625787854195, + "step": 175, + "step_time": 17.293509372044355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 544.8125, + "completions/mean_terminated_length": 544.8125, + "completions/min_length": 462.0, + "completions/min_terminated_length": 462.0, + "entropy": 1.19626072794199, + "epoch": 0.352, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2991865277290344, + "kl": 0.036185722798109055, + "learning_rate": 3e-05, + "loss": 0.11461115628480911, + "num_tokens": 1804039.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5985729694366455, + "sampling/importance_sampling_ratio/mean": 0.5072454810142517, + "sampling/importance_sampling_ratio/min": 0.07339605689048767, + "sampling/sampling_logp_difference/max": 0.3649592399597168, + "sampling/sampling_logp_difference/mean": 0.026338135823607445, + "step": 176, + "step_time": 40.05168053135276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 488.0625, + "completions/mean_terminated_length": 488.0625, + "completions/min_length": 419.0, + "completions/min_terminated_length": 419.0, + "entropy": 1.2391329184174538, + "epoch": 0.354, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21207627654075623, + "kl": 0.036609378294087946, + "learning_rate": 3e-05, + "loss": 0.11777876317501068, + "num_tokens": 1813440.0, + "reward": 7.0, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.3352530002593994, + "sampling/importance_sampling_ratio/mean": 0.6533275246620178, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35143423080444336, + "sampling/sampling_logp_difference/mean": 0.027743803337216377, + "step": 177, + "step_time": 33.59382761735469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 570.0, + "completions/max_terminated_length": 570.0, + "completions/mean_length": 495.125, + "completions/mean_terminated_length": 495.125, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "entropy": 1.3244052603840828, + "epoch": 0.356, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2076549232006073, + "kl": 0.03601150680333376, + "learning_rate": 3e-05, + "loss": -0.20823253691196442, + "num_tokens": 1823018.0, + "reward": 7.0, + "reward_std": 1.095445156097412, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 1.095445156097412, + "sampling/importance_sampling_ratio/max": 2.9768388271331787, + "sampling/importance_sampling_ratio/mean": 0.8220031261444092, + "sampling/importance_sampling_ratio/min": 0.08218635618686676, + "sampling/sampling_logp_difference/max": 0.3826625347137451, + "sampling/sampling_logp_difference/mean": 0.02992408722639084, + "step": 178, + "step_time": 19.288791641127318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 497.875, + "completions/mean_terminated_length": 497.875, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.2238815650343895, + "epoch": 0.358, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5341953039169312, + "kl": 0.0400471081957221, + "learning_rate": 3e-05, + "loss": 0.06042468547821045, + "num_tokens": 1832736.0, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.1878349781036377, + "sampling/importance_sampling_ratio/mean": 1.0708422660827637, + "sampling/importance_sampling_ratio/min": 0.037978146225214005, + "sampling/sampling_logp_difference/max": 0.5151598453521729, + "sampling/sampling_logp_difference/mean": 0.029129499569535255, + "step": 179, + "step_time": 17.113372664432973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 541.6875, + "completions/mean_terminated_length": 541.6875, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.4256544262170792, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21081708371639252, + "kl": 0.03922082995995879, + "learning_rate": 3e-05, + "loss": -0.10235662013292313, + "num_tokens": 1843203.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 1.1632962226867676, + "sampling/importance_sampling_ratio/mean": 0.4484874904155731, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31549549102783203, + "sampling/sampling_logp_difference/mean": 0.02854372188448906, + "step": 180, + "step_time": 38.8069160906598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 541.0, + "completions/mean_terminated_length": 541.0, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.2127383947372437, + "epoch": 0.362, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28472819924354553, + "kl": 0.039078159374184906, + "learning_rate": 3e-05, + "loss": -0.1838480830192566, + "num_tokens": 1853555.0, + "reward": 6.1875, + "reward_std": 0.6551081538200378, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.6551081538200378, + "sampling/importance_sampling_ratio/max": 1.6971478462219238, + "sampling/importance_sampling_ratio/mean": 0.5436820983886719, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4628629684448242, + "sampling/sampling_logp_difference/mean": 0.028904814273118973, + "step": 181, + "step_time": 22.927347922697663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 521.1875, + "completions/mean_terminated_length": 521.1875, + "completions/min_length": 481.0, + "completions/min_terminated_length": 481.0, + "entropy": 1.205168604850769, + "epoch": 0.364, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40580859780311584, + "kl": 0.03652556415181607, + "learning_rate": 3e-05, + "loss": 0.11645574867725372, + "num_tokens": 1863670.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.6977338790893555, + "sampling/importance_sampling_ratio/mean": 0.7627977132797241, + "sampling/importance_sampling_ratio/min": 0.0672435387969017, + "sampling/sampling_logp_difference/max": 0.42972230911254883, + "sampling/sampling_logp_difference/mean": 0.02727590501308441, + "step": 182, + "step_time": 17.11851307330653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 501.6875, + "completions/mean_terminated_length": 501.6875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2342532575130463, + "epoch": 0.366, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30436721444129944, + "kl": 0.04035243031103164, + "learning_rate": 3e-05, + "loss": 0.07254824787378311, + "num_tokens": 1873353.0, + "reward": 7.0, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 1.7811214923858643, + "sampling/importance_sampling_ratio/mean": 0.6635246276855469, + "sampling/importance_sampling_ratio/min": 0.05660989508032799, + "sampling/sampling_logp_difference/max": 0.3192565441131592, + "sampling/sampling_logp_difference/mean": 0.028735067695379257, + "step": 183, + "step_time": 15.842315018642694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 540.625, + "completions/mean_terminated_length": 540.625, + "completions/min_length": 488.0, + "completions/min_terminated_length": 488.0, + "entropy": 1.3501724749803543, + "epoch": 0.368, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15960462391376495, + "kl": 0.04095173126552254, + "learning_rate": 3e-05, + "loss": -0.020260833203792572, + "num_tokens": 1883907.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 1.9560747146606445, + "sampling/importance_sampling_ratio/mean": 0.6561183929443359, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.9810755252838135, + "sampling/sampling_logp_difference/mean": 0.029474109411239624, + "step": 184, + "step_time": 27.30614952277392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 532.6875, + "completions/mean_terminated_length": 532.6875, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "entropy": 1.3397118523716927, + "epoch": 0.37, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19153976440429688, + "kl": 0.045232257107272744, + "learning_rate": 3e-05, + "loss": 0.07327698916196823, + "num_tokens": 1894262.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 2.3196122646331787, + "sampling/importance_sampling_ratio/mean": 0.7404133677482605, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.378201961517334, + "sampling/sampling_logp_difference/mean": 0.02841799519956112, + "step": 185, + "step_time": 17.66303364513442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 626.0, + "completions/max_terminated_length": 626.0, + "completions/mean_length": 547.875, + "completions/mean_terminated_length": 547.875, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.4480287805199623, + "epoch": 0.372, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1921214759349823, + "kl": 0.04523745132610202, + "learning_rate": 3e-05, + "loss": -0.1904258131980896, + "num_tokens": 1904748.0, + "reward": 6.1875, + "reward_std": 1.0468206405639648, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.0468206405639648, + "sampling/importance_sampling_ratio/max": 2.1734814643859863, + "sampling/importance_sampling_ratio/mean": 0.8759132027626038, + "sampling/importance_sampling_ratio/min": 0.1473371982574463, + "sampling/sampling_logp_difference/max": 0.43239259719848633, + "sampling/sampling_logp_difference/mean": 0.030228231102228165, + "step": 186, + "step_time": 15.321936973370612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 522.5, + "completions/mean_terminated_length": 522.5, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.1799098625779152, + "epoch": 0.374, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1446281522512436, + "kl": 0.04069687286391854, + "learning_rate": 3e-05, + "loss": -0.005613957531750202, + "num_tokens": 1915044.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1428344249725342, + "sampling/importance_sampling_ratio/mean": 0.49870407581329346, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3580789566040039, + "sampling/sampling_logp_difference/mean": 0.028958076611161232, + "step": 187, + "step_time": 20.772348938975483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 526.6875, + "completions/mean_terminated_length": 526.6875, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.1814791783690453, + "epoch": 0.376, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09093533456325531, + "kl": 0.048393386183306575, + "learning_rate": 3e-05, + "loss": -0.09858276695013046, + "num_tokens": 1925055.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.5353080034255981, + "sampling/importance_sampling_ratio/mean": 0.47519102692604065, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.455765962600708, + "sampling/sampling_logp_difference/mean": 0.02906900830566883, + "step": 188, + "step_time": 19.42380119021982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 663.0, + "completions/max_terminated_length": 663.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 538.5, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "entropy": 1.2316770479083061, + "epoch": 0.378, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20003275573253632, + "kl": 0.04744360945187509, + "learning_rate": 3e-05, + "loss": -0.16722381114959717, + "num_tokens": 1935415.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.9816064834594727, + "sampling/importance_sampling_ratio/mean": 0.680183470249176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4760274887084961, + "sampling/sampling_logp_difference/mean": 0.028748787939548492, + "step": 189, + "step_time": 19.73181858472526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 555.125, + "completions/mean_terminated_length": 555.125, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.198552466928959, + "epoch": 0.38, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13213400542736053, + "kl": 0.04299823543988168, + "learning_rate": 3e-05, + "loss": 0.04805057868361473, + "num_tokens": 1945985.0, + "reward": 6.625, + "reward_std": 0.9574271440505981, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.9574271440505981, + "sampling/importance_sampling_ratio/max": 1.563953161239624, + "sampling/importance_sampling_ratio/mean": 0.41058292984962463, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3399984836578369, + "sampling/sampling_logp_difference/mean": 0.0274125337600708, + "step": 190, + "step_time": 17.826407154556364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 684.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 539.25, + "completions/mean_terminated_length": 539.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.179109387099743, + "epoch": 0.382, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2910110354423523, + "kl": 0.046097030164673924, + "learning_rate": 3e-05, + "loss": -0.3787975311279297, + "num_tokens": 1956445.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.4134271144866943, + "sampling/importance_sampling_ratio/mean": 0.8701735734939575, + "sampling/importance_sampling_ratio/min": 0.1316290944814682, + "sampling/sampling_logp_difference/max": 0.3755226135253906, + "sampling/sampling_logp_difference/mean": 0.029267774894833565, + "step": 191, + "step_time": 24.135659996420145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 540.0, + "completions/mean_terminated_length": 540.0, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.2047618329524994, + "epoch": 0.384, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20911987125873566, + "kl": 0.04525213362649083, + "learning_rate": 3e-05, + "loss": 0.013828473165631294, + "num_tokens": 1966693.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 1.9282022714614868, + "sampling/importance_sampling_ratio/mean": 0.5034524202346802, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4313013553619385, + "sampling/sampling_logp_difference/mean": 0.02878478728234768, + "step": 192, + "step_time": 14.677200393751264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 597.0, + "completions/max_terminated_length": 597.0, + "completions/mean_length": 553.125, + "completions/mean_terminated_length": 553.125, + "completions/min_length": 501.0, + "completions/min_terminated_length": 501.0, + "entropy": 1.1857876032590866, + "epoch": 0.386, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3986797034740448, + "kl": 0.04699963750317693, + "learning_rate": 3e-05, + "loss": -0.06190801039338112, + "num_tokens": 1977311.0, + "reward": 7.0, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.877324104309082, + "sampling/importance_sampling_ratio/mean": 0.9780403971672058, + "sampling/importance_sampling_ratio/min": 0.02624017745256424, + "sampling/sampling_logp_difference/max": 0.7719376087188721, + "sampling/sampling_logp_difference/mean": 0.02950127050280571, + "step": 193, + "step_time": 22.189579842612147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 531.5625, + "completions/mean_terminated_length": 531.5625, + "completions/min_length": 465.0, + "completions/min_terminated_length": 465.0, + "entropy": 1.3334204703569412, + "epoch": 0.388, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22821271419525146, + "kl": 0.04149021068587899, + "learning_rate": 3e-05, + "loss": -0.15963155031204224, + "num_tokens": 1987680.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 2.790942907333374, + "sampling/importance_sampling_ratio/mean": 0.7382668256759644, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45777153968811035, + "sampling/sampling_logp_difference/mean": 0.029790451750159264, + "step": 194, + "step_time": 21.228061076253653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 505.0625, + "completions/mean_terminated_length": 505.0625, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.2570307329297066, + "epoch": 0.39, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3617555797100067, + "kl": 0.042452862951904535, + "learning_rate": 3e-05, + "loss": -0.2362610548734665, + "num_tokens": 1997481.0, + "reward": 6.625, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.3454530239105225, + "sampling/importance_sampling_ratio/mean": 0.9099248647689819, + "sampling/importance_sampling_ratio/min": 0.09889467060565948, + "sampling/sampling_logp_difference/max": 0.6245463490486145, + "sampling/sampling_logp_difference/mean": 0.029685894027352333, + "step": 195, + "step_time": 17.880568680819124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 544.6875, + "completions/mean_terminated_length": 544.6875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.208221659064293, + "epoch": 0.392, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35804808139801025, + "kl": 0.03823408088646829, + "learning_rate": 3e-05, + "loss": 0.026877164840698242, + "num_tokens": 2007860.0, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 2.670585870742798, + "sampling/importance_sampling_ratio/mean": 0.6493271589279175, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4234185218811035, + "sampling/sampling_logp_difference/mean": 0.02899312786757946, + "step": 196, + "step_time": 20.456977635622025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 582.875, + "completions/mean_terminated_length": 582.875, + "completions/min_length": 517.0, + "completions/min_terminated_length": 517.0, + "entropy": 1.3732100576162338, + "epoch": 0.394, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.05589874088764191, + "kl": 0.03486072865780443, + "learning_rate": 3e-05, + "loss": -0.019679736346006393, + "num_tokens": 2018946.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.1570472717285156, + "sampling/importance_sampling_ratio/mean": 0.35759687423706055, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.33884429931640625, + "sampling/sampling_logp_difference/mean": 0.029303058981895447, + "step": 197, + "step_time": 16.522779263556004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 625.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 533.5, + "completions/mean_terminated_length": 533.5, + "completions/min_length": 479.0, + "completions/min_terminated_length": 479.0, + "entropy": 1.1664377599954605, + "epoch": 0.396, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23638494312763214, + "kl": 0.037777561345137656, + "learning_rate": 3e-05, + "loss": 0.0447416789829731, + "num_tokens": 2029178.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.484117865562439, + "sampling/importance_sampling_ratio/mean": 0.539449155330658, + "sampling/importance_sampling_ratio/min": 0.09831889718770981, + "sampling/sampling_logp_difference/max": 0.37561988830566406, + "sampling/sampling_logp_difference/mean": 0.029459550976753235, + "step": 198, + "step_time": 30.065524043980986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 536.0625, + "completions/mean_terminated_length": 536.0625, + "completions/min_length": 472.0, + "completions/min_terminated_length": 472.0, + "entropy": 1.254080481827259, + "epoch": 0.398, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1923029124736786, + "kl": 0.03572200902272016, + "learning_rate": 3e-05, + "loss": -0.09766081720590591, + "num_tokens": 2039347.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.6362762451171875, + "sampling/importance_sampling_ratio/mean": 0.5547572374343872, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35316991806030273, + "sampling/sampling_logp_difference/mean": 0.027488065883517265, + "step": 199, + "step_time": 30.96936017088592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 551.5, + "completions/mean_terminated_length": 551.5, + "completions/min_length": 505.0, + "completions/min_terminated_length": 505.0, + "entropy": 1.2724409252405167, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22953365743160248, + "kl": 0.036497756373137236, + "learning_rate": 3e-05, + "loss": 0.014221633784472942, + "num_tokens": 2049795.0, + "reward": 6.5625, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.4539027214050293, + "sampling/importance_sampling_ratio/mean": 0.6678089499473572, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7396450042724609, + "sampling/sampling_logp_difference/mean": 0.0300765261054039, + "step": 200, + "step_time": 16.165886579081416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 582.0, + "completions/max_terminated_length": 582.0, + "completions/mean_length": 528.0, + "completions/mean_terminated_length": 528.0, + "completions/min_length": 474.0, + "completions/min_terminated_length": 474.0, + "entropy": 1.120336215943098, + "epoch": 0.402, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0991974025964737, + "kl": 0.03168696933425963, + "learning_rate": 3e-05, + "loss": -0.13335926830768585, + "num_tokens": 2059963.0, + "reward": 6.375, + "reward_std": 1.7841898202896118, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.7841898202896118, + "sampling/importance_sampling_ratio/max": 2.3520584106445312, + "sampling/importance_sampling_ratio/mean": 0.6451135277748108, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5111579895019531, + "sampling/sampling_logp_difference/mean": 0.027531839907169342, + "step": 201, + "step_time": 38.76227374607697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 635.0, + "completions/max_terminated_length": 635.0, + "completions/mean_length": 549.9375, + "completions/mean_terminated_length": 549.9375, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.2026560828089714, + "epoch": 0.404, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07962016016244888, + "kl": 0.040914483717642725, + "learning_rate": 3e-05, + "loss": 0.04318992793560028, + "num_tokens": 2070626.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.161137580871582, + "sampling/importance_sampling_ratio/mean": 0.265199214220047, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42658185958862305, + "sampling/sampling_logp_difference/mean": 0.0287276990711689, + "step": 202, + "step_time": 20.940971142146736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 566.0, + "completions/max_terminated_length": 566.0, + "completions/mean_length": 511.1875, + "completions/mean_terminated_length": 511.1875, + "completions/min_length": 460.0, + "completions/min_terminated_length": 460.0, + "entropy": 1.2832268327474594, + "epoch": 0.406, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1376999020576477, + "kl": 0.03862898051738739, + "learning_rate": 3e-05, + "loss": -0.14832699298858643, + "num_tokens": 2080861.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.544222831726074, + "sampling/importance_sampling_ratio/mean": 0.7174543142318726, + "sampling/importance_sampling_ratio/min": 0.18541352450847626, + "sampling/sampling_logp_difference/max": 0.3160414695739746, + "sampling/sampling_logp_difference/mean": 0.02949603646993637, + "step": 203, + "step_time": 20.54771270370111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 605.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 524.25, + "completions/mean_terminated_length": 524.25, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "entropy": 1.2392274662852287, + "epoch": 0.408, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12836101651191711, + "kl": 0.032396848779171705, + "learning_rate": 3e-05, + "loss": -0.010008644312620163, + "num_tokens": 2090953.0, + "reward": 7.125, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.204956531524658, + "sampling/importance_sampling_ratio/mean": 0.5054515600204468, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.29064249992370605, + "sampling/sampling_logp_difference/mean": 0.028712285682559013, + "step": 204, + "step_time": 18.479188771452755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 708.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 566.625, + "completions/mean_terminated_length": 566.625, + "completions/min_length": 482.0, + "completions/min_terminated_length": 482.0, + "entropy": 1.3167504370212555, + "epoch": 0.41, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2566336691379547, + "kl": 0.03499211696907878, + "learning_rate": 3e-05, + "loss": 0.053824737668037415, + "num_tokens": 2101795.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2913551330566406, + "sampling/importance_sampling_ratio/mean": 0.6769458651542664, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4542531967163086, + "sampling/sampling_logp_difference/mean": 0.027578134089708328, + "step": 205, + "step_time": 17.109014553949237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 506.25, + "completions/mean_terminated_length": 506.25, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "entropy": 1.244155466556549, + "epoch": 0.412, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3224847912788391, + "kl": 0.03367950115352869, + "learning_rate": 3e-05, + "loss": -0.2610609829425812, + "num_tokens": 2111719.0, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.0974948406219482, + "sampling/importance_sampling_ratio/mean": 0.5962464809417725, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5845310688018799, + "sampling/sampling_logp_difference/mean": 0.02760414592921734, + "step": 206, + "step_time": 12.972559538204223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 623.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 515.125, + "completions/mean_terminated_length": 515.125, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.2021623700857162, + "epoch": 0.414, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22504572570323944, + "kl": 0.039928025915287435, + "learning_rate": 3e-05, + "loss": 0.08153954148292542, + "num_tokens": 2121913.0, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 1.6359574794769287, + "sampling/importance_sampling_ratio/mean": 0.4843714237213135, + "sampling/importance_sampling_ratio/min": 0.16781684756278992, + "sampling/sampling_logp_difference/max": 0.3475990295410156, + "sampling/sampling_logp_difference/mean": 0.029437636956572533, + "step": 207, + "step_time": 43.64637131197378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 613.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 512.3125, + "completions/mean_terminated_length": 512.3125, + "completions/min_length": 421.0, + "completions/min_terminated_length": 421.0, + "entropy": 1.2001312859356403, + "epoch": 0.416, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3592093884944916, + "kl": 0.036205250886268914, + "learning_rate": 3e-05, + "loss": 0.34191110730171204, + "num_tokens": 2131806.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.192653179168701, + "sampling/importance_sampling_ratio/mean": 0.6557403206825256, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3398885726928711, + "sampling/sampling_logp_difference/mean": 0.027714602649211884, + "step": 208, + "step_time": 14.901265816297382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 513.1875, + "completions/mean_terminated_length": 513.1875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2848069965839386, + "epoch": 0.418, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.47909337282180786, + "kl": 0.02966410096269101, + "learning_rate": 3e-05, + "loss": 0.2796367406845093, + "num_tokens": 2141849.0, + "reward": 6.1875, + "reward_std": 1.0468206405639648, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.0468206405639648, + "sampling/importance_sampling_ratio/max": 2.964437484741211, + "sampling/importance_sampling_ratio/mean": 0.48602545261383057, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3722946047782898, + "sampling/sampling_logp_difference/mean": 0.029922205954790115, + "step": 209, + "step_time": 26.673682311549783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 507.9375, + "completions/mean_terminated_length": 507.9375, + "completions/min_length": 422.0, + "completions/min_terminated_length": 422.0, + "entropy": 1.2064250856637955, + "epoch": 0.42, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3124609887599945, + "kl": 0.038097753771580756, + "learning_rate": 3e-05, + "loss": 0.07772707939147949, + "num_tokens": 2151864.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.737311840057373, + "sampling/importance_sampling_ratio/mean": 0.9012110233306885, + "sampling/importance_sampling_ratio/min": 0.052471309900283813, + "sampling/sampling_logp_difference/max": 0.36536455154418945, + "sampling/sampling_logp_difference/mean": 0.027899259701371193, + "step": 210, + "step_time": 23.180102336220443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 525.0, + "completions/max_terminated_length": 525.0, + "completions/mean_length": 480.9375, + "completions/mean_terminated_length": 480.9375, + "completions/min_length": 436.0, + "completions/min_terminated_length": 436.0, + "entropy": 1.2063737213611603, + "epoch": 0.422, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.44815266132354736, + "kl": 0.027290108264423907, + "learning_rate": 3e-05, + "loss": 0.28776273131370544, + "num_tokens": 2161039.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.063380479812622, + "sampling/importance_sampling_ratio/mean": 0.8898900151252747, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.32479190826416016, + "sampling/sampling_logp_difference/mean": 0.02677021361887455, + "step": 211, + "step_time": 22.177836938295513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 599.0, + "completions/max_terminated_length": 599.0, + "completions/mean_length": 504.625, + "completions/mean_terminated_length": 504.625, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.3087149783968925, + "epoch": 0.424, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35702478885650635, + "kl": 0.031013125786557794, + "learning_rate": 3e-05, + "loss": 0.17483392357826233, + "num_tokens": 2170961.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.4756224155426025, + "sampling/importance_sampling_ratio/mean": 0.7680925130844116, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35749173164367676, + "sampling/sampling_logp_difference/mean": 0.027991417795419693, + "step": 212, + "step_time": 17.408967671450227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 797.0, + "completions/max_terminated_length": 797.0, + "completions/mean_length": 640.8125, + "completions/mean_terminated_length": 640.8125, + "completions/min_length": 457.0, + "completions/min_terminated_length": 457.0, + "entropy": 1.3978670835494995, + "epoch": 0.426, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.29086264967918396, + "kl": 0.024261576938442886, + "learning_rate": 3e-05, + "loss": 0.09740053862333298, + "num_tokens": 2182910.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 1.893927812576294, + "sampling/importance_sampling_ratio/mean": 0.7117372155189514, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3868746757507324, + "sampling/sampling_logp_difference/mean": 0.028743820264935493, + "step": 213, + "step_time": 23.464720248244703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 486.75, + "completions/mean_terminated_length": 486.75, + "completions/min_length": 404.0, + "completions/min_terminated_length": 404.0, + "entropy": 1.2355968467891216, + "epoch": 0.428, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2767268419265747, + "kl": 0.02811512805055827, + "learning_rate": 3e-05, + "loss": 0.18954241275787354, + "num_tokens": 2192242.0, + "reward": 7.0625, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.0872268676757812, + "sampling/importance_sampling_ratio/mean": 0.8847656846046448, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3631856441497803, + "sampling/sampling_logp_difference/mean": 0.02797355316579342, + "step": 214, + "step_time": 15.314252337440848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 503.1875, + "completions/mean_terminated_length": 503.1875, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.2383001297712326, + "epoch": 0.43, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1873309463262558, + "kl": 0.03220478829462081, + "learning_rate": 3e-05, + "loss": -0.01179824024438858, + "num_tokens": 2202045.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 2.573594808578491, + "sampling/importance_sampling_ratio/mean": 0.7044014930725098, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6166388988494873, + "sampling/sampling_logp_difference/mean": 0.028071925044059753, + "step": 215, + "step_time": 43.65747703285888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 565.0, + "completions/max_terminated_length": 565.0, + "completions/mean_length": 503.9375, + "completions/mean_terminated_length": 503.9375, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.1677803546190262, + "epoch": 0.432, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.09634320437908173, + "kl": 0.03325538453646004, + "learning_rate": 3e-05, + "loss": 0.03834616392850876, + "num_tokens": 2212188.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 1.9774657487869263, + "sampling/importance_sampling_ratio/mean": 0.5414069890975952, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35487866401672363, + "sampling/sampling_logp_difference/mean": 0.027339771389961243, + "step": 216, + "step_time": 17.274593455251306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 650.0, + "completions/max_terminated_length": 650.0, + "completions/mean_length": 343.75, + "completions/mean_terminated_length": 343.75, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "entropy": 1.0582842603325844, + "epoch": 0.434, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3425586521625519, + "kl": 0.03517636051401496, + "learning_rate": 3e-05, + "loss": 0.11844412982463837, + "num_tokens": 2219440.0, + "reward": 3.5, + "reward_std": 3.265986442565918, + "rewards/quality_reward/mean": 3.5, + "rewards/quality_reward/std": 3.265986442565918, + "sampling/importance_sampling_ratio/max": 1.4673620462417603, + "sampling/importance_sampling_ratio/mean": 0.781898021697998, + "sampling/importance_sampling_ratio/min": 0.08477991074323654, + "sampling/sampling_logp_difference/max": 0.3881380558013916, + "sampling/sampling_logp_difference/mean": 0.02829045243561268, + "step": 217, + "step_time": 13.36990327714011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 471.4375, + "completions/mean_terminated_length": 471.4375, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "entropy": 1.1179756224155426, + "epoch": 0.436, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33612945675849915, + "kl": 0.030886436812579632, + "learning_rate": 3e-05, + "loss": 0.006120521575212479, + "num_tokens": 2228975.0, + "reward": 6.5, + "reward_std": 0.632455587387085, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "sampling/importance_sampling_ratio/max": 2.1587650775909424, + "sampling/importance_sampling_ratio/mean": 0.7267376780509949, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.32509779930114746, + "sampling/sampling_logp_difference/mean": 0.02783289924263954, + "step": 218, + "step_time": 17.99441510764882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 520.125, + "completions/mean_terminated_length": 520.125, + "completions/min_length": 462.0, + "completions/min_terminated_length": 462.0, + "entropy": 1.2453451082110405, + "epoch": 0.438, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2976357340812683, + "kl": 0.03020153450779617, + "learning_rate": 3e-05, + "loss": 0.1648026406764984, + "num_tokens": 2238953.0, + "reward": 5.3125, + "reward_std": 1.078192949295044, + "rewards/quality_reward/mean": 5.3125, + "rewards/quality_reward/std": 1.078192949295044, + "sampling/importance_sampling_ratio/max": 1.6686924695968628, + "sampling/importance_sampling_ratio/mean": 0.7682108879089355, + "sampling/importance_sampling_ratio/min": 0.06810324639081955, + "sampling/sampling_logp_difference/max": 0.3546750545501709, + "sampling/sampling_logp_difference/mean": 0.028889676555991173, + "step": 219, + "step_time": 20.669593635946512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 552.0, + "completions/max_terminated_length": 552.0, + "completions/mean_length": 465.375, + "completions/mean_terminated_length": 465.375, + "completions/min_length": 356.0, + "completions/min_terminated_length": 356.0, + "entropy": 1.3331433907151222, + "epoch": 0.44, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13799847662448883, + "kl": 0.03191920532844961, + "learning_rate": 3e-05, + "loss": -0.133737251162529, + "num_tokens": 2248199.0, + "reward": 6.0, + "reward_std": 0.9660918116569519, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.9660918116569519, + "sampling/importance_sampling_ratio/max": 2.8963325023651123, + "sampling/importance_sampling_ratio/mean": 0.5178577899932861, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35057830810546875, + "sampling/sampling_logp_difference/mean": 0.027748603373765945, + "step": 220, + "step_time": 21.38788841944188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 507.5, + "completions/mean_terminated_length": 507.5, + "completions/min_length": 474.0, + "completions/min_terminated_length": 474.0, + "entropy": 1.241542100906372, + "epoch": 0.442, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3648691773414612, + "kl": 0.03521239408291876, + "learning_rate": 3e-05, + "loss": 0.2634640634059906, + "num_tokens": 2258327.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.1063547134399414, + "sampling/importance_sampling_ratio/mean": 0.6709499359130859, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3389136791229248, + "sampling/sampling_logp_difference/mean": 0.02875763736665249, + "step": 221, + "step_time": 30.28709344472736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 459.375, + "completions/mean_terminated_length": 459.375, + "completions/min_length": 405.0, + "completions/min_terminated_length": 405.0, + "entropy": 1.141789611428976, + "epoch": 0.444, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1954081803560257, + "kl": 0.03689368430059403, + "learning_rate": 3e-05, + "loss": 0.13035088777542114, + "num_tokens": 2267381.0, + "reward": 6.5625, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.681690216064453, + "sampling/importance_sampling_ratio/mean": 0.6949984431266785, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.31617891788482666, + "sampling/sampling_logp_difference/mean": 0.02665363810956478, + "step": 222, + "step_time": 37.694539529737085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 571.0, + "completions/max_terminated_length": 571.0, + "completions/mean_length": 515.5625, + "completions/mean_terminated_length": 515.5625, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.2330311760306358, + "epoch": 0.446, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27568548917770386, + "kl": 0.032993816188536584, + "learning_rate": 3e-05, + "loss": 0.017291374504566193, + "num_tokens": 2277446.0, + "reward": 7.125, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.847236156463623, + "sampling/importance_sampling_ratio/mean": 0.7807494401931763, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.49445104598999023, + "sampling/sampling_logp_difference/mean": 0.027980424463748932, + "step": 223, + "step_time": 18.57380611775443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 634.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 523.1875, + "completions/mean_terminated_length": 523.1875, + "completions/min_length": 451.0, + "completions/min_terminated_length": 451.0, + "entropy": 1.0975877195596695, + "epoch": 0.448, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10739865154027939, + "kl": 0.02909247507341206, + "learning_rate": 3e-05, + "loss": 0.05642539635300636, + "num_tokens": 2287345.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.9240283966064453, + "sampling/importance_sampling_ratio/mean": 0.6011937856674194, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40827369689941406, + "sampling/sampling_logp_difference/mean": 0.027437299489974976, + "step": 224, + "step_time": 39.087660535704345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 533.0, + "completions/max_terminated_length": 533.0, + "completions/mean_length": 478.25, + "completions/mean_terminated_length": 478.25, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "entropy": 1.3173525258898735, + "epoch": 0.45, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22121335566043854, + "kl": 0.03540586424060166, + "learning_rate": 3e-05, + "loss": -0.02710402011871338, + "num_tokens": 2296741.0, + "reward": 6.4375, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.8824238777160645, + "sampling/importance_sampling_ratio/mean": 0.6062434315681458, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4316587448120117, + "sampling/sampling_logp_difference/mean": 0.02776467800140381, + "step": 225, + "step_time": 26.200199087150395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 499.9375, + "completions/mean_terminated_length": 499.9375, + "completions/min_length": 448.0, + "completions/min_terminated_length": 448.0, + "entropy": 1.1501530036330223, + "epoch": 0.452, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3865528702735901, + "kl": 0.03646970179397613, + "learning_rate": 3e-05, + "loss": 0.19160562753677368, + "num_tokens": 2306676.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 2.6609690189361572, + "sampling/importance_sampling_ratio/mean": 0.9618603587150574, + "sampling/importance_sampling_ratio/min": 0.0957244485616684, + "sampling/sampling_logp_difference/max": 0.35040283203125, + "sampling/sampling_logp_difference/mean": 0.028557972982525826, + "step": 226, + "step_time": 17.752630488947034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 510.75, + "completions/mean_terminated_length": 510.75, + "completions/min_length": 419.0, + "completions/min_terminated_length": 419.0, + "entropy": 1.190872348845005, + "epoch": 0.454, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1407802700996399, + "kl": 0.030792692443355918, + "learning_rate": 3e-05, + "loss": 0.04028765484690666, + "num_tokens": 2316696.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 1.2131233215332031, + "sampling/importance_sampling_ratio/mean": 0.6032290458679199, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3711676597595215, + "sampling/sampling_logp_difference/mean": 0.027024609968066216, + "step": 227, + "step_time": 27.143527323380113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 495.625, + "completions/mean_terminated_length": 495.625, + "completions/min_length": 419.0, + "completions/min_terminated_length": 419.0, + "entropy": 1.2335442155599594, + "epoch": 0.456, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18564435839653015, + "kl": 0.025341857108287513, + "learning_rate": 3e-05, + "loss": -0.26227492094039917, + "num_tokens": 2326322.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.640001058578491, + "sampling/importance_sampling_ratio/mean": 0.6159635782241821, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.34752869606018066, + "sampling/sampling_logp_difference/mean": 0.027370886877179146, + "step": 228, + "step_time": 22.369792928919196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 528.0, + "completions/max_terminated_length": 528.0, + "completions/mean_length": 487.6875, + "completions/mean_terminated_length": 487.6875, + "completions/min_length": 449.0, + "completions/min_terminated_length": 449.0, + "entropy": 1.2502131089568138, + "epoch": 0.458, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2313818782567978, + "kl": 0.03635518567170948, + "learning_rate": 3e-05, + "loss": -0.031097467988729477, + "num_tokens": 2335973.0, + "reward": 6.9375, + "reward_std": 0.8539125919342041, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.8539125919342041, + "sampling/importance_sampling_ratio/max": 2.227858543395996, + "sampling/importance_sampling_ratio/mean": 0.7488094568252563, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3627934455871582, + "sampling/sampling_logp_difference/mean": 0.02758944220840931, + "step": 229, + "step_time": 30.443659604527056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 476.375, + "completions/mean_terminated_length": 476.375, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 1.2348124124109745, + "epoch": 0.46, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5150085687637329, + "kl": 0.032692725653760135, + "learning_rate": 3e-05, + "loss": 0.5464498996734619, + "num_tokens": 2345579.0, + "reward": 6.8125, + "reward_std": 0.6551081538200378, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.6551081538200378, + "sampling/importance_sampling_ratio/max": 2.376497268676758, + "sampling/importance_sampling_ratio/mean": 0.7721551060676575, + "sampling/importance_sampling_ratio/min": 0.1553211510181427, + "sampling/sampling_logp_difference/max": 0.42708492279052734, + "sampling/sampling_logp_difference/mean": 0.026741618290543556, + "step": 230, + "step_time": 24.71390812145546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 517.5, + "completions/mean_terminated_length": 517.5, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "entropy": 1.1868174076080322, + "epoch": 0.462, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.11509153991937637, + "kl": 0.03900455019902438, + "learning_rate": 3e-05, + "loss": -0.1646902710199356, + "num_tokens": 2355499.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.2603800296783447, + "sampling/importance_sampling_ratio/mean": 0.6012319326400757, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4672989845275879, + "sampling/sampling_logp_difference/mean": 0.027583574876189232, + "step": 231, + "step_time": 23.276649605948478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 691.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 560.875, + "completions/mean_terminated_length": 560.875, + "completions/min_length": 494.0, + "completions/min_terminated_length": 494.0, + "entropy": 1.1533633023500443, + "epoch": 0.464, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17417575418949127, + "kl": 0.031763071776367724, + "learning_rate": 3e-05, + "loss": 0.00344286416657269, + "num_tokens": 2366049.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.206153154373169, + "sampling/importance_sampling_ratio/mean": 0.740157961845398, + "sampling/importance_sampling_ratio/min": 0.13027621805667877, + "sampling/sampling_logp_difference/max": 0.4202132225036621, + "sampling/sampling_logp_difference/mean": 0.028019659221172333, + "step": 232, + "step_time": 20.686087283305824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 500.8125, + "completions/mean_terminated_length": 500.8125, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.2008480802178383, + "epoch": 0.466, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22023369371891022, + "kl": 0.03284288931172341, + "learning_rate": 3e-05, + "loss": 0.03899282589554787, + "num_tokens": 2375582.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.0323734283447266, + "sampling/importance_sampling_ratio/mean": 0.6224058270454407, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4571092128753662, + "sampling/sampling_logp_difference/mean": 0.027327092364430428, + "step": 233, + "step_time": 26.555491346865892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 845.0, + "completions/max_terminated_length": 845.0, + "completions/mean_length": 587.5, + "completions/mean_terminated_length": 587.5, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.2540696933865547, + "epoch": 0.468, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13053849339485168, + "kl": 0.03163444856181741, + "learning_rate": 3e-05, + "loss": -0.06800927221775055, + "num_tokens": 2386702.0, + "reward": 6.0, + "reward_std": 1.7511900663375854, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.7511900663375854, + "sampling/importance_sampling_ratio/max": 1.4020758867263794, + "sampling/importance_sampling_ratio/mean": 0.4879741370677948, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5877337455749512, + "sampling/sampling_logp_difference/mean": 0.02704041451215744, + "step": 234, + "step_time": 34.3772664074786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 506.375, + "completions/mean_terminated_length": 506.375, + "completions/min_length": 422.0, + "completions/min_terminated_length": 422.0, + "entropy": 1.3376594334840775, + "epoch": 0.47, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1838671714067459, + "kl": 0.03747367626056075, + "learning_rate": 3e-05, + "loss": -0.1597842425107956, + "num_tokens": 2396564.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.9093716144561768, + "sampling/importance_sampling_ratio/mean": 0.7693536281585693, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37027931213378906, + "sampling/sampling_logp_difference/mean": 0.029480738565325737, + "step": 235, + "step_time": 14.542957273777574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 605.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 509.6875, + "completions/mean_terminated_length": 509.6875, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "entropy": 1.3714017421007156, + "epoch": 0.472, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.44443222880363464, + "kl": 0.03565627557691187, + "learning_rate": 3e-05, + "loss": 0.24737706780433655, + "num_tokens": 2406551.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.7392280101776123, + "sampling/importance_sampling_ratio/mean": 0.7888213396072388, + "sampling/importance_sampling_ratio/min": 0.04889443516731262, + "sampling/sampling_logp_difference/max": 0.29999852180480957, + "sampling/sampling_logp_difference/mean": 0.028956102207303047, + "step": 236, + "step_time": 19.966124589089304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 541.1875, + "completions/mean_terminated_length": 541.1875, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.199029415845871, + "epoch": 0.474, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1285122185945511, + "kl": 0.03105375764425844, + "learning_rate": 3e-05, + "loss": -0.006456274073570967, + "num_tokens": 2416754.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 1.395982265472412, + "sampling/importance_sampling_ratio/mean": 0.5822510719299316, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4087851047515869, + "sampling/sampling_logp_difference/mean": 0.027147701010107994, + "step": 237, + "step_time": 31.801921805832535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 772.0, + "completions/max_terminated_length": 772.0, + "completions/mean_length": 598.1875, + "completions/mean_terminated_length": 598.1875, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.2089053243398666, + "epoch": 0.476, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3568721413612366, + "kl": 0.03257777914404869, + "learning_rate": 3e-05, + "loss": 0.3960018455982208, + "num_tokens": 2428005.0, + "reward": 6.25, + "reward_std": 1.1254628896713257, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.1254628896713257, + "sampling/importance_sampling_ratio/max": 1.9572224617004395, + "sampling/importance_sampling_ratio/mean": 0.5545582175254822, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42554450035095215, + "sampling/sampling_logp_difference/mean": 0.027511969208717346, + "step": 238, + "step_time": 25.979049060028046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 580.0, + "completions/max_terminated_length": 580.0, + "completions/mean_length": 508.875, + "completions/mean_terminated_length": 508.875, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.4715757966041565, + "epoch": 0.478, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07262199372053146, + "kl": 0.03327966062352061, + "learning_rate": 3e-05, + "loss": 0.025357680395245552, + "num_tokens": 2437779.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.7364466190338135, + "sampling/importance_sampling_ratio/mean": 0.8696970343589783, + "sampling/importance_sampling_ratio/min": 0.07304152101278305, + "sampling/sampling_logp_difference/max": 0.37055206298828125, + "sampling/sampling_logp_difference/mean": 0.029029743745923042, + "step": 239, + "step_time": 25.994311626069248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 580.0, + "completions/max_terminated_length": 580.0, + "completions/mean_length": 527.9375, + "completions/mean_terminated_length": 527.9375, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "entropy": 1.311545416712761, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28445860743522644, + "kl": 0.03817834367509931, + "learning_rate": 3e-05, + "loss": 0.12445695698261261, + "num_tokens": 2448202.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.700259804725647, + "sampling/importance_sampling_ratio/mean": 0.5472592711448669, + "sampling/importance_sampling_ratio/min": 0.052414167672395706, + "sampling/sampling_logp_difference/max": 0.5368318557739258, + "sampling/sampling_logp_difference/mean": 0.02976268343627453, + "step": 240, + "step_time": 22.84421144844964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 598.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 543.3125, + "completions/mean_terminated_length": 543.3125, + "completions/min_length": 471.0, + "completions/min_terminated_length": 471.0, + "entropy": 1.180833749473095, + "epoch": 0.482, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16590596735477448, + "kl": 0.03646332467906177, + "learning_rate": 3e-05, + "loss": 0.04229091852903366, + "num_tokens": 2458743.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.728398323059082, + "sampling/importance_sampling_ratio/mean": 0.9457916021347046, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40506887435913086, + "sampling/sampling_logp_difference/mean": 0.02782438136637211, + "step": 241, + "step_time": 25.389156353194267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 579.0, + "completions/max_terminated_length": 579.0, + "completions/mean_length": 502.3125, + "completions/mean_terminated_length": 502.3125, + "completions/min_length": 454.0, + "completions/min_terminated_length": 454.0, + "entropy": 1.1694707348942757, + "epoch": 0.484, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.25544801354408264, + "kl": 0.03445987973827869, + "learning_rate": 3e-05, + "loss": -0.017443601042032242, + "num_tokens": 2468492.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 1.8124310970306396, + "sampling/importance_sampling_ratio/mean": 0.8106446266174316, + "sampling/importance_sampling_ratio/min": 0.08132059127092361, + "sampling/sampling_logp_difference/max": 0.6171557903289795, + "sampling/sampling_logp_difference/mean": 0.027156969532370567, + "step": 242, + "step_time": 24.112746777944267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 654.0, + "completions/max_terminated_length": 654.0, + "completions/mean_length": 538.125, + "completions/mean_terminated_length": 538.125, + "completions/min_length": 448.0, + "completions/min_terminated_length": 448.0, + "entropy": 1.297831729054451, + "epoch": 0.486, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17728154361248016, + "kl": 0.03608768491540104, + "learning_rate": 3e-05, + "loss": -0.030910439789295197, + "num_tokens": 2478878.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.7654427289962769, + "sampling/importance_sampling_ratio/mean": 0.5417827367782593, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3394327163696289, + "sampling/sampling_logp_difference/mean": 0.02802959457039833, + "step": 243, + "step_time": 39.02764433948323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 703.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 564.75, + "completions/mean_terminated_length": 564.75, + "completions/min_length": 449.0, + "completions/min_terminated_length": 449.0, + "entropy": 1.2894479781389236, + "epoch": 0.488, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1783561259508133, + "kl": 0.04178670607507229, + "learning_rate": 3e-05, + "loss": 0.010581104084849358, + "num_tokens": 2489938.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 1.9127941131591797, + "sampling/importance_sampling_ratio/mean": 0.6222037672996521, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.8026522397994995, + "sampling/sampling_logp_difference/mean": 0.028042398393154144, + "step": 244, + "step_time": 22.52857542503625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 599.0, + "completions/max_terminated_length": 599.0, + "completions/mean_length": 511.875, + "completions/mean_terminated_length": 511.875, + "completions/min_length": 455.0, + "completions/min_terminated_length": 455.0, + "entropy": 1.1273594908416271, + "epoch": 0.49, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34433481097221375, + "kl": 0.04671380412764847, + "learning_rate": 3e-05, + "loss": 0.06864061206579208, + "num_tokens": 2499760.0, + "reward": 7.1875, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.993313789367676, + "sampling/importance_sampling_ratio/mean": 0.9394024014472961, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3724935054779053, + "sampling/sampling_logp_difference/mean": 0.027506975457072258, + "step": 245, + "step_time": 24.86254589399323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 684.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 533.0, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 467.0, + "completions/min_terminated_length": 467.0, + "entropy": 1.2337471172213554, + "epoch": 0.492, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2073342353105545, + "kl": 0.03919998917263001, + "learning_rate": 3e-05, + "loss": -0.00970650464296341, + "num_tokens": 2510176.0, + "reward": 6.8125, + "reward_std": 0.75, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.6556100845336914, + "sampling/importance_sampling_ratio/mean": 0.6703793406486511, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5199446678161621, + "sampling/sampling_logp_difference/mean": 0.028828633949160576, + "step": 246, + "step_time": 27.641962222289294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 636.0, + "completions/max_terminated_length": 636.0, + "completions/mean_length": 555.8125, + "completions/mean_terminated_length": 555.8125, + "completions/min_length": 493.0, + "completions/min_terminated_length": 493.0, + "entropy": 1.1313174478709698, + "epoch": 0.494, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10448901355266571, + "kl": 0.043822019128128886, + "learning_rate": 3e-05, + "loss": -0.060149889439344406, + "num_tokens": 2520981.0, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "sampling/importance_sampling_ratio/max": 1.087956428527832, + "sampling/importance_sampling_ratio/mean": 0.4310106039047241, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36969757080078125, + "sampling/sampling_logp_difference/mean": 0.028163518756628036, + "step": 247, + "step_time": 27.198071955237538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 533.1875, + "completions/mean_terminated_length": 533.1875, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "entropy": 1.2226731404662132, + "epoch": 0.496, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21327541768550873, + "kl": 0.0436656444799155, + "learning_rate": 3e-05, + "loss": -0.15169084072113037, + "num_tokens": 2531056.0, + "reward": 6.4375, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.145639657974243, + "sampling/importance_sampling_ratio/mean": 0.924071729183197, + "sampling/importance_sampling_ratio/min": 0.042581744492053986, + "sampling/sampling_logp_difference/max": 0.5983643531799316, + "sampling/sampling_logp_difference/mean": 0.028493624180555344, + "step": 248, + "step_time": 46.37140509998426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 675.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 580.5625, + "completions/mean_terminated_length": 580.5625, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "entropy": 1.2647478878498077, + "epoch": 0.498, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1687636375427246, + "kl": 0.03778462728951126, + "learning_rate": 3e-05, + "loss": 0.024922871962189674, + "num_tokens": 2542129.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 2.1956820487976074, + "sampling/importance_sampling_ratio/mean": 0.6349776983261108, + "sampling/importance_sampling_ratio/min": 0.11655592173337936, + "sampling/sampling_logp_difference/max": 0.35921406745910645, + "sampling/sampling_logp_difference/mean": 0.029055560007691383, + "step": 249, + "step_time": 34.322586783673614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 694.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 543.6875, + "completions/mean_terminated_length": 543.6875, + "completions/min_length": 451.0, + "completions/min_terminated_length": 451.0, + "entropy": 1.2015386000275612, + "epoch": 0.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3467749357223511, + "kl": 0.03919053066056222, + "learning_rate": 3e-05, + "loss": -0.24095430970191956, + "num_tokens": 2552412.0, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.5754284858703613, + "sampling/importance_sampling_ratio/mean": 0.9581880569458008, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3446807861328125, + "sampling/sampling_logp_difference/mean": 0.028353629633784294, + "step": 250, + "step_time": 26.97987106954679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 749.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 584.5, + "completions/mean_terminated_length": 584.5, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.1965860426425934, + "epoch": 0.502, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3573401868343353, + "kl": 0.03822207520715892, + "learning_rate": 3e-05, + "loss": -0.1708906590938568, + "num_tokens": 2563580.0, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "sampling/importance_sampling_ratio/max": 2.4531548023223877, + "sampling/importance_sampling_ratio/mean": 0.8432164788246155, + "sampling/importance_sampling_ratio/min": 0.02091093361377716, + "sampling/sampling_logp_difference/max": 0.35509490966796875, + "sampling/sampling_logp_difference/mean": 0.02920239232480526, + "step": 251, + "step_time": 24.99323159083724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 565.0, + "completions/max_terminated_length": 565.0, + "completions/mean_length": 503.375, + "completions/mean_terminated_length": 503.375, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "entropy": 1.2068623006343842, + "epoch": 0.504, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12554219365119934, + "kl": 0.04616628657095134, + "learning_rate": 3e-05, + "loss": -0.07006160914897919, + "num_tokens": 2573370.0, + "reward": 6.9375, + "reward_std": 0.6800735592842102, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.6800735592842102, + "sampling/importance_sampling_ratio/max": 2.634645462036133, + "sampling/importance_sampling_ratio/mean": 0.7688348293304443, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.37473583221435547, + "sampling/sampling_logp_difference/mean": 0.029184387996792793, + "step": 252, + "step_time": 43.27480686130002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 645.0, + "completions/max_terminated_length": 645.0, + "completions/mean_length": 540.625, + "completions/mean_terminated_length": 540.625, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.2164383009076118, + "epoch": 0.506, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19425006210803986, + "kl": 0.040457896422594786, + "learning_rate": 3e-05, + "loss": 0.06332479417324066, + "num_tokens": 2583780.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 1.2259461879730225, + "sampling/importance_sampling_ratio/mean": 0.5209584832191467, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4301719665527344, + "sampling/sampling_logp_difference/mean": 0.028582781553268433, + "step": 253, + "step_time": 30.769911186769605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 626.0, + "completions/max_terminated_length": 626.0, + "completions/mean_length": 452.25, + "completions/mean_terminated_length": 452.25, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 1.2369564026594162, + "epoch": 0.508, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20388974249362946, + "kl": 0.043233372969552875, + "learning_rate": 3e-05, + "loss": -0.03424276039004326, + "num_tokens": 2592624.0, + "reward": 7.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 7.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.8529531955718994, + "sampling/importance_sampling_ratio/mean": 0.6690866947174072, + "sampling/importance_sampling_ratio/min": 0.06710651516914368, + "sampling/sampling_logp_difference/max": 0.2922825813293457, + "sampling/sampling_logp_difference/mean": 0.028338422998785973, + "step": 254, + "step_time": 18.368686333298683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 678.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 593.25, + "completions/mean_terminated_length": 593.25, + "completions/min_length": 494.0, + "completions/min_terminated_length": 494.0, + "entropy": 1.2358134537935257, + "epoch": 0.51, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3238426446914673, + "kl": 0.040533376624807715, + "learning_rate": 3e-05, + "loss": -0.26449277997016907, + "num_tokens": 2603772.0, + "reward": 6.9375, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 1.9910115003585815, + "sampling/importance_sampling_ratio/mean": 0.544727087020874, + "sampling/importance_sampling_ratio/min": 0.024249335750937462, + "sampling/sampling_logp_difference/max": 0.5587451457977295, + "sampling/sampling_logp_difference/mean": 0.02828345075249672, + "step": 255, + "step_time": 28.741963400039822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 639.0, + "completions/max_terminated_length": 639.0, + "completions/mean_length": 541.6875, + "completions/mean_terminated_length": 541.6875, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 1.2090466022491455, + "epoch": 0.512, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24497906863689423, + "kl": 0.035602600779384375, + "learning_rate": 3e-05, + "loss": 0.1972789317369461, + "num_tokens": 2614015.0, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "sampling/importance_sampling_ratio/max": 2.615037679672241, + "sampling/importance_sampling_ratio/mean": 0.6314806342124939, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4399089813232422, + "sampling/sampling_logp_difference/mean": 0.028692016378045082, + "step": 256, + "step_time": 26.5418023574166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 765.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 608.0625, + "completions/mean_terminated_length": 608.0625, + "completions/min_length": 465.0, + "completions/min_terminated_length": 465.0, + "entropy": 1.4103579595685005, + "epoch": 0.514, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23706457018852234, + "kl": 0.0430363982450217, + "learning_rate": 3e-05, + "loss": 0.08096523582935333, + "num_tokens": 2625568.0, + "reward": 6.9375, + "reward_std": 0.25, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.320277214050293, + "sampling/importance_sampling_ratio/mean": 0.4297117590904236, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40811336040496826, + "sampling/sampling_logp_difference/mean": 0.029812760651111603, + "step": 257, + "step_time": 30.523871788289398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 660.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 573.6875, + "completions/mean_terminated_length": 573.6875, + "completions/min_length": 508.0, + "completions/min_terminated_length": 508.0, + "entropy": 1.2737382426857948, + "epoch": 0.516, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3034888505935669, + "kl": 0.044887167401611805, + "learning_rate": 3e-05, + "loss": 0.02335459366440773, + "num_tokens": 2636443.0, + "reward": 6.4375, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 1.9408409595489502, + "sampling/importance_sampling_ratio/mean": 0.6644032597541809, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3945488929748535, + "sampling/sampling_logp_difference/mean": 0.028646407648921013, + "step": 258, + "step_time": 24.04052680823952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 692.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 574.0, + "completions/mean_terminated_length": 574.0, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.1968051716685295, + "epoch": 0.518, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15292863547801971, + "kl": 0.03639746748376638, + "learning_rate": 3e-05, + "loss": -0.16838416457176208, + "num_tokens": 2647171.0, + "reward": 7.1875, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.4874011278152466, + "sampling/importance_sampling_ratio/mean": 0.4699386954307556, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3294107913970947, + "sampling/sampling_logp_difference/mean": 0.028790012001991272, + "step": 259, + "step_time": 24.238027889747173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 546.25, + "completions/mean_terminated_length": 546.25, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "entropy": 1.1241988725960255, + "epoch": 0.52, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.303418904542923, + "kl": 0.04550225310958922, + "learning_rate": 3e-05, + "loss": 0.18645404279232025, + "num_tokens": 2657583.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.91849422454834, + "sampling/importance_sampling_ratio/mean": 0.6766756772994995, + "sampling/importance_sampling_ratio/min": 0.03731733188033104, + "sampling/sampling_logp_difference/max": 0.3931598663330078, + "sampling/sampling_logp_difference/mean": 0.027895493432879448, + "step": 260, + "step_time": 23.580054661724716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 604.0, + "completions/max_terminated_length": 604.0, + "completions/mean_length": 559.1875, + "completions/mean_terminated_length": 559.1875, + "completions/min_length": 526.0, + "completions/min_terminated_length": 526.0, + "entropy": 1.2270803824067116, + "epoch": 0.522, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20534871518611908, + "kl": 0.05870963633060455, + "learning_rate": 3e-05, + "loss": -0.01714285835623741, + "num_tokens": 2668042.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.1291964054107666, + "sampling/importance_sampling_ratio/mean": 0.8631129264831543, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3395042419433594, + "sampling/sampling_logp_difference/mean": 0.029195168986916542, + "step": 261, + "step_time": 23.93122593825683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 592.0, + "completions/max_terminated_length": 592.0, + "completions/mean_length": 515.25, + "completions/mean_terminated_length": 515.25, + "completions/min_length": 328.0, + "completions/min_terminated_length": 328.0, + "entropy": 1.372651383280754, + "epoch": 0.524, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3122904300689697, + "kl": 0.044736934592947364, + "learning_rate": 3e-05, + "loss": 0.10568767786026001, + "num_tokens": 2678318.0, + "reward": 6.25, + "reward_std": 1.5705626010894775, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.5705626010894775, + "sampling/importance_sampling_ratio/max": 2.7710750102996826, + "sampling/importance_sampling_ratio/mean": 0.7784014940261841, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6918299198150635, + "sampling/sampling_logp_difference/mean": 0.030075252056121826, + "step": 262, + "step_time": 37.408678135834634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 623.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 557.5, + "completions/mean_terminated_length": 557.5, + "completions/min_length": 426.0, + "completions/min_terminated_length": 426.0, + "entropy": 1.4175518676638603, + "epoch": 0.526, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14924532175064087, + "kl": 0.04041226860135794, + "learning_rate": 3e-05, + "loss": -0.14915889501571655, + "num_tokens": 2688798.0, + "reward": 6.125, + "reward_std": 1.1474609375, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.1474609375, + "sampling/importance_sampling_ratio/max": 1.2867430448532104, + "sampling/importance_sampling_ratio/mean": 0.46721577644348145, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36669039726257324, + "sampling/sampling_logp_difference/mean": 0.029747022315859795, + "step": 263, + "step_time": 26.848117691930383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 741.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 593.9375, + "completions/mean_terminated_length": 593.9375, + "completions/min_length": 525.0, + "completions/min_terminated_length": 525.0, + "entropy": 1.1905437037348747, + "epoch": 0.528, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2876143157482147, + "kl": 0.0446856344351545, + "learning_rate": 3e-05, + "loss": 0.23506437242031097, + "num_tokens": 2699949.0, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 2.0369813442230225, + "sampling/importance_sampling_ratio/mean": 0.6758359670639038, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5987787246704102, + "sampling/sampling_logp_difference/mean": 0.028063040226697922, + "step": 264, + "step_time": 36.39752811612561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 542.4375, + "completions/mean_terminated_length": 542.4375, + "completions/min_length": 444.0, + "completions/min_terminated_length": 444.0, + "entropy": 1.0300748609006405, + "epoch": 0.53, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10611388832330704, + "kl": 0.04348581004887819, + "learning_rate": 3e-05, + "loss": -0.13722549378871918, + "num_tokens": 2710444.0, + "reward": 7.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 7.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.752124309539795, + "sampling/importance_sampling_ratio/mean": 0.4690064787864685, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.49287891387939453, + "sampling/sampling_logp_difference/mean": 0.026776142418384552, + "step": 265, + "step_time": 25.34836289891973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 773.0, + "completions/max_terminated_length": 773.0, + "completions/mean_length": 604.9375, + "completions/mean_terminated_length": 604.9375, + "completions/min_length": 477.0, + "completions/min_terminated_length": 477.0, + "entropy": 1.2693111822009087, + "epoch": 0.532, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3129880428314209, + "kl": 0.05965391919016838, + "learning_rate": 3e-05, + "loss": 0.3793664872646332, + "num_tokens": 2721859.0, + "reward": 7.0625, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.9960079193115234, + "sampling/importance_sampling_ratio/mean": 1.0858334302902222, + "sampling/importance_sampling_ratio/min": 0.09143810719251633, + "sampling/sampling_logp_difference/max": 0.48268747329711914, + "sampling/sampling_logp_difference/mean": 0.029427004978060722, + "step": 266, + "step_time": 25.01661626342684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 652.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 580.0625, + "completions/mean_terminated_length": 580.0625, + "completions/min_length": 478.0, + "completions/min_terminated_length": 478.0, + "entropy": 1.193462796509266, + "epoch": 0.534, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0628323182463646, + "kl": 0.0641073645092547, + "learning_rate": 3e-05, + "loss": 0.004537707194685936, + "num_tokens": 2732788.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.5967259407043457, + "sampling/importance_sampling_ratio/mean": 0.5058171153068542, + "sampling/importance_sampling_ratio/min": 0.05455247685313225, + "sampling/sampling_logp_difference/max": 0.4802044630050659, + "sampling/sampling_logp_difference/mean": 0.030082745477557182, + "step": 267, + "step_time": 26.015314052347094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 1024.0, + "completions/max_terminated_length": 1021.0, + "completions/mean_length": 757.9375, + "completions/mean_terminated_length": 719.9285888671875, + "completions/min_length": 557.0, + "completions/min_terminated_length": 557.0, + "entropy": 1.0249068401753902, + "epoch": 0.536, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18749572336673737, + "kl": 0.04782780213281512, + "learning_rate": 3e-05, + "loss": 0.10845151543617249, + "num_tokens": 2747027.0, + "reward": 5.75, + "reward_std": 2.9325757026672363, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 2.9325757026672363, + "sampling/importance_sampling_ratio/max": 1.6593483686447144, + "sampling/importance_sampling_ratio/mean": 0.6503843069076538, + "sampling/importance_sampling_ratio/min": 0.038960449397563934, + "sampling/sampling_logp_difference/max": 0.6482839584350586, + "sampling/sampling_logp_difference/mean": 0.02539048343896866, + "step": 268, + "step_time": 24.50977089582011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 720.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 625.125, + "completions/mean_terminated_length": 625.125, + "completions/min_length": 548.0, + "completions/min_terminated_length": 548.0, + "entropy": 1.3341968581080437, + "epoch": 0.538, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40060457587242126, + "kl": 0.07021735375747085, + "learning_rate": 3e-05, + "loss": -0.12464538961648941, + "num_tokens": 2758653.0, + "reward": 7.125, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.5361881256103516, + "sampling/importance_sampling_ratio/mean": 0.7976686358451843, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3937206268310547, + "sampling/sampling_logp_difference/mean": 0.028948824852705002, + "step": 269, + "step_time": 29.484100938774645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 744.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 591.625, + "completions/mean_terminated_length": 591.625, + "completions/min_length": 485.0, + "completions/min_terminated_length": 485.0, + "entropy": 1.2878348901867867, + "epoch": 0.54, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2377263456583023, + "kl": 0.07358641736209393, + "learning_rate": 3e-05, + "loss": 0.1288338303565979, + "num_tokens": 2769927.0, + "reward": 6.875, + "reward_std": 1.0878112316131592, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 1.0878112316131592, + "sampling/importance_sampling_ratio/max": 1.932815670967102, + "sampling/importance_sampling_ratio/mean": 0.6740471124649048, + "sampling/importance_sampling_ratio/min": 0.046515896916389465, + "sampling/sampling_logp_difference/max": 0.4420192837715149, + "sampling/sampling_logp_difference/mean": 0.029194451868534088, + "step": 270, + "step_time": 31.247754107695073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 788.0, + "completions/max_terminated_length": 788.0, + "completions/mean_length": 593.1875, + "completions/mean_terminated_length": 593.1875, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 1.2879671305418015, + "epoch": 0.542, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19190217554569244, + "kl": 0.06670599663630128, + "learning_rate": 3e-05, + "loss": -0.19648313522338867, + "num_tokens": 2781162.0, + "reward": 6.6875, + "reward_std": 0.8732125163078308, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.8732125163078308, + "sampling/importance_sampling_ratio/max": 2.6719114780426025, + "sampling/importance_sampling_ratio/mean": 1.090332269668579, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3736577033996582, + "sampling/sampling_logp_difference/mean": 0.030100077390670776, + "step": 271, + "step_time": 20.30791286379099 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 776.0, + "completions/max_terminated_length": 776.0, + "completions/mean_length": 627.375, + "completions/mean_terminated_length": 627.375, + "completions/min_length": 545.0, + "completions/min_terminated_length": 545.0, + "entropy": 1.147661603987217, + "epoch": 0.544, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20936760306358337, + "kl": 0.06371736479923129, + "learning_rate": 3e-05, + "loss": 0.07189144194126129, + "num_tokens": 2792824.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.1611435413360596, + "sampling/importance_sampling_ratio/mean": 0.5998578667640686, + "sampling/importance_sampling_ratio/min": 0.0627136304974556, + "sampling/sampling_logp_difference/max": 0.6159329414367676, + "sampling/sampling_logp_difference/mean": 0.029183225706219673, + "step": 272, + "step_time": 24.871985231991857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 827.0, + "completions/max_terminated_length": 827.0, + "completions/mean_length": 635.0, + "completions/mean_terminated_length": 635.0, + "completions/min_length": 525.0, + "completions/min_terminated_length": 525.0, + "entropy": 1.2339624986052513, + "epoch": 0.546, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1734493374824524, + "kl": 0.05470140045508742, + "learning_rate": 3e-05, + "loss": 0.14990417659282684, + "num_tokens": 2804816.0, + "reward": 3.625, + "reward_std": 3.7572154998779297, + "rewards/quality_reward/mean": 3.625, + "rewards/quality_reward/std": 3.7572154998779297, + "sampling/importance_sampling_ratio/max": 2.1873209476470947, + "sampling/importance_sampling_ratio/mean": 0.5959733128547668, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4317042827606201, + "sampling/sampling_logp_difference/mean": 0.02763475477695465, + "step": 273, + "step_time": 17.013169853948057 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 703.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 609.5, + "completions/mean_terminated_length": 609.5, + "completions/min_length": 509.0, + "completions/min_terminated_length": 509.0, + "entropy": 1.1710311695933342, + "epoch": 0.548, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22998619079589844, + "kl": 0.0687105453107506, + "learning_rate": 3e-05, + "loss": -0.18302924931049347, + "num_tokens": 2816328.0, + "reward": 7.375, + "reward_std": 0.5, + "rewards/quality_reward/mean": 7.375, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.3319060802459717, + "sampling/importance_sampling_ratio/mean": 0.6655144691467285, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3631110191345215, + "sampling/sampling_logp_difference/mean": 0.028044190257787704, + "step": 274, + "step_time": 39.533187630586326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 926.0, + "completions/max_terminated_length": 926.0, + "completions/mean_length": 646.0625, + "completions/mean_terminated_length": 646.0625, + "completions/min_length": 537.0, + "completions/min_terminated_length": 537.0, + "entropy": 1.16922065615654, + "epoch": 0.55, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.07465694099664688, + "kl": 0.08889741986058652, + "learning_rate": 3e-05, + "loss": -0.034880224615335464, + "num_tokens": 2828313.0, + "reward": 6.75, + "reward_std": 1.879716396331787, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 1.879716396331787, + "sampling/importance_sampling_ratio/max": 2.2781238555908203, + "sampling/importance_sampling_ratio/mean": 0.5912219882011414, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7433638572692871, + "sampling/sampling_logp_difference/mean": 0.029086390510201454, + "step": 275, + "step_time": 25.293008426669985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 760.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 637.6875, + "completions/mean_terminated_length": 637.6875, + "completions/min_length": 512.0, + "completions/min_terminated_length": 512.0, + "entropy": 1.2918337136507034, + "epoch": 0.552, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.01674834080040455, + "kl": 0.0760874121915549, + "learning_rate": 3e-05, + "loss": 0.0015373103087767959, + "num_tokens": 2840084.0, + "reward": 7.0, + "reward_std": 0.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.5048863887786865, + "sampling/importance_sampling_ratio/mean": 0.43593448400497437, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.42199182510375977, + "sampling/sampling_logp_difference/mean": 0.029371261596679688, + "step": 276, + "step_time": 22.80712998472154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 757.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 598.4375, + "completions/mean_terminated_length": 598.4375, + "completions/min_length": 477.0, + "completions/min_terminated_length": 477.0, + "entropy": 1.265094794332981, + "epoch": 0.554, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18090865015983582, + "kl": 0.07271571340970695, + "learning_rate": 3e-05, + "loss": 0.09041914343833923, + "num_tokens": 2851795.0, + "reward": 7.125, + "reward_std": 0.5, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.8326914310455322, + "sampling/importance_sampling_ratio/mean": 0.5874509811401367, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5233750343322754, + "sampling/sampling_logp_difference/mean": 0.028774311766028404, + "step": 277, + "step_time": 22.066202180925757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 733.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 612.5, + "completions/mean_terminated_length": 612.5, + "completions/min_length": 521.0, + "completions/min_terminated_length": 521.0, + "entropy": 1.1233563423156738, + "epoch": 0.556, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23268753290176392, + "kl": 0.06670796708203852, + "learning_rate": 3e-05, + "loss": -0.230061873793602, + "num_tokens": 2863051.0, + "reward": 7.0625, + "reward_std": 0.57373046875, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.57373046875, + "sampling/importance_sampling_ratio/max": 2.076450824737549, + "sampling/importance_sampling_ratio/mean": 0.4506221413612366, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7588817477226257, + "sampling/sampling_logp_difference/mean": 0.028104104101657867, + "step": 278, + "step_time": 22.9273390378803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 748.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 628.125, + "completions/mean_terminated_length": 628.125, + "completions/min_length": 575.0, + "completions/min_terminated_length": 575.0, + "entropy": 1.2298871502280235, + "epoch": 0.558, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0667547881603241, + "kl": 0.06879452662542462, + "learning_rate": 3e-05, + "loss": -0.07226230949163437, + "num_tokens": 2874765.0, + "reward": 7.3125, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 7.3125, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 1.8878028392791748, + "sampling/importance_sampling_ratio/mean": 0.49886637926101685, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3805513381958008, + "sampling/sampling_logp_difference/mean": 0.029238566756248474, + "step": 279, + "step_time": 22.18924847058952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 599.5, + "completions/mean_terminated_length": 599.5, + "completions/min_length": 503.0, + "completions/min_terminated_length": 503.0, + "entropy": 1.2089616432785988, + "epoch": 0.56, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.14418913424015045, + "kl": 0.06538815144449472, + "learning_rate": 3e-05, + "loss": -0.038590408861637115, + "num_tokens": 2886149.0, + "reward": 7.1875, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.162408709526062, + "sampling/importance_sampling_ratio/mean": 0.4855646789073944, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3773159980773926, + "sampling/sampling_logp_difference/mean": 0.027626870200037956, + "step": 280, + "step_time": 44.65077885752544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 703.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 609.8125, + "completions/mean_terminated_length": 609.8125, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "entropy": 1.1101448722183704, + "epoch": 0.562, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.38874274492263794, + "kl": 0.052375795086845756, + "learning_rate": 3e-05, + "loss": 0.07250604778528214, + "num_tokens": 2897730.0, + "reward": 6.625, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.67866849899292, + "sampling/importance_sampling_ratio/mean": 0.8224037885665894, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4089934825897217, + "sampling/sampling_logp_difference/mean": 0.027132270857691765, + "step": 281, + "step_time": 19.093744847457856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 675.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 597.5625, + "completions/mean_terminated_length": 597.5625, + "completions/min_length": 539.0, + "completions/min_terminated_length": 539.0, + "entropy": 1.4377392679452896, + "epoch": 0.564, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5568169355392456, + "kl": 0.06042969529516995, + "learning_rate": 3e-05, + "loss": 0.18828153610229492, + "num_tokens": 2908907.0, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "sampling/importance_sampling_ratio/max": 2.7599775791168213, + "sampling/importance_sampling_ratio/mean": 1.130048155784607, + "sampling/importance_sampling_ratio/min": 0.12026017159223557, + "sampling/sampling_logp_difference/max": 0.41974592208862305, + "sampling/sampling_logp_difference/mean": 0.03135806694626808, + "step": 282, + "step_time": 17.58286938164383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 843.0, + "completions/max_terminated_length": 843.0, + "completions/mean_length": 652.9375, + "completions/mean_terminated_length": 652.9375, + "completions/min_length": 567.0, + "completions/min_terminated_length": 567.0, + "entropy": 1.3557419702410698, + "epoch": 0.566, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10773341357707977, + "kl": 0.06649435753934085, + "learning_rate": 3e-05, + "loss": 0.16503384709358215, + "num_tokens": 2920914.0, + "reward": 6.3125, + "reward_std": 1.25, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.25, + "sampling/importance_sampling_ratio/max": 1.6338802576065063, + "sampling/importance_sampling_ratio/mean": 0.3585829436779022, + "sampling/importance_sampling_ratio/min": 0.02368989959359169, + "sampling/sampling_logp_difference/max": 0.4577202796936035, + "sampling/sampling_logp_difference/mean": 0.02949238382279873, + "step": 283, + "step_time": 27.548663158435374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 887.0, + "completions/max_terminated_length": 887.0, + "completions/mean_length": 595.9375, + "completions/mean_terminated_length": 595.9375, + "completions/min_length": 503.0, + "completions/min_terminated_length": 503.0, + "entropy": 1.1663579158484936, + "epoch": 0.568, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22535188496112823, + "kl": 0.06671181181445718, + "learning_rate": 3e-05, + "loss": 0.14570997655391693, + "num_tokens": 2932113.0, + "reward": 6.5, + "reward_std": 1.7511900663375854, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.7511900663375854, + "sampling/importance_sampling_ratio/max": 1.3565518856048584, + "sampling/importance_sampling_ratio/mean": 0.5842634439468384, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.34877145290374756, + "sampling/sampling_logp_difference/mean": 0.02905319817364216, + "step": 284, + "step_time": 22.762956948485225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 791.0, + "completions/max_terminated_length": 791.0, + "completions/mean_length": 523.75, + "completions/mean_terminated_length": 523.75, + "completions/min_length": 318.0, + "completions/min_terminated_length": 318.0, + "entropy": 1.2438273057341576, + "epoch": 0.57, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24843654036521912, + "kl": 0.056764260167256, + "learning_rate": 3e-05, + "loss": -0.13110309839248657, + "num_tokens": 2942261.0, + "reward": 7.25, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.508721351623535, + "sampling/importance_sampling_ratio/mean": 0.6225794553756714, + "sampling/importance_sampling_ratio/min": 0.10829401761293411, + "sampling/sampling_logp_difference/max": 0.7687346339225769, + "sampling/sampling_logp_difference/mean": 0.027958959341049194, + "step": 285, + "step_time": 17.391164038795978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 635.0, + "completions/max_terminated_length": 635.0, + "completions/mean_length": 575.6875, + "completions/mean_terminated_length": 575.6875, + "completions/min_length": 498.0, + "completions/min_terminated_length": 498.0, + "entropy": 1.278195135295391, + "epoch": 0.572, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.04263085126876831, + "kl": 0.06749766110442579, + "learning_rate": 3e-05, + "loss": 0.0612037368118763, + "num_tokens": 2953400.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 1.4990487098693848, + "sampling/importance_sampling_ratio/mean": 0.38760584592819214, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.563321590423584, + "sampling/sampling_logp_difference/mean": 0.029160577803850174, + "step": 286, + "step_time": 27.7587122018449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 654.0, + "completions/max_terminated_length": 654.0, + "completions/mean_length": 596.0625, + "completions/mean_terminated_length": 596.0625, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "entropy": 1.1991046443581581, + "epoch": 0.574, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15738150477409363, + "kl": 0.06657169410027564, + "learning_rate": 3e-05, + "loss": -0.042751215398311615, + "num_tokens": 2964441.0, + "reward": 7.1875, + "reward_std": 0.75, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 1.975547432899475, + "sampling/importance_sampling_ratio/mean": 0.594788670539856, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36002135276794434, + "sampling/sampling_logp_difference/mean": 0.028714079409837723, + "step": 287, + "step_time": 17.97522668587044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 575.0625, + "completions/mean_terminated_length": 575.0625, + "completions/min_length": 506.0, + "completions/min_terminated_length": 506.0, + "entropy": 1.2212486639618874, + "epoch": 0.576, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.13965441286563873, + "kl": 0.05857925652526319, + "learning_rate": 3e-05, + "loss": -0.055044494569301605, + "num_tokens": 2975274.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 2.8281614780426025, + "sampling/importance_sampling_ratio/mean": 0.768635630607605, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.34395354986190796, + "sampling/sampling_logp_difference/mean": 0.027103029191493988, + "step": 288, + "step_time": 15.630491987802088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 1008.0, + "completions/max_terminated_length": 1008.0, + "completions/mean_length": 637.75, + "completions/mean_terminated_length": 637.75, + "completions/min_length": 538.0, + "completions/min_terminated_length": 538.0, + "entropy": 1.1949424967169762, + "epoch": 0.578, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2643212676048279, + "kl": 0.049948622239753604, + "learning_rate": 3e-05, + "loss": 0.2526615262031555, + "num_tokens": 2987238.0, + "reward": 6.375, + "reward_std": 1.8574175834655762, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.8574175834655762, + "sampling/importance_sampling_ratio/max": 1.345821738243103, + "sampling/importance_sampling_ratio/mean": 0.4440639913082123, + "sampling/importance_sampling_ratio/min": 0.06620145589113235, + "sampling/sampling_logp_difference/max": 0.411831259727478, + "sampling/sampling_logp_difference/mean": 0.028626006096601486, + "step": 289, + "step_time": 40.26761668827385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 596.125, + "completions/mean_terminated_length": 596.125, + "completions/min_length": 502.0, + "completions/min_terminated_length": 502.0, + "entropy": 1.1830484792590141, + "epoch": 0.58, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2776772677898407, + "kl": 0.06474088784307241, + "learning_rate": 3e-05, + "loss": -0.28555187582969666, + "num_tokens": 2998528.0, + "reward": 7.25, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.459705114364624, + "sampling/importance_sampling_ratio/mean": 0.9335561990737915, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4653182029724121, + "sampling/sampling_logp_difference/mean": 0.02966075949370861, + "step": 290, + "step_time": 25.85909090936184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 685.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 593.6875, + "completions/mean_terminated_length": 593.6875, + "completions/min_length": 532.0, + "completions/min_terminated_length": 532.0, + "entropy": 1.0875738225877285, + "epoch": 0.582, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23154416680335999, + "kl": 0.05466599250212312, + "learning_rate": 3e-05, + "loss": 0.31567299365997314, + "num_tokens": 3009683.0, + "reward": 6.5625, + "reward_std": 1.412739634513855, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 1.412739634513855, + "sampling/importance_sampling_ratio/max": 2.0061569213867188, + "sampling/importance_sampling_ratio/mean": 0.6376235485076904, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6209149360656738, + "sampling/sampling_logp_difference/mean": 0.027346499264240265, + "step": 291, + "step_time": 21.547887252643704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 597.0, + "completions/max_terminated_length": 597.0, + "completions/mean_length": 552.5, + "completions/mean_terminated_length": 552.5, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.238592490553856, + "epoch": 0.584, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2807851731777191, + "kl": 0.05562997469678521, + "learning_rate": 3e-05, + "loss": 0.036074671894311905, + "num_tokens": 3020003.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 2.0109286308288574, + "sampling/importance_sampling_ratio/mean": 0.5984793901443481, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3504910469055176, + "sampling/sampling_logp_difference/mean": 0.02795255184173584, + "step": 292, + "step_time": 16.77090792078525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 658.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 592.3125, + "completions/mean_terminated_length": 592.3125, + "completions/min_length": 518.0, + "completions/min_terminated_length": 518.0, + "entropy": 1.180466279387474, + "epoch": 0.586, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15615873038768768, + "kl": 0.054585910867899656, + "learning_rate": 3e-05, + "loss": -0.1038510799407959, + "num_tokens": 3031240.0, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.661234736442566, + "sampling/importance_sampling_ratio/mean": 0.544894814491272, + "sampling/importance_sampling_ratio/min": 0.007596802897751331, + "sampling/sampling_logp_difference/max": 0.7037668228149414, + "sampling/sampling_logp_difference/mean": 0.03072209097445011, + "step": 293, + "step_time": 19.1619019433856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 645.0, + "completions/max_terminated_length": 645.0, + "completions/mean_length": 571.75, + "completions/mean_terminated_length": 571.75, + "completions/min_length": 497.0, + "completions/min_terminated_length": 497.0, + "entropy": 1.3075302839279175, + "epoch": 0.588, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16230997443199158, + "kl": 0.05031474749557674, + "learning_rate": 3e-05, + "loss": -0.12249961495399475, + "num_tokens": 3042436.0, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "sampling/importance_sampling_ratio/max": 2.383788585662842, + "sampling/importance_sampling_ratio/mean": 0.621848464012146, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5716366767883301, + "sampling/sampling_logp_difference/mean": 0.028705568984150887, + "step": 294, + "step_time": 20.532019450329244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 581.1875, + "completions/mean_terminated_length": 581.1875, + "completions/min_length": 492.0, + "completions/min_terminated_length": 492.0, + "entropy": 1.3449261263012886, + "epoch": 0.59, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.10194012522697449, + "kl": 0.0485304044559598, + "learning_rate": 3e-05, + "loss": 0.14228732883930206, + "num_tokens": 3053575.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.1675894260406494, + "sampling/importance_sampling_ratio/mean": 0.5425832271575928, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.408158540725708, + "sampling/sampling_logp_difference/mean": 0.028240300714969635, + "step": 295, + "step_time": 26.749822621699423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 613.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 549.8125, + "completions/mean_terminated_length": 549.8125, + "completions/min_length": 501.0, + "completions/min_terminated_length": 501.0, + "entropy": 1.345760464668274, + "epoch": 0.592, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.32519620656967163, + "kl": 0.05967968609184027, + "learning_rate": 3e-05, + "loss": -0.013912655413150787, + "num_tokens": 3064324.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.0540385246276855, + "sampling/importance_sampling_ratio/mean": 0.8356897830963135, + "sampling/importance_sampling_ratio/min": 0.05226827412843704, + "sampling/sampling_logp_difference/max": 0.37021374702453613, + "sampling/sampling_logp_difference/mean": 0.029748085886240005, + "step": 296, + "step_time": 16.617265206295997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 579.0, + "completions/mean_terminated_length": 579.0, + "completions/min_length": 489.0, + "completions/min_terminated_length": 489.0, + "entropy": 1.1249969974160194, + "epoch": 0.594, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6430356502532959, + "kl": 0.04937166138552129, + "learning_rate": 3e-05, + "loss": -0.4677557647228241, + "num_tokens": 3075172.0, + "reward": 7.0625, + "reward_std": 0.25, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "sampling/importance_sampling_ratio/max": 2.9863228797912598, + "sampling/importance_sampling_ratio/mean": 0.8718785047531128, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4263639450073242, + "sampling/sampling_logp_difference/mean": 0.02859537862241268, + "step": 297, + "step_time": 25.6860204776749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 710.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 555.625, + "completions/mean_terminated_length": 555.625, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "entropy": 1.1057527735829353, + "epoch": 0.596, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.28020939230918884, + "kl": 0.035641232039779425, + "learning_rate": 3e-05, + "loss": 0.03550681099295616, + "num_tokens": 3085846.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.5230090618133545, + "sampling/importance_sampling_ratio/mean": 0.7580450177192688, + "sampling/importance_sampling_ratio/min": 0.08425833284854889, + "sampling/sampling_logp_difference/max": 0.35025501251220703, + "sampling/sampling_logp_difference/mean": 0.026732780039310455, + "step": 298, + "step_time": 18.157300523016602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 598.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 530.1875, + "completions/mean_terminated_length": 530.1875, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "entropy": 1.1326002702116966, + "epoch": 0.598, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17403694987297058, + "kl": 0.04222825774922967, + "learning_rate": 3e-05, + "loss": -0.09046114981174469, + "num_tokens": 3096033.0, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.300510883331299, + "sampling/importance_sampling_ratio/mean": 0.6789309978485107, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.314223051071167, + "sampling/sampling_logp_difference/mean": 0.027955062687397003, + "step": 299, + "step_time": 15.391770029906183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 613.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 535.125, + "completions/mean_terminated_length": 535.125, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "entropy": 1.1391064934432507, + "epoch": 0.6, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.007904416881501675, + "kl": 0.03820930456276983, + "learning_rate": 3e-05, + "loss": 0.0007643185090273619, + "num_tokens": 3106243.0, + "reward": 7.0, + "reward_std": 0.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4233957529067993, + "sampling/importance_sampling_ratio/mean": 0.6089779138565063, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4390592575073242, + "sampling/sampling_logp_difference/mean": 0.027106385678052902, + "step": 300, + "step_time": 23.217237341683358 + } + ], + "logging_steps": 1, + "max_steps": 300, + "num_input_tokens_seen": 3106243, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/outputs/E0-baseline/checkpoint-300/training_args.bin b/outputs/E0-baseline/checkpoint-300/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c3a2693682d14c331dbd12e8872552fc28d54b49 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-300/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6 +size 7697 diff --git a/outputs/E0-baseline/checkpoint-50/README.md b/outputs/E0-baseline/checkpoint-50/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88990785a4695219513364eafae195070ddd3182 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-50/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507 +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.20.0 \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-50/adapter_config.json b/outputs/E0-baseline/checkpoint-50/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3b383bef044f277e3ff79f0fc5addabd3c441593 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-50/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.20.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-50/adapter_model.safetensors b/outputs/E0-baseline/checkpoint-50/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..01747e1145b981c9a15640996aa673d41d9d0029 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-50/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75620b4583564edd639fd6f92613b6bf52d8fca52850fd95cf3f82686b022bdd +size 264308896 diff --git a/outputs/E0-baseline/checkpoint-50/chat_template.jinja b/outputs/E0-baseline/checkpoint-50/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-50/chat_template.jinja @@ -0,0 +1,61 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/outputs/E0-baseline/checkpoint-50/tokenizer.json b/outputs/E0-baseline/checkpoint-50/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-50/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/outputs/E0-baseline/checkpoint-50/tokenizer_config.json b/outputs/E0-baseline/checkpoint-50/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..86d67326516569b4f1cec3ac7b51b8903062c648 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-50/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "truncation_side": "left", + "unk_token": null +} diff --git a/outputs/E0-baseline/checkpoint-50/trainer_state.json b/outputs/E0-baseline/checkpoint-50/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0700d866dedaecfba6acafd1f85a03bb14b55842 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-50/trainer_state.json @@ -0,0 +1,1684 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.1, + "eval_steps": 500, + "global_step": 50, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 529.0, + "completions/mean_terminated_length": 529.0, + "completions/min_length": 482.0, + "completions/min_terminated_length": 482.0, + "entropy": 1.2025159299373627, + "epoch": 0.002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22668755054473877, + "kl": 0.0, + "learning_rate": 0.0, + "loss": 0.2398601919412613, + "num_tokens": 10400.0, + "reward": 6.375, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.6477458477020264, + "sampling/importance_sampling_ratio/mean": 0.5064857602119446, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.40894174575805664, + "sampling/sampling_logp_difference/mean": 0.026567919179797173, + "step": 1, + "step_time": 12.760562099982053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 644.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 562.25, + "completions/mean_terminated_length": 562.25, + "completions/min_length": 497.0, + "completions/min_terminated_length": 497.0, + "entropy": 1.1930748969316483, + "epoch": 0.004, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12967805564403534, + "kl": 0.0, + "learning_rate": 3e-06, + "loss": -0.08571265637874603, + "num_tokens": 20924.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 1.0037232637405396, + "sampling/importance_sampling_ratio/mean": 0.41275694966316223, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.45699238777160645, + "sampling/sampling_logp_difference/mean": 0.025086138397455215, + "step": 2, + "step_time": 11.406366533134133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 483.625, + "completions/mean_terminated_length": 483.625, + "completions/min_length": 407.0, + "completions/min_terminated_length": 407.0, + "entropy": 1.1096689626574516, + "epoch": 0.006, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22966289520263672, + "kl": 0.0008355328354809899, + "learning_rate": 6e-06, + "loss": 0.020913563668727875, + "num_tokens": 30222.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 1.927120566368103, + "sampling/importance_sampling_ratio/mean": 0.6114993095397949, + "sampling/importance_sampling_ratio/min": 0.11067161709070206, + "sampling/sampling_logp_difference/max": 0.4620504379272461, + "sampling/sampling_logp_difference/mean": 0.024864500388503075, + "step": 3, + "step_time": 26.480680393986404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 608.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 524.75, + "completions/mean_terminated_length": 524.75, + "completions/min_length": 473.0, + "completions/min_terminated_length": 473.0, + "entropy": 1.211122527718544, + "epoch": 0.008, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30378466844558716, + "kl": 0.0008403196770814247, + "learning_rate": 9e-06, + "loss": -0.12959149479866028, + "num_tokens": 40410.0, + "reward": 6.25, + "reward_std": 0.8563488721847534, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 0.8563488721847534, + "sampling/importance_sampling_ratio/max": 2.406888961791992, + "sampling/importance_sampling_ratio/mean": 0.5457419753074646, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3680229187011719, + "sampling/sampling_logp_difference/mean": 0.02656388282775879, + "step": 4, + "step_time": 22.95301443943754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 487.5625, + "completions/mean_terminated_length": 487.5625, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "entropy": 1.247180238366127, + "epoch": 0.01, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5199307799339294, + "kl": 0.0008723233368073124, + "learning_rate": 1.2e-05, + "loss": 0.11524428427219391, + "num_tokens": 49915.0, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.1544158458709717, + "sampling/importance_sampling_ratio/mean": 0.963020920753479, + "sampling/importance_sampling_ratio/min": 0.04948288947343826, + "sampling/sampling_logp_difference/max": 0.4774587154388428, + "sampling/sampling_logp_difference/mean": 0.02625642716884613, + "step": 5, + "step_time": 22.37928077671677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 533.0, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 394.0, + "completions/min_terminated_length": 394.0, + "entropy": 1.1693861335515976, + "epoch": 0.012, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.27643197774887085, + "kl": 0.0009261858467652928, + "learning_rate": 1.5e-05, + "loss": 0.15093231201171875, + "num_tokens": 60219.0, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "sampling/importance_sampling_ratio/max": 2.125091791152954, + "sampling/importance_sampling_ratio/mean": 0.7733402252197266, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.7049552202224731, + "sampling/sampling_logp_difference/mean": 0.025986071676015854, + "step": 6, + "step_time": 21.00841654697433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 505.6875, + "completions/mean_terminated_length": 505.6875, + "completions/min_length": 425.0, + "completions/min_terminated_length": 425.0, + "entropy": 1.2473183795809746, + "epoch": 0.014, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2166663259267807, + "kl": 0.0009701631606731098, + "learning_rate": 1.8e-05, + "loss": -0.08582288026809692, + "num_tokens": 69902.0, + "reward": 7.1875, + "reward_std": 0.75, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "sampling/importance_sampling_ratio/max": 2.8287386894226074, + "sampling/importance_sampling_ratio/mean": 0.5286832451820374, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.38030898571014404, + "sampling/sampling_logp_difference/mean": 0.0264718160033226, + "step": 7, + "step_time": 46.596127359196544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "entropy": 1.1040107272565365, + "epoch": 0.016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.18874908983707428, + "kl": 0.0010721117541834246, + "learning_rate": 2.1e-05, + "loss": -0.1946130096912384, + "num_tokens": 79501.0, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "sampling/importance_sampling_ratio/max": 2.3561792373657227, + "sampling/importance_sampling_ratio/mean": 0.6918502449989319, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.26114892959594727, + "sampling/sampling_logp_difference/mean": 0.02554757334291935, + "step": 8, + "step_time": 25.433595282491297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 554.0, + "completions/max_terminated_length": 554.0, + "completions/mean_length": 490.1875, + "completions/mean_terminated_length": 490.1875, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "entropy": 1.1377170644700527, + "epoch": 0.018, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.204215869307518, + "kl": 0.002002388624532614, + "learning_rate": 2.4e-05, + "loss": -0.08130021393299103, + "num_tokens": 88976.0, + "reward": 6.5, + "reward_std": 0.5163977742195129, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.5163977742195129, + "sampling/importance_sampling_ratio/max": 2.0982444286346436, + "sampling/importance_sampling_ratio/mean": 0.5873216986656189, + "sampling/importance_sampling_ratio/min": 0.04890051856637001, + "sampling/sampling_logp_difference/max": 0.8512144088745117, + "sampling/sampling_logp_difference/mean": 0.02638406865298748, + "step": 9, + "step_time": 18.427699581719935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 500.1875, + "completions/mean_terminated_length": 500.1875, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.1230391450226307, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1985069215297699, + "kl": 0.0026735300780273974, + "learning_rate": 2.7000000000000002e-05, + "loss": -0.12387816607952118, + "num_tokens": 98603.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 1.983199954032898, + "sampling/importance_sampling_ratio/mean": 0.5639468431472778, + "sampling/importance_sampling_ratio/min": 0.012905921787023544, + "sampling/sampling_logp_difference/max": 0.3653905391693115, + "sampling/sampling_logp_difference/mean": 0.025383003056049347, + "step": 10, + "step_time": 14.99981931829825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 459.6875, + "completions/mean_terminated_length": 459.6875, + "completions/min_length": 379.0, + "completions/min_terminated_length": 379.0, + "entropy": 1.213625729084015, + "epoch": 0.022, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3695620596408844, + "kl": 0.00424640768324025, + "learning_rate": 3e-05, + "loss": -0.06913074851036072, + "num_tokens": 107734.0, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "sampling/importance_sampling_ratio/max": 2.000136375427246, + "sampling/importance_sampling_ratio/mean": 0.8144867420196533, + "sampling/importance_sampling_ratio/min": 0.06302778422832489, + "sampling/sampling_logp_difference/max": 0.3647327423095703, + "sampling/sampling_logp_difference/mean": 0.026674197986721992, + "step": 11, + "step_time": 30.46549107739702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 1.191277615725994, + "epoch": 0.024, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35457733273506165, + "kl": 0.007200263120466843, + "learning_rate": 3e-05, + "loss": 0.22097699344158173, + "num_tokens": 117199.0, + "reward": 5.9375, + "reward_std": 0.9979145526885986, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 0.9979145526885986, + "sampling/importance_sampling_ratio/max": 1.5660414695739746, + "sampling/importance_sampling_ratio/mean": 0.649204432964325, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3921785354614258, + "sampling/sampling_logp_difference/mean": 0.02726689912378788, + "step": 12, + "step_time": 15.719243939965963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 464.3125, + "completions/mean_terminated_length": 464.3125, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "entropy": 1.24251464381814, + "epoch": 0.026, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.30047014355659485, + "kl": 0.0058551667898427695, + "learning_rate": 3e-05, + "loss": -0.07746122777462006, + "num_tokens": 126556.0, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "sampling/importance_sampling_ratio/max": 2.6028401851654053, + "sampling/importance_sampling_ratio/mean": 0.7561360597610474, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4946432113647461, + "sampling/sampling_logp_difference/mean": 0.02639186754822731, + "step": 13, + "step_time": 18.08984712138772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 444.8125, + "completions/mean_terminated_length": 444.8125, + "completions/min_length": 389.0, + "completions/min_terminated_length": 389.0, + "entropy": 1.1501125395298004, + "epoch": 0.028, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.24105942249298096, + "kl": 0.012796793889719993, + "learning_rate": 3e-05, + "loss": 0.10855278372764587, + "num_tokens": 135417.0, + "reward": 3.1875, + "reward_std": 3.310966730117798, + "rewards/quality_reward/mean": 3.1875, + "rewards/quality_reward/std": 3.310966730117798, + "sampling/importance_sampling_ratio/max": 2.541518211364746, + "sampling/importance_sampling_ratio/mean": 0.5903321504592896, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.8390979766845703, + "sampling/sampling_logp_difference/mean": 0.02838001400232315, + "step": 14, + "step_time": 20.055794408079237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 497.0, + "completions/max_terminated_length": 497.0, + "completions/mean_length": 442.25, + "completions/mean_terminated_length": 442.25, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 1.1262825280427933, + "epoch": 0.03, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.19040776789188385, + "kl": 0.010701361010433175, + "learning_rate": 3e-05, + "loss": -0.007966680452227592, + "num_tokens": 144205.0, + "reward": 5.875, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.0754293203353882, + "sampling/importance_sampling_ratio/mean": 0.41446638107299805, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3982200622558594, + "sampling/sampling_logp_difference/mean": 0.027210108935832977, + "step": 15, + "step_time": 14.176074750721455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 478.125, + "completions/mean_terminated_length": 478.125, + "completions/min_length": 433.0, + "completions/min_terminated_length": 433.0, + "entropy": 1.2985924184322357, + "epoch": 0.032, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23625652492046356, + "kl": 0.0213887135614641, + "learning_rate": 3e-05, + "loss": -0.21546132862567902, + "num_tokens": 153543.0, + "reward": 6.5, + "reward_std": 0.730296790599823, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.730296790599823, + "sampling/importance_sampling_ratio/max": 2.0074336528778076, + "sampling/importance_sampling_ratio/mean": 0.49076417088508606, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5095968246459961, + "sampling/sampling_logp_difference/mean": 0.028833162039518356, + "step": 16, + "step_time": 15.848205763846636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 469.25, + "completions/mean_terminated_length": 469.25, + "completions/min_length": 423.0, + "completions/min_terminated_length": 423.0, + "entropy": 1.3148910626769066, + "epoch": 0.034, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.17765119671821594, + "kl": 0.02128879475640133, + "learning_rate": 3e-05, + "loss": -0.0828079879283905, + "num_tokens": 163043.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 1.5988941192626953, + "sampling/importance_sampling_ratio/mean": 0.5021570324897766, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.591062068939209, + "sampling/sampling_logp_difference/mean": 0.030804751440882683, + "step": 17, + "step_time": 28.313011147081852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 447.0, + "completions/mean_terminated_length": 447.0, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 1.390664003789425, + "epoch": 0.036, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37244123220443726, + "kl": 0.019650122558232397, + "learning_rate": 3e-05, + "loss": -0.01184748113155365, + "num_tokens": 172379.0, + "reward": 6.0, + "reward_std": 0.8944272398948669, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.8944272398948669, + "sampling/importance_sampling_ratio/max": 2.675238847732544, + "sampling/importance_sampling_ratio/mean": 0.9581233263015747, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5762512683868408, + "sampling/sampling_logp_difference/mean": 0.03126702085137367, + "step": 18, + "step_time": 30.991567107848823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 498.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 447.75, + "completions/mean_terminated_length": 447.75, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 1.3287223279476166, + "epoch": 0.038, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23734751343727112, + "kl": 0.022738213243428618, + "learning_rate": 3e-05, + "loss": 0.040024783462285995, + "num_tokens": 181239.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.095722198486328, + "sampling/importance_sampling_ratio/mean": 0.6408629417419434, + "sampling/importance_sampling_ratio/min": 0.1203346848487854, + "sampling/sampling_logp_difference/max": 0.4722297191619873, + "sampling/sampling_logp_difference/mean": 0.030378391966223717, + "step": 19, + "step_time": 18.615950418636203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 452.3125, + "completions/mean_terminated_length": 452.3125, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 1.1001618690788746, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34637194871902466, + "kl": 0.015380491153337061, + "learning_rate": 3e-05, + "loss": 0.1691148728132248, + "num_tokens": 190068.0, + "reward": 6.125, + "reward_std": 1.3102163076400757, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.3102163076400757, + "sampling/importance_sampling_ratio/max": 2.6566128730773926, + "sampling/importance_sampling_ratio/mean": 0.7767290472984314, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.6531033515930176, + "sampling/sampling_logp_difference/mean": 0.030463142320513725, + "step": 20, + "step_time": 15.741292077116668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 457.5625, + "completions/mean_terminated_length": 457.5625, + "completions/min_length": 390.0, + "completions/min_terminated_length": 390.0, + "entropy": 1.3708399310708046, + "epoch": 0.042, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2514922618865967, + "kl": 0.018277494702488184, + "learning_rate": 3e-05, + "loss": -0.13425321877002716, + "num_tokens": 198941.0, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "sampling/importance_sampling_ratio/max": 1.5612297058105469, + "sampling/importance_sampling_ratio/mean": 0.601022481918335, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4818992614746094, + "sampling/sampling_logp_difference/mean": 0.0315740592777729, + "step": 21, + "step_time": 17.24192419089377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 421.4375, + "completions/mean_terminated_length": 421.4375, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 1.136269599199295, + "epoch": 0.044, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2268020063638687, + "kl": 0.017973962530959398, + "learning_rate": 3e-05, + "loss": 0.010622119531035423, + "num_tokens": 207300.0, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "sampling/importance_sampling_ratio/max": 1.7986358404159546, + "sampling/importance_sampling_ratio/mean": 0.46136727929115295, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.5335149765014648, + "sampling/sampling_logp_difference/mean": 0.02804401144385338, + "step": 22, + "step_time": 19.37282825494185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 452.6875, + "completions/mean_terminated_length": 452.6875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3299130946397781, + "epoch": 0.046, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33994877338409424, + "kl": 0.021804221265483648, + "learning_rate": 3e-05, + "loss": -0.24404363334178925, + "num_tokens": 216343.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.452249526977539, + "sampling/importance_sampling_ratio/mean": 0.747193455696106, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6328549385070801, + "sampling/sampling_logp_difference/mean": 0.02918298915028572, + "step": 23, + "step_time": 15.356728344224393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 462.4375, + "completions/mean_terminated_length": 462.4375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.379701942205429, + "epoch": 0.048, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3076202869415283, + "kl": 0.020299295720178634, + "learning_rate": 3e-05, + "loss": -0.09123071283102036, + "num_tokens": 225550.0, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "sampling/importance_sampling_ratio/max": 1.5845099687576294, + "sampling/importance_sampling_ratio/mean": 0.6625345945358276, + "sampling/importance_sampling_ratio/min": 0.04495502635836601, + "sampling/sampling_logp_difference/max": 0.36547183990478516, + "sampling/sampling_logp_difference/mean": 0.028946854174137115, + "step": 24, + "step_time": 14.95462113339454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 482.4375, + "completions/mean_terminated_length": 482.4375, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.2471638694405556, + "epoch": 0.05, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22180576622486115, + "kl": 0.018309170845896006, + "learning_rate": 3e-05, + "loss": -0.1412944793701172, + "num_tokens": 235005.0, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "sampling/importance_sampling_ratio/max": 2.6407876014709473, + "sampling/importance_sampling_ratio/mean": 0.7712795734405518, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4363563060760498, + "sampling/sampling_logp_difference/mean": 0.02898716926574707, + "step": 25, + "step_time": 16.46686205593869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 455.75, + "completions/mean_terminated_length": 455.75, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 1.28530602902174, + "epoch": 0.052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37535253167152405, + "kl": 0.020504546992015094, + "learning_rate": 3e-05, + "loss": 0.10839397460222244, + "num_tokens": 243953.0, + "reward": 6.25, + "reward_std": 1.0645813941955566, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.0645813941955566, + "sampling/importance_sampling_ratio/max": 2.0567266941070557, + "sampling/importance_sampling_ratio/mean": 0.5800145864486694, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4523625373840332, + "sampling/sampling_logp_difference/mean": 0.027610119432210922, + "step": 26, + "step_time": 22.121026155073196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.0, + "completions/max_terminated_length": 514.0, + "completions/mean_length": 462.75, + "completions/mean_terminated_length": 462.75, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 1.2253629937767982, + "epoch": 0.054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2340388149023056, + "kl": 0.020236384589225054, + "learning_rate": 3e-05, + "loss": 0.04823978245258331, + "num_tokens": 253237.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 1.1560932397842407, + "sampling/importance_sampling_ratio/mean": 0.40753045678138733, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35022830963134766, + "sampling/sampling_logp_difference/mean": 0.028367744758725166, + "step": 27, + "step_time": 15.10967448912561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 538.0, + "completions/max_terminated_length": 538.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "entropy": 1.3779077678918839, + "epoch": 0.056, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5531017184257507, + "kl": 0.01706669357372448, + "learning_rate": 3e-05, + "loss": 0.0933581069111824, + "num_tokens": 262454.0, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "sampling/importance_sampling_ratio/max": 2.5711586475372314, + "sampling/importance_sampling_ratio/mean": 0.7730721831321716, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.343827486038208, + "sampling/sampling_logp_difference/mean": 0.028883326798677444, + "step": 28, + "step_time": 38.59647103771567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 563.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 477.0, + "completions/mean_terminated_length": 477.0, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.3791070505976677, + "epoch": 0.058, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33771538734436035, + "kl": 0.02141271048458293, + "learning_rate": 3e-05, + "loss": 0.010216176509857178, + "num_tokens": 271918.0, + "reward": 5.75, + "reward_std": 1.2909945249557495, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.2909945249557495, + "sampling/importance_sampling_ratio/max": 2.4999797344207764, + "sampling/importance_sampling_ratio/mean": 1.0250636339187622, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3258817195892334, + "sampling/sampling_logp_difference/mean": 0.029029540717601776, + "step": 29, + "step_time": 23.610272648278624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 471.1875, + "completions/mean_terminated_length": 471.1875, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.4652926102280617, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21955685317516327, + "kl": 0.019562674744520336, + "learning_rate": 3e-05, + "loss": -0.014814459718763828, + "num_tokens": 281305.0, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 2.802098274230957, + "sampling/importance_sampling_ratio/mean": 0.866391658782959, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6354451179504395, + "sampling/sampling_logp_difference/mean": 0.03177585452795029, + "step": 30, + "step_time": 16.961607525125146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 490.5, + "completions/mean_terminated_length": 490.5, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 1.1957123205065727, + "epoch": 0.062, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12176825106143951, + "kl": 0.026934220048133284, + "learning_rate": 3e-05, + "loss": -0.08307373523712158, + "num_tokens": 291409.0, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "sampling/importance_sampling_ratio/max": 1.610858678817749, + "sampling/importance_sampling_ratio/mean": 0.6937527656555176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4253416061401367, + "sampling/sampling_logp_difference/mean": 0.02871524728834629, + "step": 31, + "step_time": 15.012207658961415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 462.0, + "completions/mean_terminated_length": 462.0, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 1.3018206283450127, + "epoch": 0.064, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4979296624660492, + "kl": 0.03539742121938616, + "learning_rate": 3e-05, + "loss": 0.0017175457905977964, + "num_tokens": 300649.0, + "reward": 6.875, + "reward_std": 0.5, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "sampling/importance_sampling_ratio/max": 1.8527640104293823, + "sampling/importance_sampling_ratio/mean": 0.7824680209159851, + "sampling/importance_sampling_ratio/min": 0.07447884231805801, + "sampling/sampling_logp_difference/max": 0.5221483707427979, + "sampling/sampling_logp_difference/mean": 0.029107660055160522, + "step": 32, + "step_time": 22.78309725271538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 474.0, + "completions/mean_terminated_length": 474.0, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 1.321175642311573, + "epoch": 0.066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22016967833042145, + "kl": 0.029592803912237287, + "learning_rate": 3e-05, + "loss": 0.05625719577074051, + "num_tokens": 309889.0, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "sampling/importance_sampling_ratio/max": 2.9875097274780273, + "sampling/importance_sampling_ratio/mean": 0.7832435369491577, + "sampling/importance_sampling_ratio/min": 0.18006731569766998, + "sampling/sampling_logp_difference/max": 0.38585615158081055, + "sampling/sampling_logp_difference/mean": 0.027828343212604523, + "step": 33, + "step_time": 44.51360382232815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 459.9375, + "completions/mean_terminated_length": 459.9375, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "entropy": 1.2222414836287498, + "epoch": 0.068, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2873040437698364, + "kl": 0.02840927499346435, + "learning_rate": 3e-05, + "loss": -0.037432070821523666, + "num_tokens": 318904.0, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "sampling/importance_sampling_ratio/max": 2.6647069454193115, + "sampling/importance_sampling_ratio/mean": 0.5744763016700745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.41987133026123047, + "sampling/sampling_logp_difference/mean": 0.029470665380358696, + "step": 34, + "step_time": 133.03877451224253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 522.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 462.1875, + "completions/mean_terminated_length": 462.1875, + "completions/min_length": 413.0, + "completions/min_terminated_length": 413.0, + "entropy": 1.1665974482893944, + "epoch": 0.07, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2490653246641159, + "kl": 0.025841041060630232, + "learning_rate": 3e-05, + "loss": -0.20422951877117157, + "num_tokens": 328011.0, + "reward": 6.9375, + "reward_std": 1.1814539432525635, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "sampling/importance_sampling_ratio/max": 2.510730266571045, + "sampling/importance_sampling_ratio/mean": 0.7954420447349548, + "sampling/importance_sampling_ratio/min": 0.020566223189234734, + "sampling/sampling_logp_difference/max": 0.36430132389068604, + "sampling/sampling_logp_difference/mean": 0.026844775304198265, + "step": 35, + "step_time": 21.5843787365593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 549.0, + "completions/max_terminated_length": 549.0, + "completions/mean_length": 492.0, + "completions/mean_terminated_length": 492.0, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3467887043952942, + "epoch": 0.072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15268851816654205, + "kl": 0.023660800594370812, + "learning_rate": 3e-05, + "loss": -0.11188814043998718, + "num_tokens": 337731.0, + "reward": 6.5625, + "reward_std": 0.7274384498596191, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.7274384498596191, + "sampling/importance_sampling_ratio/max": 1.6270908117294312, + "sampling/importance_sampling_ratio/mean": 0.614537239074707, + "sampling/importance_sampling_ratio/min": 0.10853960365056992, + "sampling/sampling_logp_difference/max": 0.4023854732513428, + "sampling/sampling_logp_difference/mean": 0.028961554169654846, + "step": 36, + "step_time": 18.843947287183255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.0, + "completions/max_terminated_length": 513.0, + "completions/mean_length": 460.75, + "completions/mean_terminated_length": 460.75, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "entropy": 1.2476315572857857, + "epoch": 0.074, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42548227310180664, + "kl": 0.022181478852871805, + "learning_rate": 3e-05, + "loss": 0.37223517894744873, + "num_tokens": 346815.0, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "sampling/importance_sampling_ratio/max": 2.6624510288238525, + "sampling/importance_sampling_ratio/mean": 0.7942180633544922, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4037543535232544, + "sampling/sampling_logp_difference/mean": 0.028780322521924973, + "step": 37, + "step_time": 42.04662919091061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 452.0, + "completions/mean_terminated_length": 452.0, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "entropy": 1.1745503433048725, + "epoch": 0.076, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.16980381309986115, + "kl": 0.017483733594417572, + "learning_rate": 3e-05, + "loss": -0.09029137343168259, + "num_tokens": 355711.0, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "sampling/importance_sampling_ratio/max": 2.3201518058776855, + "sampling/importance_sampling_ratio/mean": 0.721072793006897, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.30433225631713867, + "sampling/sampling_logp_difference/mean": 0.026646045967936516, + "step": 38, + "step_time": 38.63075139513239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 466.8125, + "completions/mean_terminated_length": 466.8125, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.2664988860487938, + "epoch": 0.078, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.21305795013904572, + "kl": 0.021121050289366394, + "learning_rate": 3e-05, + "loss": -0.03154226019978523, + "num_tokens": 364860.0, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "sampling/importance_sampling_ratio/max": 2.0297553539276123, + "sampling/importance_sampling_ratio/mean": 0.674609363079071, + "sampling/importance_sampling_ratio/min": 0.11245065927505493, + "sampling/sampling_logp_difference/max": 0.37443917989730835, + "sampling/sampling_logp_difference/mean": 0.028257746249437332, + "step": 39, + "step_time": 30.028073193039745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 485.1875, + "completions/mean_terminated_length": 485.1875, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "entropy": 1.3458357080817223, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.12235487997531891, + "kl": 0.018535695446189493, + "learning_rate": 3e-05, + "loss": -0.035816628485918045, + "num_tokens": 374607.0, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "sampling/importance_sampling_ratio/max": 2.242042303085327, + "sampling/importance_sampling_ratio/mean": 0.5344723463058472, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36985254287719727, + "sampling/sampling_logp_difference/mean": 0.029600802809000015, + "step": 40, + "step_time": 15.446288945619017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 498.875, + "completions/mean_terminated_length": 498.875, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 1.3402792811393738, + "epoch": 0.082, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.15642686188220978, + "kl": 0.013967045990284532, + "learning_rate": 3e-05, + "loss": 0.0011727213859558105, + "num_tokens": 384317.0, + "reward": 6.375, + "reward_std": 0.8850612044334412, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8850612044334412, + "sampling/importance_sampling_ratio/max": 1.5691092014312744, + "sampling/importance_sampling_ratio/mean": 0.3696203827857971, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.6487679481506348, + "sampling/sampling_logp_difference/mean": 0.03018251620233059, + "step": 41, + "step_time": 18.15720977121964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 482.0625, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 1.3317564576864243, + "epoch": 0.084, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3236794173717499, + "kl": 0.01707366103073582, + "learning_rate": 3e-05, + "loss": 0.10363321751356125, + "num_tokens": 393934.0, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "sampling/importance_sampling_ratio/max": 2.0568668842315674, + "sampling/importance_sampling_ratio/mean": 0.6415404677391052, + "sampling/importance_sampling_ratio/min": 0.07682990282773972, + "sampling/sampling_logp_difference/max": 0.7769032716751099, + "sampling/sampling_logp_difference/mean": 0.02936164103448391, + "step": 42, + "step_time": 15.066733688581735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 510.9375, + "completions/mean_terminated_length": 510.9375, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "entropy": 1.2099780030548573, + "epoch": 0.086, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.1771102100610733, + "kl": 0.01784718461567536, + "learning_rate": 3e-05, + "loss": -0.027566466480493546, + "num_tokens": 403893.0, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "sampling/importance_sampling_ratio/max": 2.3430161476135254, + "sampling/importance_sampling_ratio/mean": 0.7769261598587036, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.36153650283813477, + "sampling/sampling_logp_difference/mean": 0.028799494728446007, + "step": 43, + "step_time": 16.90863296529278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 494.1875, + "completions/mean_terminated_length": 494.1875, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "entropy": 1.2924985438585281, + "epoch": 0.088, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3432070314884186, + "kl": 0.014529847539961338, + "learning_rate": 3e-05, + "loss": -0.04157561436295509, + "num_tokens": 413312.0, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "sampling/importance_sampling_ratio/max": 2.726816177368164, + "sampling/importance_sampling_ratio/mean": 0.8989821672439575, + "sampling/importance_sampling_ratio/min": 0.07410597801208496, + "sampling/sampling_logp_difference/max": 0.31444358825683594, + "sampling/sampling_logp_difference/mean": 0.028122060000896454, + "step": 44, + "step_time": 16.7880685236305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 543.5, + "completions/mean_terminated_length": 543.5, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "entropy": 1.206408604979515, + "epoch": 0.09, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36233776807785034, + "kl": 0.015796773659531027, + "learning_rate": 3e-05, + "loss": 0.029176680371165276, + "num_tokens": 423624.0, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "sampling/importance_sampling_ratio/max": 1.677195429801941, + "sampling/importance_sampling_ratio/mean": 0.6537867188453674, + "sampling/importance_sampling_ratio/min": 0.09822077304124832, + "sampling/sampling_logp_difference/max": 0.31381869316101074, + "sampling/sampling_logp_difference/mean": 0.02717999368906021, + "step": 45, + "step_time": 23.35960763087496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 643.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 534.5625, + "completions/mean_terminated_length": 534.5625, + "completions/min_length": 453.0, + "completions/min_terminated_length": 453.0, + "entropy": 1.2577891275286674, + "epoch": 0.092, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.20219561457633972, + "kl": 0.014481160265859216, + "learning_rate": 3e-05, + "loss": 0.18850615620613098, + "num_tokens": 433817.0, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "sampling/importance_sampling_ratio/max": 2.203894853591919, + "sampling/importance_sampling_ratio/mean": 0.697495698928833, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.35237741470336914, + "sampling/sampling_logp_difference/mean": 0.02723248302936554, + "step": 46, + "step_time": 19.56032704282552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 538.25, + "completions/mean_terminated_length": 538.25, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "entropy": 1.3271892964839935, + "epoch": 0.094, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.22262753546237946, + "kl": 0.012554377142805606, + "learning_rate": 3e-05, + "loss": 0.06984467804431915, + "num_tokens": 444045.0, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "sampling/importance_sampling_ratio/max": 2.2261502742767334, + "sampling/importance_sampling_ratio/mean": 0.6712950468063354, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.4496123790740967, + "sampling/sampling_logp_difference/mean": 0.028838323429226875, + "step": 47, + "step_time": 21.226045075803995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 514.25, + "completions/mean_terminated_length": 514.25, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "entropy": 1.1054812744259834, + "epoch": 0.096, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3244606554508209, + "kl": 0.0157591889728792, + "learning_rate": 3e-05, + "loss": 0.09024985134601593, + "num_tokens": 453945.0, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "sampling/importance_sampling_ratio/max": 2.6745388507843018, + "sampling/importance_sampling_ratio/mean": 0.7607913017272949, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3650559186935425, + "sampling/sampling_logp_difference/mean": 0.026116060093045235, + "step": 48, + "step_time": 17.565261672716588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 585.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 502.4375, + "completions/mean_terminated_length": 502.4375, + "completions/min_length": 439.0, + "completions/min_terminated_length": 439.0, + "entropy": 1.2281213253736496, + "epoch": 0.098, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4033137857913971, + "kl": 0.015613102470524609, + "learning_rate": 3e-05, + "loss": -0.2898017466068268, + "num_tokens": 463576.0, + "reward": 6.4375, + "reward_std": 1.2632629871368408, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.2632629871368408, + "sampling/importance_sampling_ratio/max": 2.6974196434020996, + "sampling/importance_sampling_ratio/mean": 0.7124314308166504, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 0.3675417900085449, + "sampling/sampling_logp_difference/mean": 0.028341906145215034, + "step": 49, + "step_time": 29.838082558009773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 530.1875, + "completions/mean_terminated_length": 530.1875, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 1.1955150067806244, + "epoch": 0.1, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.23144562542438507, + "kl": 0.01374751579714939, + "learning_rate": 3e-05, + "loss": -0.19065965712070465, + "num_tokens": 473915.0, + "reward": 6.3125, + "reward_std": 1.1954776048660278, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.1954776048660278, + "sampling/importance_sampling_ratio/max": 2.54063081741333, + "sampling/importance_sampling_ratio/mean": 0.8162041902542114, + "sampling/importance_sampling_ratio/min": 0.1628064066171646, + "sampling/sampling_logp_difference/max": 0.2777421474456787, + "sampling/sampling_logp_difference/mean": 0.02853373810648918, + "step": 50, + "step_time": 16.047010839451104 + } + ], + "logging_steps": 1, + "max_steps": 300, + "num_input_tokens_seen": 473915, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/outputs/E0-baseline/checkpoint-50/training_args.bin b/outputs/E0-baseline/checkpoint-50/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c3a2693682d14c331dbd12e8872552fc28d54b49 --- /dev/null +++ b/outputs/E0-baseline/checkpoint-50/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6 +size 7697 diff --git a/outputs/E0-baseline/final/README.md b/outputs/E0-baseline/final/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88990785a4695219513364eafae195070ddd3182 --- /dev/null +++ b/outputs/E0-baseline/final/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3-4B-Instruct-2507 +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507 +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.20.0 \ No newline at end of file diff --git a/outputs/E0-baseline/final/adapter_config.json b/outputs/E0-baseline/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3b383bef044f277e3ff79f0fc5addabd3c441593 --- /dev/null +++ b/outputs/E0-baseline/final/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.20.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "down_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/outputs/E0-baseline/final/adapter_model.safetensors b/outputs/E0-baseline/final/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eef8d99a1c73429a073cd00c931eb0288438ca32 --- /dev/null +++ b/outputs/E0-baseline/final/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f546b7b6fd17327ebb9554c32f1730033ed34378b532ee5fe945489c3a60734 +size 264308896 diff --git a/outputs/E0-baseline/final/chat_template.jinja b/outputs/E0-baseline/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286 --- /dev/null +++ b/outputs/E0-baseline/final/chat_template.jinja @@ -0,0 +1,61 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/outputs/E0-baseline/final/tokenizer.json b/outputs/E0-baseline/final/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/outputs/E0-baseline/final/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/outputs/E0-baseline/final/tokenizer_config.json b/outputs/E0-baseline/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cb8796246653e62fc1182bdf92f80223a4c820f0 --- /dev/null +++ b/outputs/E0-baseline/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/outputs/E0-baseline/final/training_args.bin b/outputs/E0-baseline/final/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c3a2693682d14c331dbd12e8872552fc28d54b49 --- /dev/null +++ b/outputs/E0-baseline/final/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054a6cc4f7643325a59faa12921b65b0b0cb89abc2567355f5241d625f6a64f6 +size 7697 diff --git a/outputs/E0-baseline/judge_cost.json b/outputs/E0-baseline/judge_cost.json new file mode 100644 index 0000000000000000000000000000000000000000..8cddd4307f4ee6a946137355ce0522f58ff75c50 --- /dev/null +++ b/outputs/E0-baseline/judge_cost.json @@ -0,0 +1,10 @@ +{ + "calls": 5109, + "tok_in": 5592730, + "tok_out": 219835, + "usd": 0.8445, + "rows": 26510, + "hits": 32, + "misses": 4722, + "hit_rate": 0.006731173748422381 +} \ No newline at end of file diff --git a/outputs/E0-baseline/reward_history.json b/outputs/E0-baseline/reward_history.json new file mode 100644 index 0000000000000000000000000000000000000000..54e3e65d0f66128cdaa5bf74f38dfa02b154a9d0 --- /dev/null +++ b/outputs/E0-baseline/reward_history.json @@ -0,0 +1,4550 @@ +[ + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 5.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.09786997271303743, + "mean_logdet": -14.7618362097306, + "mean_marginal": -5.689893001203927e-16, + "mean_words": 388.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.1875, + "mean_quality_passing": 6.1875, + "mean_novelty": 6.125, + "frac_above_tau": 0.875, + "mean_deviation": 0.15230412729235487, + "mean_logdet": -11.656306706583043, + "mean_marginal": -5.551115123125783e-17, + "mean_words": 405.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.13520624696384095, + "mean_logdet": -12.56569596438312, + "mean_marginal": 6.106226635438361e-16, + "mean_words": 362.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.25, + "mean_quality_passing": 6.25, + "mean_novelty": 5.75, + "frac_above_tau": 0.9375, + "mean_deviation": 0.09828178674419147, + "mean_logdet": -14.591064867581835, + "mean_marginal": 0.0, + "mean_words": 375.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 5.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.12220399520689137, + "mean_logdet": -13.191277739016584, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 370.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.1875, + "mean_quality_passing": 6.1875, + "mean_novelty": 5.25, + "frac_above_tau": 0.9375, + "mean_deviation": 0.16270416556476885, + "mean_logdet": -11.446132313435005, + "mean_marginal": 2.498001805406602e-16, + "mean_words": 390.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.1875, + "mean_quality_passing": 7.1875, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.133979640877814, + "mean_logdet": -12.621398539221682, + "mean_marginal": 8.049116928532385e-16, + "mean_words": 384.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.15088045464189098, + "mean_logdet": -12.104591366197608, + "mean_marginal": 2.0816681711721685e-17, + "mean_words": 361.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.15967408945213435, + "mean_logdet": -11.44422095456766, + "mean_marginal": 6.938893903907228e-18, + "mean_words": 362.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 5.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.16095105490753808, + "mean_logdet": -11.450433035801154, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 378.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 5.8125, + "frac_above_tau": 1.0, + "mean_deviation": 0.14783913862469467, + "mean_logdet": -11.923802901974232, + "mean_marginal": 2.0816681711721685e-17, + "mean_words": 342.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 5.9375, + "mean_quality_passing": 5.9375, + "mean_novelty": 5.1875, + "frac_above_tau": 0.875, + "mean_deviation": 0.1178422009575646, + "mean_logdet": -13.35662228222406, + "mean_marginal": 1.5959455978986625e-15, + "mean_words": 356.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.18731156755230277, + "mean_logdet": -10.616158081469674, + "mean_marginal": -1.942890293094024e-16, + "mean_words": 337.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.5, + "ends_cleanly": 0.5, + "mean_quality": 3.1875, + "mean_quality_passing": 6.375, + "mean_novelty": 6.25, + "frac_above_tau": 0.5, + "mean_deviation": 0.15106223637168276, + "mean_logdet": -11.98268410285274, + "mean_marginal": -4.163336342344337e-17, + "mean_words": 335.6875, + "frac_groups_degenerate": 0.5, + "reasons": { + "no_terminal_punctuation": 8 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 5.875, + "mean_quality_passing": 5.875, + "mean_novelty": 5.3125, + "frac_above_tau": 0.9375, + "mean_deviation": 0.19060850785201539, + "mean_logdet": -10.107508014321265, + "mean_marginal": -2.636779683484747e-16, + "mean_words": 339.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.8125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13592213642437956, + "mean_logdet": -12.339188496832326, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 359.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.0625, + "mean_quality_passing": 6.0625, + "mean_novelty": 5.1875, + "frac_above_tau": 0.875, + "mean_deviation": 0.10470612046395067, + "mean_logdet": -14.218690072100909, + "mean_marginal": 7.771561172376096e-16, + "mean_words": 348.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.0, + "mean_quality_passing": 6.0, + "mean_novelty": 5.25, + "frac_above_tau": 0.9375, + "mean_deviation": 0.12026318109330966, + "mean_logdet": -13.129123048587536, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 329.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.1875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1539853735469412, + "mean_logdet": -11.972482875566364, + "mean_marginal": 2.7755575615628914e-16, + "mean_words": 330.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.125, + "mean_quality_passing": 6.125, + "mean_novelty": 5.875, + "frac_above_tau": 0.875, + "mean_deviation": 0.1898921125946579, + "mean_logdet": -10.720649656673649, + "mean_marginal": 2.636779683484747e-16, + "mean_words": 336.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.75, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1339140093470833, + "mean_logdet": -12.980463755519185, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 340.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.5, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11054886724987965, + "mean_logdet": -13.965441916304744, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 316.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 5.9375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.13599822801484918, + "mean_logdet": -12.531752218285948, + "mean_marginal": 8.049116928532385e-16, + "mean_words": 331.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.1249719138946247, + "mean_logdet": -12.930470838601156, + "mean_marginal": -4.996003610813204e-16, + "mean_words": 342.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13237185569115262, + "mean_logdet": -12.6473124503175, + "mean_marginal": 0.0, + "mean_words": 360.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.25, + "mean_quality_passing": 6.25, + "mean_novelty": 5.6875, + "frac_above_tau": 0.875, + "mean_deviation": 0.15909527668073709, + "mean_logdet": -11.634718226410852, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 331.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 5.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.11076120953132665, + "mean_logdet": -14.06673227895079, + "mean_marginal": 7.216449660063518e-16, + "mean_words": 341.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.1551268508905698, + "mean_logdet": -11.936592938496995, + "mean_marginal": 0.0, + "mean_words": 351.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 5.75, + "mean_quality_passing": 5.75, + "mean_novelty": 5.4375, + "frac_above_tau": 0.75, + "mean_deviation": 0.14338677175399492, + "mean_logdet": -12.002256195421825, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 337.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.12388036964885671, + "mean_logdet": -13.147093291032002, + "mean_marginal": 7.216449660063518e-16, + "mean_words": 373.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.0625, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1258629932492184, + "mean_logdet": -12.88837531003945, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 370.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.14247678262070435, + "mean_logdet": -12.386798162899067, + "mean_marginal": 8.465450562766819e-16, + "mean_words": 344.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 5.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.15125762557854008, + "mean_logdet": -11.59586482407463, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 364.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.1875, + "mean_quality_passing": 6.1875, + "mean_novelty": 5.5625, + "frac_above_tau": 0.875, + "mean_deviation": 0.09902503116220267, + "mean_logdet": -14.818470226783445, + "mean_marginal": -2.7755575615628914e-16, + "mean_words": 340.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.13815591564004964, + "mean_logdet": -12.605248953488616, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 331.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13577130615525512, + "mean_logdet": -12.562972344685072, + "mean_marginal": 5.273559366969494e-16, + "mean_words": 364.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 5.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.10220116724166231, + "mean_logdet": -14.459259442615565, + "mean_marginal": -7.494005416219807e-16, + "mean_words": 352.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.19752187026142903, + "mean_logdet": -10.002953827531085, + "mean_marginal": 1.6653345369377348e-16, + "mean_words": 328.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.11405571886287445, + "mean_logdet": -13.595777815337794, + "mean_marginal": -7.7021722333370235e-16, + "mean_words": 340.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.16114464966903938, + "mean_logdet": -11.337665796590851, + "mean_marginal": 3.469446951953614e-18, + "mean_words": 342.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 5.3125, + "frac_above_tau": 0.9375, + "mean_deviation": 0.12454516608121377, + "mean_logdet": -13.417377003979897, + "mean_marginal": 0.0, + "mean_words": 383.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.25, + "frac_above_tau": 0.9375, + "mean_deviation": 0.08996690458960871, + "mean_logdet": -15.324811133417143, + "mean_marginal": 7.355227538141662e-16, + "mean_words": 364.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 5.75, + "frac_above_tau": 0.875, + "mean_deviation": 0.09914124635805122, + "mean_logdet": -14.643872128520876, + "mean_marginal": -7.771561172376096e-16, + "mean_words": 374.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.15539429364210366, + "mean_logdet": -11.869435699552167, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 370.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 5.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.1161994658458167, + "mean_logdet": -14.000720221570045, + "mean_marginal": -4.440892098500626e-16, + "mean_words": 399.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.11230565770634138, + "mean_logdet": -13.697275580643637, + "mean_marginal": 5.134781488891349e-16, + "mean_words": 390.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.0625, + "mean_quality_passing": 6.0625, + "mean_novelty": 5.0625, + "frac_above_tau": 0.875, + "mean_deviation": 0.15247239934764706, + "mean_logdet": -11.82234785995431, + "mean_marginal": -2.0816681711721685e-17, + "mean_words": 403.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 6.0625, + "frac_above_tau": 1.0, + "mean_deviation": 0.11718332484411352, + "mean_logdet": -14.484779946222977, + "mean_marginal": 6.106226635438361e-16, + "mean_words": 363.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.0, + "frac_above_tau": 0.875, + "mean_deviation": 0.07803340952704749, + "mean_logdet": -16.351623212651077, + "mean_marginal": -4.440892098500626e-16, + "mean_words": 378.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.3125, + "mean_quality_passing": 6.3125, + "mean_novelty": 6.0625, + "frac_above_tau": 0.875, + "mean_deviation": 0.08511093625514117, + "mean_logdet": -16.04576182139644, + "mean_marginal": -4.163336342344337e-16, + "mean_words": 382.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 5.6875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.08682172232453064, + "mean_logdet": -15.652428344240645, + "mean_marginal": 2.0816681711721685e-17, + "mean_words": 406.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.1157885528308996, + "mean_logdet": -13.990894412064026, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 413.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.17069189666683626, + "mean_logdet": -11.302930706418547, + "mean_marginal": 2.498001805406602e-16, + "mean_words": 419.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 5.6875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.14691874661049353, + "mean_logdet": -12.048122319157285, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 359.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.1475665712137442, + "mean_logdet": -12.063264317976776, + "mean_marginal": 0.0, + "mean_words": 399.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 5.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.11225661054995166, + "mean_logdet": -13.71431546429357, + "mean_marginal": 1.1379786002407855e-15, + "mean_words": 429.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.15983024764624826, + "mean_logdet": -11.32595058341754, + "mean_marginal": 0.0, + "mean_words": 463.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.0625, + "mean_quality_passing": 6.0625, + "mean_novelty": 6.4375, + "frac_above_tau": 0.8125, + "mean_deviation": 0.1210169015411428, + "mean_logdet": -13.863600142119644, + "mean_marginal": 3.608224830031759e-16, + "mean_words": 407.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.3125, + "frac_above_tau": 0.875, + "mean_deviation": 0.09190717937317547, + "mean_logdet": -15.04151994699056, + "mean_marginal": -5.551115123125783e-16, + "mean_words": 448.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.0625, + "mean_quality_passing": 6.0625, + "mean_novelty": 5.3125, + "frac_above_tau": 0.875, + "mean_deviation": 0.1082573035090067, + "mean_logdet": -14.034084727334724, + "mean_marginal": -1.0547118733938987e-15, + "mean_words": 425.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.0625, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1501440560502062, + "mean_logdet": -11.895229519310638, + "mean_marginal": 0.0, + "mean_words": 414.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.1251602834224983, + "mean_logdet": -12.802501974298153, + "mean_marginal": -1.0547118733938987e-15, + "mean_words": 451.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.11151255859880993, + "mean_logdet": -13.85709043730332, + "mean_marginal": -5.551115123125783e-16, + "mean_words": 445.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.10373559501783933, + "mean_logdet": -14.575698297629078, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 416.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 6.875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.23518681002712816, + "mean_logdet": -8.649666188454407, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 458.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.1841847478190639, + "mean_logdet": -10.413375827187682, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 468.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.125, + "mean_quality_passing": 7.125, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.11944207909166402, + "mean_logdet": -13.414441213796582, + "mean_marginal": 1.0408340855860843e-17, + "mean_words": 386.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 1.0, + "mean_quality": 5.875, + "mean_quality_passing": 6.266666666666667, + "mean_novelty": 6.4, + "frac_above_tau": 0.75, + "mean_deviation": 0.20893125082045177, + "mean_logdet": -9.916211420337628, + "mean_marginal": 2.498001805406602e-16, + "mean_words": 501.25, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_long(612>600)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.1218264080684763, + "mean_logdet": -13.53173130158396, + "mean_marginal": -6.106226635438361e-16, + "mean_words": 454.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.11764344362735751, + "mean_logdet": -13.651268335713993, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 452.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.1593957607803725, + "mean_logdet": -12.488311325224537, + "mean_marginal": 6.245004513516506e-16, + "mean_words": 437.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 7.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.1308096513170694, + "mean_logdet": -13.394646296671329, + "mean_marginal": -4.85722573273506e-16, + "mean_words": 447.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12184697872794464, + "mean_logdet": -13.220245400272093, + "mean_marginal": 1.0824674490095276e-15, + "mean_words": 438.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.0625, + "frac_above_tau": 1.0, + "mean_deviation": 0.143407892138322, + "mean_logdet": -12.343552512125262, + "mean_marginal": -1.2351231148954867e-15, + "mean_words": 435.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.15988422887145914, + "mean_logdet": -11.385245550783733, + "mean_marginal": 8.326672684688674e-17, + "mean_words": 449.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.125, + "frac_above_tau": 0.9375, + "mean_deviation": 0.13972692539589843, + "mean_logdet": -12.287716251751123, + "mean_marginal": 6.869504964868156e-16, + "mean_words": 438.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.75, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1285094667902555, + "mean_logdet": -13.171774872877641, + "mean_marginal": 2.0816681711721685e-17, + "mean_words": 417.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.11708625944519734, + "mean_logdet": -13.43965222749237, + "mean_marginal": 6.38378239159465e-16, + "mean_words": 436.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.0, + "frac_above_tau": 0.9375, + "mean_deviation": 0.15894742231928105, + "mean_logdet": -11.420743315371023, + "mean_marginal": -4.163336342344337e-17, + "mean_words": 425.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.125981756260979, + "mean_logdet": -13.3476875375876, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 419.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.0, + "mean_quality_passing": 6.0, + "mean_novelty": 5.6875, + "frac_above_tau": 0.875, + "mean_deviation": 0.13661337477307775, + "mean_logdet": -13.266699884592903, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 400.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.5625, + "frac_above_tau": 0.875, + "mean_deviation": 0.13353369262878798, + "mean_logdet": -12.554825670738783, + "mean_marginal": -6.938893903907228e-18, + "mean_words": 429.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 5.75, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1162713781406164, + "mean_logdet": -13.705588883023417, + "mean_marginal": 4.996003610813204e-16, + "mean_words": 431.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 6.0, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11914784601983847, + "mean_logdet": -13.615859341479759, + "mean_marginal": 0.0, + "mean_words": 410.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.11124396653457128, + "mean_logdet": -13.939581199690982, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 416.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 5.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.13266923340044512, + "mean_logdet": -13.41300487597974, + "mean_marginal": 8.465450562766819e-16, + "mean_words": 405.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.3125, + "mean_quality_passing": 6.3125, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.12566745638133991, + "mean_logdet": -13.198134240470594, + "mean_marginal": -4.718447854656915e-16, + "mean_words": 383.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 5.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.15250590735982172, + "mean_logdet": -11.728449685493974, + "mean_marginal": 0.0, + "mean_words": 398.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 6.4375, + "frac_above_tau": 0.875, + "mean_deviation": 0.1783465446758607, + "mean_logdet": -10.77694623788883, + "mean_marginal": -6.661338147750939e-16, + "mean_words": 404.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.125, + "mean_quality_passing": 6.125, + "mean_novelty": 5.875, + "frac_above_tau": 0.875, + "mean_deviation": 0.12855895246060378, + "mean_logdet": -12.979653511632446, + "mean_marginal": 7.494005416219807e-16, + "mean_words": 412.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.8125, + "frac_above_tau": 1.0, + "mean_deviation": 0.1314127230809479, + "mean_logdet": -14.061509546154367, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 402.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.125, + "mean_quality_passing": 7.125, + "mean_novelty": 6.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.13476143846047117, + "mean_logdet": -12.44866662157046, + "mean_marginal": 9.43689570931383e-16, + "mean_words": 393.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.12622053265913538, + "mean_logdet": -13.210381074853387, + "mean_marginal": -7.216449660063518e-16, + "mean_words": 432.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 5.8125, + "mean_quality_passing": 5.8125, + "mean_novelty": 5.0625, + "frac_above_tau": 1.0, + "mean_deviation": 0.15574201433664958, + "mean_logdet": -11.759901108435688, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 422.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.15580650051889283, + "mean_logdet": -12.886897732171226, + "mean_marginal": -8.049116928532385e-16, + "mean_words": 387.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.13549382001966823, + "mean_logdet": -13.67481167958628, + "mean_marginal": 4.440892098500626e-16, + "mean_words": 409.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 6.0625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12723171959842478, + "mean_logdet": -13.027850898882782, + "mean_marginal": 6.38378239159465e-16, + "mean_words": 415.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.16856760510416308, + "mean_logdet": -11.205796756673246, + "mean_marginal": 0.0, + "mean_words": 419.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.17697900357128887, + "mean_logdet": -10.752425558436077, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 445.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.10477131660399691, + "mean_logdet": -14.454118734926121, + "mean_marginal": -5.828670879282072e-16, + "mean_words": 418.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12839187440640615, + "mean_logdet": -13.432310179606578, + "mean_marginal": -3.885780586188048e-16, + "mean_words": 426.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.09704676107192865, + "mean_logdet": -14.65299004127765, + "mean_marginal": -3.7470027081099033e-16, + "mean_words": 425.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.3125, + "mean_quality_passing": 6.3125, + "mean_novelty": 5.4375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.15374735399760808, + "mean_logdet": -12.139568641130438, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 410.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.14058099706620755, + "mean_logdet": -12.25214711936951, + "mean_marginal": 6.938893903907228e-18, + "mean_words": 416.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.12759388144836586, + "mean_logdet": -13.052671449246361, + "mean_marginal": -8.604228440844963e-16, + "mean_words": 422.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 6.1875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11616106845517195, + "mean_logdet": -13.49342294383992, + "mean_marginal": 0.0, + "mean_words": 420.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.14802689146249487, + "mean_logdet": -11.83114232256915, + "mean_marginal": 6.661338147750939e-16, + "mean_words": 417.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.125, + "mean_quality_passing": 6.125, + "mean_novelty": 5.75, + "frac_above_tau": 0.875, + "mean_deviation": 0.11719600150717478, + "mean_logdet": -13.651241244256846, + "mean_marginal": -7.91033905045424e-16, + "mean_words": 427.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.11564276840533921, + "mean_logdet": -14.173454286450585, + "mean_marginal": -3.885780586188048e-16, + "mean_words": 436.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13405761578816783, + "mean_logdet": -12.359945121007646, + "mean_marginal": 5.551115123125783e-17, + "mean_words": 425.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13294969090571002, + "mean_logdet": -13.125215227507773, + "mean_marginal": 5.551115123125783e-17, + "mean_words": 375.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.16423785184050393, + "mean_logdet": -11.485847642390297, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 425.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.11751043366502496, + "mean_logdet": -13.59463259331558, + "mean_marginal": 3.885780586188048e-16, + "mean_words": 420.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.25, + "mean_quality_passing": 7.25, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.1302397988085617, + "mean_logdet": -12.95339163304962, + "mean_marginal": -3.885780586188048e-16, + "mean_words": 430.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.1436680614638018, + "mean_logdet": -12.570784317504858, + "mean_marginal": 6.661338147750939e-16, + "mean_words": 438.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.09900842459225148, + "mean_logdet": -14.614595772321366, + "mean_marginal": -1.1657341758564144e-15, + "mean_words": 466.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.12513092231622044, + "mean_logdet": -13.05651510837423, + "mean_marginal": -6.661338147750939e-16, + "mean_words": 423.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 5.625, + "frac_above_tau": 0.875, + "mean_deviation": 0.14487563011359858, + "mean_logdet": -12.456356677946896, + "mean_marginal": 1.2212453270876722e-15, + "mean_words": 430.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 1.0, + "mean_quality": 6.25, + "mean_quality_passing": 6.666666666666667, + "mean_novelty": 5.8, + "frac_above_tau": 0.8125, + "mean_deviation": 0.15264299767263673, + "mean_logdet": -11.882797170450445, + "mean_marginal": 0.0, + "mean_words": 496.8125, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_long(645>600)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.11718429210575709, + "mean_logdet": -13.545069400758905, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 464.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 1.0, + "mean_quality": 5.625, + "mean_quality_passing": 6.0, + "mean_novelty": 5.4, + "frac_above_tau": 0.8125, + "mean_deviation": 0.13849432793048355, + "mean_logdet": -13.462648207779125, + "mean_marginal": -5.48172618408671e-16, + "mean_words": 455.375, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_short(45<150)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.0625, + "mean_quality_passing": 6.0625, + "mean_novelty": 5.1875, + "frac_above_tau": 0.875, + "mean_deviation": 0.09890563151149617, + "mean_logdet": -14.81050343778649, + "mean_marginal": 5.273559366969494e-16, + "mean_words": 436.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.11395571665711396, + "mean_logdet": -13.916459700875164, + "mean_marginal": -6.661338147750939e-16, + "mean_words": 429.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.11991485098575086, + "mean_logdet": -13.221380327136362, + "mean_marginal": 6.591949208711867e-16, + "mean_words": 480.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.12344019849750124, + "mean_logdet": -13.085973268377654, + "mean_marginal": -6.800116025829084e-16, + "mean_words": 444.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.11971363346611608, + "mean_logdet": -13.365873935273022, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 415.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.3125, + "mean_quality_passing": 6.3125, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.19053185368265582, + "mean_logdet": -10.790294853183116, + "mean_marginal": -3.885780586188048e-16, + "mean_words": 478.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.1517905326781584, + "mean_logdet": -11.839973163118719, + "mean_marginal": 0.0, + "mean_words": 462.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.10243797735037757, + "mean_logdet": -14.573296681157181, + "mean_marginal": -3.885780586188048e-16, + "mean_words": 441.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.1875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.10704709386552685, + "mean_logdet": -14.814769261176863, + "mean_marginal": 7.216449660063518e-16, + "mean_words": 412.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.0916240097218435, + "mean_logdet": -15.304532986637831, + "mean_marginal": -4.996003610813204e-16, + "mean_words": 429.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 6.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.13639266756462481, + "mean_logdet": -13.029325192218263, + "mean_marginal": 0.0, + "mean_words": 436.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 6.0, + "frac_above_tau": 0.9375, + "mean_deviation": 0.10438587714711356, + "mean_logdet": -15.382692638666597, + "mean_marginal": 0.0, + "mean_words": 438.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.5, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1581758512634862, + "mean_logdet": -13.307439573703395, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 392.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.09628726489791903, + "mean_logdet": -15.443665695935891, + "mean_marginal": -1.8041124150158794e-16, + "mean_words": 425.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 5.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13097423122421867, + "mean_logdet": -12.692819323895908, + "mean_marginal": -4.163336342344337e-17, + "mean_words": 392.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13568350014947517, + "mean_logdet": -12.534297862639056, + "mean_marginal": -7.494005416219807e-16, + "mean_words": 395.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11949799187459331, + "mean_logdet": -13.46453789817933, + "mean_marginal": 5.551115123125783e-17, + "mean_words": 403.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.14309652918403892, + "mean_logdet": -12.51254675757987, + "mean_marginal": -6.522560269672795e-16, + "mean_words": 391.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.375, + "mean_novelty": 5.6875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.10417984825673109, + "mean_logdet": -14.392782411109817, + "mean_marginal": -4.163336342344337e-16, + "mean_words": 373.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.1277135539987932, + "mean_logdet": -12.859520047233403, + "mean_marginal": -3.608224830031759e-16, + "mean_words": 379.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.10899042872911399, + "mean_logdet": -14.61918815399735, + "mean_marginal": 3.3306690738754696e-16, + "mean_words": 345.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 5.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.1424924747940422, + "mean_logdet": -12.502701008608376, + "mean_marginal": 0.0, + "mean_words": 355.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 0.9375, + "mean_quality": 5.75, + "mean_quality_passing": 6.133333333333334, + "mean_novelty": 5.533333333333333, + "frac_above_tau": 0.875, + "mean_deviation": 0.21721328481425384, + "mean_logdet": -9.519226690303357, + "mean_marginal": -2.914335439641036e-16, + "mean_words": 378.0, + "frac_groups_degenerate": 0.0, + "reasons": { + "no_terminal_punctuation": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 5.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.12881791403898868, + "mean_logdet": -13.036507073698688, + "mean_marginal": 7.077671781985373e-16, + "mean_words": 370.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.9375, + "frac_above_tau": 0.875, + "mean_deviation": 0.1336420422808151, + "mean_logdet": -12.855465869110972, + "mean_marginal": -4.3021142204224816e-16, + "mean_words": 340.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.0891753621011418, + "mean_logdet": -15.228748045675413, + "mean_marginal": -1.0408340855860843e-16, + "mean_words": 344.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 5.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.12369272716412127, + "mean_logdet": -14.35106671345763, + "mean_marginal": 2.7755575615628914e-16, + "mean_words": 339.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 5.9375, + "mean_quality_passing": 5.9375, + "mean_novelty": 6.125, + "frac_above_tau": 0.8125, + "mean_deviation": 0.13744421703594692, + "mean_logdet": -12.430593609579773, + "mean_marginal": 0.0, + "mean_words": 349.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.10341495357907568, + "mean_logdet": -14.640445739205655, + "mean_marginal": -4.163336342344337e-16, + "mean_words": 357.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 5.8125, + "mean_quality_passing": 5.8125, + "mean_novelty": 4.9375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.15846343940627827, + "mean_logdet": -11.69441082917363, + "mean_marginal": -3.122502256758253e-17, + "mean_words": 375.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.5, + "ends_cleanly": 1.0, + "mean_quality": 3.4375, + "mean_quality_passing": 6.875, + "mean_novelty": 5.875, + "frac_above_tau": 0.5, + "mean_deviation": 0.2122631656780177, + "mean_logdet": -11.510883334358976, + "mean_marginal": 3.608224830031759e-16, + "mean_words": 219.0625, + "frac_groups_degenerate": 0.5, + "reasons": { + "too_short(87<150)": 1, + "too_short(93<150)": 1, + "too_short(81<150)": 1, + "too_short(98<150)": 1, + "too_short(86<150)": 1, + "too_short(97<150)": 1, + "too_short(104<150)": 1, + "too_short(101<150)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.11888299717240455, + "mean_logdet": -13.354443060360346, + "mean_marginal": -5.551115123125783e-17, + "mean_words": 372.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 7.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.0769693354493873, + "mean_logdet": -16.426433835106103, + "mean_marginal": 3.608224830031759e-16, + "mean_words": 349.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 5.5625, + "mean_quality_passing": 5.5625, + "mean_novelty": 4.375, + "frac_above_tau": 0.75, + "mean_deviation": 0.1573195232567261, + "mean_logdet": -12.858953073624175, + "mean_marginal": -3.191891195797325e-16, + "mean_words": 297.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.1323987734792343, + "mean_logdet": -12.812682513704925, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 371.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.09845539438540339, + "mean_logdet": -14.789518341876345, + "mean_marginal": -4.163336342344337e-17, + "mean_words": 337.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.1875, + "mean_quality_passing": 6.1875, + "mean_novelty": 5.25, + "frac_above_tau": 0.9375, + "mean_deviation": 0.16440390083925777, + "mean_logdet": -11.31021155015073, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 363.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.14547538355778158, + "mean_logdet": -12.527284802817931, + "mean_marginal": -5.273559366969494e-16, + "mean_words": 348.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.8125, + "ends_cleanly": 1.0, + "mean_quality": 5.8125, + "mean_quality_passing": 7.153846153846154, + "mean_novelty": 6.230769230769231, + "frac_above_tau": 0.8125, + "mean_deviation": 0.13968857635058884, + "mean_logdet": -12.989708177772297, + "mean_marginal": -3.3306690738754696e-16, + "mean_words": 356.1875, + "frac_groups_degenerate": 0.0, + "reasons": { + "ngram_loop(0.30)": 1, + "ngram_loop(0.26)": 1, + "line_loop(0.38)": 1, + "line_loop(0.28)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.144928384920603, + "mean_logdet": -11.927452722130392, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 371.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.0625, + "frac_above_tau": 1.0, + "mean_deviation": 0.15232611476132524, + "mean_logdet": -11.659839502794297, + "mean_marginal": 6.938893903907228e-18, + "mean_words": 350.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.16595883938291156, + "mean_logdet": -11.761776076786333, + "mean_marginal": -6.938893903907228e-18, + "mean_words": 372.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.4375, + "frac_above_tau": 0.875, + "mean_deviation": 0.08198768858712464, + "mean_logdet": -16.11695950720972, + "mean_marginal": 5.551115123125783e-16, + "mean_words": 369.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.5, + "frac_above_tau": 0.9375, + "mean_deviation": 0.10906162083997575, + "mean_logdet": -13.929824398803156, + "mean_marginal": -7.216449660063518e-16, + "mean_words": 373.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.25, + "mean_quality_passing": 7.25, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.14449811329419812, + "mean_logdet": -12.316020662407329, + "mean_marginal": 2.914335439641036e-16, + "mean_words": 386.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.16575391379975934, + "mean_logdet": -11.017926582944304, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 367.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.08673547402623392, + "mean_logdet": -15.894324193509398, + "mean_marginal": 0.0, + "mean_words": 392.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.1727244710134887, + "mean_logdet": -10.805489864996122, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 378.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.17793137804765907, + "mean_logdet": -11.461678688991858, + "mean_marginal": -5.828670879282072e-16, + "mean_words": 406.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 5.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.13930656763998034, + "mean_logdet": -12.92637049636929, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 362.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.13932218692001141, + "mean_logdet": -12.903903142762935, + "mean_marginal": -3.885780586188048e-16, + "mean_words": 404.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.125, + "mean_quality_passing": 7.125, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.16005518459619356, + "mean_logdet": -11.332929140533931, + "mean_marginal": 0.0, + "mean_words": 363.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11978119633234902, + "mean_logdet": -13.43929553540443, + "mean_marginal": -3.677613769070831e-16, + "mean_words": 398.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.09215090834489108, + "mean_logdet": -15.38712179715149, + "mean_marginal": 0.0, + "mean_words": 346.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.19757689308293389, + "mean_logdet": -10.277751391408481, + "mean_marginal": 0.0, + "mean_words": 394.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.139438935111374, + "mean_logdet": -12.350342578201762, + "mean_marginal": 4.2327252813834093e-16, + "mean_words": 364.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.5, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1618533581681497, + "mean_logdet": -11.246987061873957, + "mean_marginal": 0.0, + "mean_words": 387.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 6.125, + "frac_above_tau": 0.9375, + "mean_deviation": 0.14720983853984587, + "mean_logdet": -12.599033057397133, + "mean_marginal": 3.885780586188048e-16, + "mean_words": 369.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.1875, + "mean_quality_passing": 6.1875, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.16011961322412338, + "mean_logdet": -11.798202309146689, + "mean_marginal": 5.273559366969494e-16, + "mean_words": 390.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.1875, + "mean_quality_passing": 6.1875, + "mean_novelty": 5.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.1282391817943335, + "mean_logdet": -12.722548712678925, + "mean_marginal": 0.0, + "mean_words": 399.3125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.12509100665056133, + "mean_logdet": -13.296856834874832, + "mean_marginal": -4.0072112295064244e-16, + "mean_words": 377.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.15637460346531531, + "mean_logdet": -12.121852290453214, + "mean_marginal": 0.0, + "mean_words": 377.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.5625, + "frac_above_tau": 0.9375, + "mean_deviation": 0.13596189543046983, + "mean_logdet": -13.027390033086302, + "mean_marginal": 3.0531133177191805e-16, + "mean_words": 394.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.6875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.09257618504036458, + "mean_logdet": -15.273386054790627, + "mean_marginal": 8.049116928532385e-16, + "mean_words": 397.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.1875, + "mean_quality_passing": 6.1875, + "mean_novelty": 5.75, + "frac_above_tau": 0.875, + "mean_deviation": 0.11354435998500062, + "mean_logdet": -13.662896239467624, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 404.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.11460196351779986, + "mean_logdet": -14.330507123721537, + "mean_marginal": -1.7208456881689926e-15, + "mean_words": 390.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.1405598338592944, + "mean_logdet": -12.768757878338018, + "mean_marginal": -3.3306690738754696e-16, + "mean_words": 399.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.0625, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11653434239775987, + "mean_logdet": -13.890986712362785, + "mean_marginal": -4.718447854656915e-16, + "mean_words": 394.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11532890637745657, + "mean_logdet": -13.954164960914014, + "mean_marginal": 4.163336342344337e-17, + "mean_words": 399.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.10395924422350841, + "mean_logdet": -14.430167666104964, + "mean_marginal": -1.1102230246251565e-15, + "mean_words": 398.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.1400266797739445, + "mean_logdet": -12.259383248346117, + "mean_marginal": 0.0, + "mean_words": 408.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.14790738624082983, + "mean_logdet": -11.93381741876119, + "mean_marginal": -3.469446951953614e-17, + "mean_words": 421.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 6.1875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.10406809814332789, + "mean_logdet": -14.463493015620417, + "mean_marginal": 3.3306690738754696e-16, + "mean_words": 397.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 5.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.1347067333764395, + "mean_logdet": -13.076535379369354, + "mean_marginal": 0.0, + "mean_words": 377.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.3125, + "mean_quality_passing": 6.3125, + "mean_novelty": 5.5, + "frac_above_tau": 0.9375, + "mean_deviation": 0.154202899209226, + "mean_logdet": -11.572811851071913, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 403.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 7.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.1718740058951315, + "mean_logdet": -12.558150442352208, + "mean_marginal": 5.551115123125783e-16, + "mean_words": 420.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.09498261765077246, + "mean_logdet": -14.948254145514724, + "mean_marginal": -7.216449660063518e-16, + "mean_words": 401.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.13037173580250155, + "mean_logdet": -12.643626491135151, + "mean_marginal": -2.0816681711721685e-17, + "mean_words": 396.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.5625, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11933336734498387, + "mean_logdet": -13.179920904728311, + "mean_marginal": 7.216449660063518e-16, + "mean_words": 408.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.8, + "mean_novelty": 6.066666666666666, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11415659386552243, + "mean_logdet": -13.504330327202421, + "mean_marginal": 2.220446049250313e-16, + "mean_words": 410.3125, + "frac_groups_degenerate": 0.0, + "reasons": { + "line_loop(0.29)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.5625, + "frac_above_tau": 0.9375, + "mean_deviation": 0.17932329051208992, + "mean_logdet": -11.556905015030027, + "mean_marginal": -2.0816681711721685e-17, + "mean_words": 394.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.09948738657691389, + "mean_logdet": -14.518316434083879, + "mean_marginal": -1.1934897514720433e-15, + "mean_words": 380.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.125, + "mean_quality_passing": 7.125, + "mean_novelty": 7.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13013665645996128, + "mean_logdet": -12.86196899826972, + "mean_marginal": 6.730727086790012e-16, + "mean_words": 397.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.14862364885388635, + "mean_logdet": -12.284833345935686, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 409.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.125, + "mean_quality_passing": 7.125, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12708334393141993, + "mean_logdet": -13.210618642070862, + "mean_marginal": 3.469446951953614e-17, + "mean_words": 370.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.14357059330097707, + "mean_logdet": -12.42836359457539, + "mean_marginal": 5.828670879282072e-16, + "mean_words": 385.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.11579484855846708, + "mean_logdet": -13.461290146938303, + "mean_marginal": 3.8163916471489756e-16, + "mean_words": 374.4375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.1875, + "mean_quality_passing": 6.1875, + "mean_novelty": 5.375, + "frac_above_tau": 0.875, + "mean_deviation": 0.15373189740568907, + "mean_logdet": -11.772180800944668, + "mean_marginal": 0.0, + "mean_words": 381.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 5.8125, + "frac_above_tau": 1.0, + "mean_deviation": 0.14641437222669823, + "mean_logdet": -12.033958659267466, + "mean_marginal": 0.0, + "mean_words": 380.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 7.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.17528702773990912, + "mean_logdet": -10.809049234769796, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 366.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.22178239169478908, + "mean_logdet": -9.30999318230487, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 376.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 7.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.13910005596973804, + "mean_logdet": -12.276814755358286, + "mean_marginal": 2.0816681711721685e-17, + "mean_words": 480.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.1310753320790689, + "mean_logdet": -13.07218091102327, + "mean_marginal": 6.869504964868156e-16, + "mean_words": 365.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.125, + "frac_above_tau": 0.9375, + "mean_deviation": 0.13953192913303053, + "mean_logdet": -12.59752730431732, + "mean_marginal": 4.440892098500626e-16, + "mean_words": 389.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 7.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.12551304137841449, + "mean_logdet": -13.331163513716053, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 362.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.5625, + "ends_cleanly": 0.9375, + "mean_quality": 3.5, + "mean_quality_passing": 6.222222222222222, + "mean_novelty": 5.666666666666667, + "frac_above_tau": 0.5, + "mean_deviation": 0.25851854531654817, + "mean_logdet": -9.993086124035823, + "mean_marginal": -4.163336342344337e-17, + "mean_words": 237.75, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_short(4<150)": 1, + "too_short(32<150)": 2, + "too_short(52<150)": 1, + "no_terminal_punctuation": 1, + "too_short(9<150)": 1, + "too_short(5<150)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.5, + "mean_novelty": 5.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12138596474166662, + "mean_logdet": -13.304399925332302, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 360.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 5.3125, + "mean_quality_passing": 5.3125, + "mean_novelty": 5.0625, + "frac_above_tau": 0.6875, + "mean_deviation": 0.10998090467406714, + "mean_logdet": -14.122926027945915, + "mean_marginal": 6.245004513516506e-16, + "mean_words": 387.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.0, + "mean_quality_passing": 6.0, + "mean_novelty": 5.1875, + "frac_above_tau": 0.875, + "mean_deviation": 0.11039768854062143, + "mean_logdet": -14.00141387971972, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 322.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12912381058403605, + "mean_logdet": -12.808227958452855, + "mean_marginal": -9.43689570931383e-16, + "mean_words": 370.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.14748565837792044, + "mean_logdet": -12.464007797261406, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 330.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.125, + "mean_quality_passing": 7.125, + "mean_novelty": 6.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.13157983869123407, + "mean_logdet": -13.010769095508046, + "mean_marginal": 0.0, + "mean_words": 372.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12006212744240236, + "mean_logdet": -14.634805294801241, + "mean_marginal": 2.0816681711721685e-17, + "mean_words": 386.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.09855835571427833, + "mean_logdet": -15.128641433125328, + "mean_marginal": 5.828670879282072e-16, + "mean_words": 344.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.11560281413938547, + "mean_logdet": -14.150744402594729, + "mean_marginal": -1.1379786002407855e-15, + "mean_words": 370.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 5.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.10082311337580709, + "mean_logdet": -14.834820277297139, + "mean_marginal": -4.440892098500626e-16, + "mean_words": 364.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.16101128235442502, + "mean_logdet": -11.906054880994544, + "mean_marginal": 5.065392549852277e-16, + "mean_words": 370.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 7.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.1305644828636369, + "mean_logdet": -12.733112417045126, + "mean_marginal": -6.38378239159465e-16, + "mean_words": 363.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.14454451220370398, + "mean_logdet": -12.349376955371788, + "mean_marginal": -5.551115123125783e-16, + "mean_words": 354.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.5, + "frac_above_tau": 1.0, + "mean_deviation": 0.1551332640085773, + "mean_logdet": -11.550732102229297, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 382.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.3125, + "frac_above_tau": 0.9375, + "mean_deviation": 0.15088196647342322, + "mean_logdet": -11.840913320127799, + "mean_marginal": 0.0, + "mean_words": 403.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.14555720038562203, + "mean_logdet": -12.05560256152185, + "mean_marginal": 0.0, + "mean_words": 366.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 1.0, + "mean_quality": 6.0, + "mean_quality_passing": 6.4, + "mean_novelty": 5.866666666666666, + "frac_above_tau": 0.9375, + "mean_deviation": 0.14647583671086684, + "mean_logdet": -12.120996267761335, + "mean_marginal": 0.0, + "mean_words": 447.1875, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_long(619>600)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1737384270052124, + "mean_logdet": -10.873742942389256, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 370.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.14630641968247587, + "mean_logdet": -12.259702745737412, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 376.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.14382956803699642, + "mean_logdet": -12.282050858761234, + "mean_marginal": 0.0, + "mean_words": 399.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.25, + "mean_quality_passing": 6.25, + "mean_novelty": 6.5, + "frac_above_tau": 0.875, + "mean_deviation": 0.15808013975810814, + "mean_logdet": -11.874932497252935, + "mean_marginal": -3.469446951953614e-16, + "mean_words": 410.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.8125, + "frac_above_tau": 1.0, + "mean_deviation": 0.12056115078318502, + "mean_logdet": -13.49561315518799, + "mean_marginal": 5.551115123125783e-17, + "mean_words": 374.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.13865788828231052, + "mean_logdet": -12.628439307295753, + "mean_marginal": 5.551115123125783e-16, + "mean_words": 372.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 7.125, + "frac_above_tau": 1.0, + "mean_deviation": 0.12677764793542828, + "mean_logdet": -13.12835871779884, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 390.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.0625, + "frac_above_tau": 1.0, + "mean_deviation": 0.141857356661145, + "mean_logdet": -12.35347548770886, + "mean_marginal": -4.163336342344337e-17, + "mean_words": 366.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.12503180795816282, + "mean_logdet": -13.58423525991707, + "mean_marginal": 0.0, + "mean_words": 402.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.14581444098543422, + "mean_logdet": -12.178363866040032, + "mean_marginal": 3.0531133177191805e-16, + "mean_words": 416.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.1875, + "mean_quality_passing": 7.1875, + "mean_novelty": 6.6875, + "frac_above_tau": 1.0, + "mean_deviation": 0.1448991641683054, + "mean_logdet": -11.983641664728076, + "mean_marginal": 4.163336342344337e-17, + "mean_words": 371.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.8125, + "frac_above_tau": 1.0, + "mean_deviation": 0.1642909796726994, + "mean_logdet": -11.138104253184821, + "mean_marginal": 0.0, + "mean_words": 390.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.1207576623108361, + "mean_logdet": -13.4763012105994, + "mean_marginal": 5.134781488891349e-16, + "mean_words": 399.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 5.6875, + "frac_above_tau": 0.9375, + "mean_deviation": 0.15591528798088672, + "mean_logdet": -11.597378875362123, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 402.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.16404885268230754, + "mean_logdet": -11.420825966666523, + "mean_marginal": 7.771561172376096e-16, + "mean_words": 426.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.16339224477666747, + "mean_logdet": -11.273245210372322, + "mean_marginal": 0.0, + "mean_words": 394.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.12324468781972474, + "mean_logdet": -13.154259203405964, + "mean_marginal": -5.551115123125783e-16, + "mean_words": 434.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.1511541072757482, + "mean_logdet": -12.122752962636236, + "mean_marginal": 0.0, + "mean_words": 375.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12190462642011411, + "mean_logdet": -13.923255589162197, + "mean_marginal": -5.273559366969494e-16, + "mean_words": 402.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.375, + "mean_quality_passing": 7.375, + "mean_novelty": 7.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.19347646269860053, + "mean_logdet": -10.360875106940618, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 330.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.8125, + "frac_above_tau": 1.0, + "mean_deviation": 0.1336976110547623, + "mean_logdet": -12.602397376653819, + "mean_marginal": -7.771561172376096e-16, + "mean_words": 429.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.4375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11099036713439564, + "mean_logdet": -14.60371656738684, + "mean_marginal": 0.0, + "mean_words": 406.6875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.9375, + "mean_quality_passing": 6.9375, + "mean_novelty": 6.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.15413916875957306, + "mean_logdet": -11.842240595154166, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 424.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.5, + "frac_above_tau": 0.9375, + "mean_deviation": 0.15350823816653142, + "mean_logdet": -11.913410780447279, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 426.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.1875, + "mean_quality_passing": 7.1875, + "mean_novelty": 6.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.1609600905121346, + "mean_logdet": -11.345944135941487, + "mean_marginal": 5.551115123125783e-17, + "mean_words": 415.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.5625, + "frac_above_tau": 1.0, + "mean_deviation": 0.16339142215270644, + "mean_logdet": -11.577787008520708, + "mean_marginal": -3.0531133177191805e-16, + "mean_words": 403.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 7.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.11790729675753073, + "mean_logdet": -13.61985360288708, + "mean_marginal": 1.0685896612017132e-15, + "mean_words": 407.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.25, + "mean_quality_passing": 6.25, + "mean_novelty": 6.8125, + "frac_above_tau": 0.875, + "mean_deviation": 0.15466536610472004, + "mean_logdet": -12.648793472152853, + "mean_marginal": -5.551115123125783e-17, + "mean_words": 378.875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.125, + "mean_quality_passing": 6.125, + "mean_novelty": 6.375, + "frac_above_tau": 0.8125, + "mean_deviation": 0.1384532327100735, + "mean_logdet": -12.24652210201448, + "mean_marginal": 2.7755575615628914e-17, + "mean_words": 409.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.3125, + "mean_quality_passing": 6.3125, + "mean_novelty": 6.375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.18715747563751708, + "mean_logdet": -10.74380221938543, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 434.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.4375, + "mean_quality_passing": 7.4375, + "mean_novelty": 7.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.17662433469242872, + "mean_logdet": -10.761462795555019, + "mean_marginal": 0.0, + "mean_words": 363.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.18131081083954914, + "mean_logdet": -10.478790107457886, + "mean_marginal": 0.0, + "mean_words": 468.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.1823017914732138, + "mean_logdet": -10.620606550592914, + "mean_marginal": -3.5041414214731503e-16, + "mean_words": 437.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.8125, + "ends_cleanly": 0.9375, + "mean_quality": 5.75, + "mean_quality_passing": 7.076923076923077, + "mean_novelty": 6.923076923076923, + "frac_above_tau": 0.8125, + "mean_deviation": 0.1614293608451402, + "mean_logdet": -11.93976865777038, + "mean_marginal": 5.273559366969494e-16, + "mean_words": 536.5625, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_long(712>600)": 1, + "too_long(690>600)": 1, + "no_terminal_punctuation": 1, + "too_long(731>600)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.125, + "mean_quality_passing": 7.125, + "mean_novelty": 6.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.1527832598583887, + "mean_logdet": -11.54784803507784, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 453.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.5625, + "frac_above_tau": 0.9375, + "mean_deviation": 0.16397710533619958, + "mean_logdet": -10.983527516816377, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 446.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.6875, + "mean_quality_passing": 6.6875, + "mean_novelty": 5.75, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1336493934211432, + "mean_logdet": -12.622672865793817, + "mean_marginal": -4.85722573273506e-16, + "mean_words": 453.125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 6.375, + "frac_above_tau": 1.0, + "mean_deviation": 0.11412909054442336, + "mean_logdet": -13.715100624942567, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 456.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.5, + "ends_cleanly": 0.5, + "mean_quality": 3.625, + "mean_quality_passing": 7.25, + "mean_novelty": 7.625, + "frac_above_tau": 0.5, + "mean_deviation": 0.13467925615004905, + "mean_logdet": -12.775760790165464, + "mean_marginal": -3.3306690738754696e-16, + "mean_words": 461.375, + "frac_groups_degenerate": 0.5, + "reasons": { + "no_terminal_punctuation": 8 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.375, + "mean_quality_passing": 7.375, + "mean_novelty": 6.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.09664447527560815, + "mean_logdet": -14.891437316142081, + "mean_marginal": 0.0, + "mean_words": 449.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 7.2, + "mean_novelty": 7.0, + "frac_above_tau": 0.9375, + "mean_deviation": 0.17732405864256953, + "mean_logdet": -10.715316402835057, + "mean_marginal": -2.7755575615628914e-16, + "mean_words": 483.0625, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_long(687>600)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12208967826748783, + "mean_logdet": -13.374961200587403, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 452.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.125, + "mean_quality_passing": 7.125, + "mean_novelty": 7.0625, + "frac_above_tau": 1.0, + "mean_deviation": 0.14128801773110833, + "mean_logdet": -12.340161360223249, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 440.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.12321893830500202, + "mean_logdet": -13.582013793877781, + "mean_marginal": -2.7755575615628914e-17, + "mean_words": 458.9375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.3125, + "mean_quality_passing": 7.3125, + "mean_novelty": 7.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.10076390555094664, + "mean_logdet": -14.519430427772653, + "mean_marginal": -6.175615574477433e-16, + "mean_words": 461.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.1875, + "mean_quality_passing": 7.1875, + "mean_novelty": 7.1875, + "frac_above_tau": 1.0, + "mean_deviation": 0.1089326138183201, + "mean_logdet": -14.141475821082281, + "mean_marginal": -3.7470027081099033e-16, + "mean_words": 437.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.625, + "mean_quality_passing": 6.625, + "mean_novelty": 7.0, + "frac_above_tau": 1.0, + "mean_deviation": 0.10793341505275031, + "mean_logdet": -14.251201820404166, + "mean_marginal": 5.134781488891349e-16, + "mean_words": 430.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.9375, + "frac_above_tau": 1.0, + "mean_deviation": 0.10507939284994924, + "mean_logdet": -14.810529505184231, + "mean_marginal": 6.938893903907228e-18, + "mean_words": 428.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.3125, + "mean_quality_passing": 6.3125, + "mean_novelty": 6.1875, + "frac_above_tau": 0.875, + "mean_deviation": 0.13574448883209908, + "mean_logdet": -12.632304348447079, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 474.1875, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 1.0, + "mean_quality": 6.5, + "mean_quality_passing": 6.933333333333334, + "mean_novelty": 6.466666666666667, + "frac_above_tau": 0.9375, + "mean_deviation": 0.17535854716311827, + "mean_logdet": -10.80274362302392, + "mean_marginal": 2.636779683484747e-16, + "mean_words": 439.5, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_long(631>600)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.25, + "mean_quality_passing": 7.25, + "mean_novelty": 6.875, + "frac_above_tau": 1.0, + "mean_deviation": 0.16636396540772538, + "mean_logdet": -11.31932315558659, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 365.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.08633544990816813, + "mean_logdet": -15.92664297637447, + "mean_marginal": -3.3306690738754696e-16, + "mean_words": 424.8125, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.1875, + "mean_quality_passing": 7.1875, + "mean_novelty": 7.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.14157768307424584, + "mean_logdet": -12.731569792199645, + "mean_marginal": 1.3877787807814457e-17, + "mean_words": 454.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12304184835995113, + "mean_logdet": -13.207540865006168, + "mean_marginal": -9.71445146547012e-16, + "mean_words": 432.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 0.9375, + "ends_cleanly": 1.0, + "mean_quality": 6.375, + "mean_quality_passing": 6.8, + "mean_novelty": 6.733333333333333, + "frac_above_tau": 0.9375, + "mean_deviation": 0.1151546353833603, + "mean_logdet": -13.946416537712434, + "mean_marginal": 3.608224830031759e-16, + "mean_words": 475.9375, + "frac_groups_degenerate": 0.0, + "reasons": { + "too_long(789>600)": 1 + } + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.25, + "mean_quality_passing": 7.25, + "mean_novelty": 6.75, + "frac_above_tau": 1.0, + "mean_deviation": 0.10210141601169535, + "mean_logdet": -14.492420726903738, + "mean_marginal": -5.551115123125783e-17, + "mean_words": 422.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.5625, + "mean_quality_passing": 6.5625, + "mean_novelty": 6.6875, + "frac_above_tau": 0.8125, + "mean_deviation": 0.1100131281186035, + "mean_logdet": -14.262225175740657, + "mean_marginal": -1.3877787807814457e-17, + "mean_words": 433.625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.8125, + "mean_quality_passing": 6.8125, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.16635877545748728, + "mean_logdet": -11.781169902636393, + "mean_marginal": -9.992007221626409e-16, + "mean_words": 411.75, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.0625, + "frac_above_tau": 1.0, + "mean_deviation": 0.12078328728375161, + "mean_logdet": -13.279292912353284, + "mean_marginal": -4.718447854656915e-16, + "mean_words": 437.375, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.4375, + "mean_quality_passing": 6.4375, + "mean_novelty": 6.375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11303157988627499, + "mean_logdet": -14.041637324214832, + "mean_marginal": 3.885780586188048e-16, + "mean_words": 388.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.625, + "frac_above_tau": 1.0, + "mean_deviation": 0.13922230116192016, + "mean_logdet": -12.422455605074287, + "mean_marginal": 0.0, + "mean_words": 410.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.875, + "mean_quality_passing": 6.875, + "mean_novelty": 6.4375, + "frac_above_tau": 1.0, + "mean_deviation": 0.10085184988185772, + "mean_logdet": -14.66995690305722, + "mean_marginal": 0.0, + "mean_words": 403.25, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0625, + "mean_quality_passing": 7.0625, + "mean_novelty": 6.25, + "frac_above_tau": 1.0, + "mean_deviation": 0.13287621731026153, + "mean_logdet": -12.787490653665866, + "mean_marginal": 5.967448757360216e-16, + "mean_words": 415.5625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.625, + "frac_above_tau": 0.9375, + "mean_deviation": 0.11570964599371401, + "mean_logdet": -13.740873865822472, + "mean_marginal": -3.608224830031759e-16, + "mean_words": 399.0625, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 6.75, + "mean_quality_passing": 6.75, + "mean_novelty": 6.375, + "frac_above_tau": 0.9375, + "mean_deviation": 0.15970000014064228, + "mean_logdet": -11.50540441464274, + "mean_marginal": 0.0, + "mean_words": 383.0, + "frac_groups_degenerate": 0.0, + "reasons": {} + }, + { + "n": 16, + "gate_pass": 1.0, + "ends_cleanly": 1.0, + "mean_quality": 7.0, + "mean_quality_passing": 7.0, + "mean_novelty": 6.3125, + "frac_above_tau": 1.0, + "mean_deviation": 0.09427983000302391, + "mean_logdet": -14.740970233909776, + "mean_marginal": 6.938893903907228e-18, + "mean_words": 396.5, + "frac_groups_degenerate": 0.0, + "reasons": {} + } +] \ No newline at end of file diff --git a/outputs/E0-baseline/trl_log_history.json b/outputs/E0-baseline/trl_log_history.json new file mode 100644 index 0000000000000000000000000000000000000000..14e7e8e612c300fa86b03f8e4b318e45f5c3afa1 --- /dev/null +++ b/outputs/E0-baseline/trl_log_history.json @@ -0,0 +1,9911 @@ +[ + { + "loss": 0.2398601919412613, + "grad_norm": 0.22668755054473877, + "learning_rate": 0.0, + "num_tokens": 10400.0, + "completions/mean_length": 529.0, + "completions/min_length": 482.0, + "completions/max_length": 593.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 529.0, + "completions/min_terminated_length": 482.0, + "completions/max_terminated_length": 593.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.5, + "reward": 6.375, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026567919179797173, + "sampling/sampling_logp_difference/max": 0.40894174575805664, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5064857602119446, + "sampling/importance_sampling_ratio/max": 1.6477458477020264, + "kl": 0.0, + "entropy": 1.2025159299373627, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 12.760562099982053, + "epoch": 0.002, + "step": 1 + }, + { + "loss": -0.08571265637874603, + "grad_norm": 0.12967805564403534, + "learning_rate": 3e-06, + "num_tokens": 20924.0, + "completions/mean_length": 562.25, + "completions/min_length": 497.0, + "completions/max_length": 644.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 562.25, + "completions/min_terminated_length": 497.0, + "completions/max_terminated_length": 644.0, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.025086138397455215, + "sampling/sampling_logp_difference/max": 0.45699238777160645, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.41275694966316223, + "sampling/importance_sampling_ratio/max": 1.0037232637405396, + "kl": 0.0, + "entropy": 1.1930748969316483, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 11.406366533134133, + "epoch": 0.004, + "step": 2 + }, + { + "loss": 0.020913563668727875, + "grad_norm": 0.22966289520263672, + "learning_rate": 6e-06, + "num_tokens": 30222.0, + "completions/mean_length": 483.625, + "completions/min_length": 407.0, + "completions/max_length": 542.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 483.625, + "completions/min_terminated_length": 407.0, + "completions/max_terminated_length": 542.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.024864500388503075, + "sampling/sampling_logp_difference/max": 0.4620504379272461, + "sampling/importance_sampling_ratio/min": 0.11067161709070206, + "sampling/importance_sampling_ratio/mean": 0.6114993095397949, + "sampling/importance_sampling_ratio/max": 1.927120566368103, + "kl": 0.0008355328354809899, + "entropy": 1.1096689626574516, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.480680393986404, + "epoch": 0.006, + "step": 3 + }, + { + "loss": -0.12959149479866028, + "grad_norm": 0.30378466844558716, + "learning_rate": 9e-06, + "num_tokens": 40410.0, + "completions/mean_length": 524.75, + "completions/min_length": 473.0, + "completions/max_length": 608.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 524.75, + "completions/min_terminated_length": 473.0, + "completions/max_terminated_length": 608.0, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 0.8563488721847534, + "reward": 6.25, + "reward_std": 0.8563488721847534, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02656388282775879, + "sampling/sampling_logp_difference/max": 0.3680229187011719, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5457419753074646, + "sampling/importance_sampling_ratio/max": 2.406888961791992, + "kl": 0.0008403196770814247, + "entropy": 1.211122527718544, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.95301443943754, + "epoch": 0.008, + "step": 4 + }, + { + "loss": 0.11524428427219391, + "grad_norm": 0.5199307799339294, + "learning_rate": 1.2e-05, + "num_tokens": 49915.0, + "completions/mean_length": 487.5625, + "completions/min_length": 441.0, + "completions/max_length": 546.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 487.5625, + "completions/min_terminated_length": 441.0, + "completions/max_terminated_length": 546.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02625642716884613, + "sampling/sampling_logp_difference/max": 0.4774587154388428, + "sampling/importance_sampling_ratio/min": 0.04948288947343826, + "sampling/importance_sampling_ratio/mean": 0.963020920753479, + "sampling/importance_sampling_ratio/max": 2.1544158458709717, + "kl": 0.0008723233368073124, + "entropy": 1.247180238366127, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.37928077671677, + "epoch": 0.01, + "step": 5 + }, + { + "loss": 0.15093231201171875, + "grad_norm": 0.27643197774887085, + "learning_rate": 1.5e-05, + "num_tokens": 60219.0, + "completions/mean_length": 533.0, + "completions/min_length": 394.0, + "completions/max_length": 648.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 533.0, + "completions/min_terminated_length": 394.0, + "completions/max_terminated_length": 648.0, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.025986071676015854, + "sampling/sampling_logp_difference/max": 0.7049552202224731, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7733402252197266, + "sampling/importance_sampling_ratio/max": 2.125091791152954, + "kl": 0.0009261858467652928, + "entropy": 1.1693861335515976, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.00841654697433, + "epoch": 0.012, + "step": 6 + }, + { + "loss": -0.08582288026809692, + "grad_norm": 0.2166663259267807, + "learning_rate": 1.8e-05, + "num_tokens": 69902.0, + "completions/mean_length": 505.6875, + "completions/min_length": 425.0, + "completions/max_length": 574.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 505.6875, + "completions/min_terminated_length": 425.0, + "completions/max_terminated_length": 574.0, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "reward": 7.1875, + "reward_std": 0.75, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0264718160033226, + "sampling/sampling_logp_difference/max": 0.38030898571014404, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5286832451820374, + "sampling/importance_sampling_ratio/max": 2.8287386894226074, + "kl": 0.0009701631606731098, + "entropy": 1.2473183795809746, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 46.596127359196544, + "epoch": 0.014, + "step": 7 + }, + { + "loss": -0.1946130096912384, + "grad_norm": 0.18874908983707428, + "learning_rate": 2.1e-05, + "num_tokens": 79501.0, + "completions/mean_length": 482.4375, + "completions/min_length": 392.0, + "completions/max_length": 537.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 482.4375, + "completions/min_terminated_length": 392.0, + "completions/max_terminated_length": 537.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02554757334291935, + "sampling/sampling_logp_difference/max": 0.26114892959594727, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6918502449989319, + "sampling/importance_sampling_ratio/max": 2.3561792373657227, + "kl": 0.0010721117541834246, + "entropy": 1.1040107272565365, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.433595282491297, + "epoch": 0.016, + "step": 8 + }, + { + "loss": -0.08130021393299103, + "grad_norm": 0.204215869307518, + "learning_rate": 2.4e-05, + "num_tokens": 88976.0, + "completions/mean_length": 490.1875, + "completions/min_length": 463.0, + "completions/max_length": 554.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 490.1875, + "completions/min_terminated_length": 463.0, + "completions/max_terminated_length": 554.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.5163977742195129, + "reward": 6.5, + "reward_std": 0.5163977742195129, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02638406865298748, + "sampling/sampling_logp_difference/max": 0.8512144088745117, + "sampling/importance_sampling_ratio/min": 0.04890051856637001, + "sampling/importance_sampling_ratio/mean": 0.5873216986656189, + "sampling/importance_sampling_ratio/max": 2.0982444286346436, + "kl": 0.002002388624532614, + "entropy": 1.1377170644700527, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.427699581719935, + "epoch": 0.018, + "step": 9 + }, + { + "loss": -0.12387816607952118, + "grad_norm": 0.1985069215297699, + "learning_rate": 2.7000000000000002e-05, + "num_tokens": 98603.0, + "completions/mean_length": 500.1875, + "completions/min_length": 400.0, + "completions/max_length": 588.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 500.1875, + "completions/min_terminated_length": 400.0, + "completions/max_terminated_length": 588.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.025383003056049347, + "sampling/sampling_logp_difference/max": 0.3653905391693115, + "sampling/importance_sampling_ratio/min": 0.012905921787023544, + "sampling/importance_sampling_ratio/mean": 0.5639468431472778, + "sampling/importance_sampling_ratio/max": 1.983199954032898, + "kl": 0.0026735300780273974, + "entropy": 1.1230391450226307, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.99981931829825, + "epoch": 0.02, + "step": 10 + }, + { + "loss": -0.06913074851036072, + "grad_norm": 0.3695620596408844, + "learning_rate": 3e-05, + "num_tokens": 107734.0, + "completions/mean_length": 459.6875, + "completions/min_length": 379.0, + "completions/max_length": 526.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 459.6875, + "completions/min_terminated_length": 379.0, + "completions/max_terminated_length": 526.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026674197986721992, + "sampling/sampling_logp_difference/max": 0.3647327423095703, + "sampling/importance_sampling_ratio/min": 0.06302778422832489, + "sampling/importance_sampling_ratio/mean": 0.8144867420196533, + "sampling/importance_sampling_ratio/max": 2.000136375427246, + "kl": 0.00424640768324025, + "entropy": 1.213625729084015, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.46549107739702, + "epoch": 0.022, + "step": 11 + }, + { + "loss": 0.22097699344158173, + "grad_norm": 0.35457733273506165, + "learning_rate": 3e-05, + "num_tokens": 117199.0, + "completions/mean_length": 482.0625, + "completions/min_length": 428.0, + "completions/max_length": 555.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 482.0625, + "completions/min_terminated_length": 428.0, + "completions/max_terminated_length": 555.0, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 0.9979145526885986, + "reward": 5.9375, + "reward_std": 0.9979145526885986, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02726689912378788, + "sampling/sampling_logp_difference/max": 0.3921785354614258, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.649204432964325, + "sampling/importance_sampling_ratio/max": 1.5660414695739746, + "kl": 0.007200263120466843, + "entropy": 1.191277615725994, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.719243939965963, + "epoch": 0.024, + "step": 12 + }, + { + "loss": -0.07746122777462006, + "grad_norm": 0.30047014355659485, + "learning_rate": 3e-05, + "num_tokens": 126556.0, + "completions/mean_length": 464.3125, + "completions/min_length": 391.0, + "completions/max_length": 577.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 464.3125, + "completions/min_terminated_length": 391.0, + "completions/max_terminated_length": 577.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02639186754822731, + "sampling/sampling_logp_difference/max": 0.4946432113647461, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7561360597610474, + "sampling/importance_sampling_ratio/max": 2.6028401851654053, + "kl": 0.0058551667898427695, + "entropy": 1.24251464381814, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.08984712138772, + "epoch": 0.026, + "step": 13 + }, + { + "loss": 0.10855278372764587, + "grad_norm": 0.24105942249298096, + "learning_rate": 3e-05, + "num_tokens": 135417.0, + "completions/mean_length": 444.8125, + "completions/min_length": 389.0, + "completions/max_length": 511.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 444.8125, + "completions/min_terminated_length": 389.0, + "completions/max_terminated_length": 511.0, + "rewards/quality_reward/mean": 3.1875, + "rewards/quality_reward/std": 3.310966730117798, + "reward": 3.1875, + "reward_std": 3.310966730117798, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02838001400232315, + "sampling/sampling_logp_difference/max": 0.8390979766845703, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5903321504592896, + "sampling/importance_sampling_ratio/max": 2.541518211364746, + "kl": 0.012796793889719993, + "entropy": 1.1501125395298004, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.055794408079237, + "epoch": 0.028, + "step": 14 + }, + { + "loss": -0.007966680452227592, + "grad_norm": 0.19040776789188385, + "learning_rate": 3e-05, + "num_tokens": 144205.0, + "completions/mean_length": 442.25, + "completions/min_length": 386.0, + "completions/max_length": 497.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 442.25, + "completions/min_terminated_length": 386.0, + "completions/max_terminated_length": 497.0, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 0.8850612044334412, + "reward": 5.875, + "reward_std": 0.8850612044334412, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027210108935832977, + "sampling/sampling_logp_difference/max": 0.3982200622558594, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.41446638107299805, + "sampling/importance_sampling_ratio/max": 1.0754293203353882, + "kl": 0.010701361010433175, + "entropy": 1.1262825280427933, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.176074750721455, + "epoch": 0.03, + "step": 15 + }, + { + "loss": -0.21546132862567902, + "grad_norm": 0.23625652492046356, + "learning_rate": 3e-05, + "num_tokens": 153543.0, + "completions/mean_length": 478.125, + "completions/min_length": 433.0, + "completions/max_length": 531.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 478.125, + "completions/min_terminated_length": 433.0, + "completions/max_terminated_length": 531.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.730296790599823, + "reward": 6.5, + "reward_std": 0.730296790599823, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028833162039518356, + "sampling/sampling_logp_difference/max": 0.5095968246459961, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.49076417088508606, + "sampling/importance_sampling_ratio/max": 2.0074336528778076, + "kl": 0.0213887135614641, + "entropy": 1.2985924184322357, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.848205763846636, + "epoch": 0.032, + "step": 16 + }, + { + "loss": -0.0828079879283905, + "grad_norm": 0.17765119671821594, + "learning_rate": 3e-05, + "num_tokens": 163043.0, + "completions/mean_length": 469.25, + "completions/min_length": 423.0, + "completions/max_length": 526.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 469.25, + "completions/min_terminated_length": 423.0, + "completions/max_terminated_length": 526.0, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030804751440882683, + "sampling/sampling_logp_difference/max": 0.591062068939209, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5021570324897766, + "sampling/importance_sampling_ratio/max": 1.5988941192626953, + "kl": 0.02128879475640133, + "entropy": 1.3148910626769066, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 28.313011147081852, + "epoch": 0.034, + "step": 17 + }, + { + "loss": -0.01184748113155365, + "grad_norm": 0.37244123220443726, + "learning_rate": 3e-05, + "num_tokens": 172379.0, + "completions/mean_length": 447.0, + "completions/min_length": 388.0, + "completions/max_length": 501.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 447.0, + "completions/min_terminated_length": 388.0, + "completions/max_terminated_length": 501.0, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.8944272398948669, + "reward": 6.0, + "reward_std": 0.8944272398948669, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03126702085137367, + "sampling/sampling_logp_difference/max": 0.5762512683868408, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.9581233263015747, + "sampling/importance_sampling_ratio/max": 2.675238847732544, + "kl": 0.019650122558232397, + "entropy": 1.390664003789425, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.991567107848823, + "epoch": 0.036, + "step": 18 + }, + { + "loss": 0.040024783462285995, + "grad_norm": 0.23734751343727112, + "learning_rate": 3e-05, + "num_tokens": 181239.0, + "completions/mean_length": 447.75, + "completions/min_length": 383.0, + "completions/max_length": 498.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 447.75, + "completions/min_terminated_length": 383.0, + "completions/max_terminated_length": 498.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030378391966223717, + "sampling/sampling_logp_difference/max": 0.4722297191619873, + "sampling/importance_sampling_ratio/min": 0.1203346848487854, + "sampling/importance_sampling_ratio/mean": 0.6408629417419434, + "sampling/importance_sampling_ratio/max": 2.095722198486328, + "kl": 0.022738213243428618, + "entropy": 1.3287223279476166, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.615950418636203, + "epoch": 0.038, + "step": 19 + }, + { + "loss": 0.1691148728132248, + "grad_norm": 0.34637194871902466, + "learning_rate": 3e-05, + "num_tokens": 190068.0, + "completions/mean_length": 452.3125, + "completions/min_length": 376.0, + "completions/max_length": 659.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 452.3125, + "completions/min_terminated_length": 376.0, + "completions/max_terminated_length": 659.0, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.3102163076400757, + "reward": 6.125, + "reward_std": 1.3102163076400757, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030463142320513725, + "sampling/sampling_logp_difference/max": 2.6531033515930176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7767290472984314, + "sampling/importance_sampling_ratio/max": 2.6566128730773926, + "kl": 0.015380491153337061, + "entropy": 1.1001618690788746, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.741292077116668, + "epoch": 0.04, + "step": 20 + }, + { + "loss": -0.13425321877002716, + "grad_norm": 0.2514922618865967, + "learning_rate": 3e-05, + "num_tokens": 198941.0, + "completions/mean_length": 457.5625, + "completions/min_length": 390.0, + "completions/max_length": 589.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 457.5625, + "completions/min_terminated_length": 390.0, + "completions/max_terminated_length": 589.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0315740592777729, + "sampling/sampling_logp_difference/max": 0.4818992614746094, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.601022481918335, + "sampling/importance_sampling_ratio/max": 1.5612297058105469, + "kl": 0.018277494702488184, + "entropy": 1.3708399310708046, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.24192419089377, + "epoch": 0.042, + "step": 21 + }, + { + "loss": 0.010622119531035423, + "grad_norm": 0.2268020063638687, + "learning_rate": 3e-05, + "num_tokens": 207300.0, + "completions/mean_length": 421.4375, + "completions/min_length": 360.0, + "completions/max_length": 508.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 421.4375, + "completions/min_terminated_length": 360.0, + "completions/max_terminated_length": 508.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02804401144385338, + "sampling/sampling_logp_difference/max": 0.5335149765014648, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.46136727929115295, + "sampling/importance_sampling_ratio/max": 1.7986358404159546, + "kl": 0.017973962530959398, + "entropy": 1.136269599199295, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.37282825494185, + "epoch": 0.044, + "step": 22 + }, + { + "loss": -0.24404363334178925, + "grad_norm": 0.33994877338409424, + "learning_rate": 3e-05, + "num_tokens": 216343.0, + "completions/mean_length": 452.6875, + "completions/min_length": 381.0, + "completions/max_length": 503.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 452.6875, + "completions/min_terminated_length": 381.0, + "completions/max_terminated_length": 503.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02918298915028572, + "sampling/sampling_logp_difference/max": 0.6328549385070801, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.747193455696106, + "sampling/importance_sampling_ratio/max": 2.452249526977539, + "kl": 0.021804221265483648, + "entropy": 1.3299130946397781, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.356728344224393, + "epoch": 0.046, + "step": 23 + }, + { + "loss": -0.09123071283102036, + "grad_norm": 0.3076202869415283, + "learning_rate": 3e-05, + "num_tokens": 225550.0, + "completions/mean_length": 462.4375, + "completions/min_length": 410.0, + "completions/max_length": 524.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 462.4375, + "completions/min_terminated_length": 410.0, + "completions/max_terminated_length": 524.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028946854174137115, + "sampling/sampling_logp_difference/max": 0.36547183990478516, + "sampling/importance_sampling_ratio/min": 0.04495502635836601, + "sampling/importance_sampling_ratio/mean": 0.6625345945358276, + "sampling/importance_sampling_ratio/max": 1.5845099687576294, + "kl": 0.020299295720178634, + "entropy": 1.379701942205429, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.95462113339454, + "epoch": 0.048, + "step": 24 + }, + { + "loss": -0.1412944793701172, + "grad_norm": 0.22180576622486115, + "learning_rate": 3e-05, + "num_tokens": 235005.0, + "completions/mean_length": 482.4375, + "completions/min_length": 382.0, + "completions/max_length": 618.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 482.4375, + "completions/min_terminated_length": 382.0, + "completions/max_terminated_length": 618.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02898716926574707, + "sampling/sampling_logp_difference/max": 0.4363563060760498, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7712795734405518, + "sampling/importance_sampling_ratio/max": 2.6407876014709473, + "kl": 0.018309170845896006, + "entropy": 1.2471638694405556, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.46686205593869, + "epoch": 0.05, + "step": 25 + }, + { + "loss": 0.10839397460222244, + "grad_norm": 0.37535253167152405, + "learning_rate": 3e-05, + "num_tokens": 243953.0, + "completions/mean_length": 455.75, + "completions/min_length": 364.0, + "completions/max_length": 555.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 455.75, + "completions/min_terminated_length": 364.0, + "completions/max_terminated_length": 555.0, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.0645813941955566, + "reward": 6.25, + "reward_std": 1.0645813941955566, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027610119432210922, + "sampling/sampling_logp_difference/max": 0.4523625373840332, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5800145864486694, + "sampling/importance_sampling_ratio/max": 2.0567266941070557, + "kl": 0.020504546992015094, + "entropy": 1.28530602902174, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.121026155073196, + "epoch": 0.052, + "step": 26 + }, + { + "loss": 0.04823978245258331, + "grad_norm": 0.2340388149023056, + "learning_rate": 3e-05, + "num_tokens": 253237.0, + "completions/mean_length": 462.75, + "completions/min_length": 400.0, + "completions/max_length": 514.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 462.75, + "completions/min_terminated_length": 400.0, + "completions/max_terminated_length": 514.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028367744758725166, + "sampling/sampling_logp_difference/max": 0.35022830963134766, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.40753045678138733, + "sampling/importance_sampling_ratio/max": 1.1560932397842407, + "kl": 0.020236384589225054, + "entropy": 1.2253629937767982, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.10967448912561, + "epoch": 0.054, + "step": 27 + }, + { + "loss": 0.0933581069111824, + "grad_norm": 0.5531017184257507, + "learning_rate": 3e-05, + "num_tokens": 262454.0, + "completions/mean_length": 482.0625, + "completions/min_length": 438.0, + "completions/max_length": 538.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 482.0625, + "completions/min_terminated_length": 438.0, + "completions/max_terminated_length": 538.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028883326798677444, + "sampling/sampling_logp_difference/max": 0.343827486038208, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7730721831321716, + "sampling/importance_sampling_ratio/max": 2.5711586475372314, + "kl": 0.01706669357372448, + "entropy": 1.3779077678918839, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 38.59647103771567, + "epoch": 0.056, + "step": 28 + }, + { + "loss": 0.010216176509857178, + "grad_norm": 0.33771538734436035, + "learning_rate": 3e-05, + "num_tokens": 271918.0, + "completions/mean_length": 477.0, + "completions/min_length": 410.0, + "completions/max_length": 563.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 477.0, + "completions/min_terminated_length": 410.0, + "completions/max_terminated_length": 563.0, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.2909945249557495, + "reward": 5.75, + "reward_std": 1.2909945249557495, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029029540717601776, + "sampling/sampling_logp_difference/max": 0.3258817195892334, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 1.0250636339187622, + "sampling/importance_sampling_ratio/max": 2.4999797344207764, + "kl": 0.02141271048458293, + "entropy": 1.3791070505976677, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.610272648278624, + "epoch": 0.058, + "step": 29 + }, + { + "loss": -0.014814459718763828, + "grad_norm": 0.21955685317516327, + "learning_rate": 3e-05, + "num_tokens": 281305.0, + "completions/mean_length": 471.1875, + "completions/min_length": 381.0, + "completions/max_length": 575.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 471.1875, + "completions/min_terminated_length": 381.0, + "completions/max_terminated_length": 575.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03177585452795029, + "sampling/sampling_logp_difference/max": 0.6354451179504395, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.866391658782959, + "sampling/importance_sampling_ratio/max": 2.802098274230957, + "kl": 0.019562674744520336, + "entropy": 1.4652926102280617, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.961607525125146, + "epoch": 0.06, + "step": 30 + }, + { + "loss": -0.08307373523712158, + "grad_norm": 0.12176825106143951, + "learning_rate": 3e-05, + "num_tokens": 291409.0, + "completions/mean_length": 490.5, + "completions/min_length": 411.0, + "completions/max_length": 602.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 490.5, + "completions/min_terminated_length": 411.0, + "completions/max_terminated_length": 602.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02871524728834629, + "sampling/sampling_logp_difference/max": 0.4253416061401367, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6937527656555176, + "sampling/importance_sampling_ratio/max": 1.610858678817749, + "kl": 0.026934220048133284, + "entropy": 1.1957123205065727, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.012207658961415, + "epoch": 0.062, + "step": 31 + }, + { + "loss": 0.0017175457905977964, + "grad_norm": 0.4979296624660492, + "learning_rate": 3e-05, + "num_tokens": 300649.0, + "completions/mean_length": 462.0, + "completions/min_length": 381.0, + "completions/max_length": 508.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 462.0, + "completions/min_terminated_length": 381.0, + "completions/max_terminated_length": 508.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "reward": 6.875, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029107660055160522, + "sampling/sampling_logp_difference/max": 0.5221483707427979, + "sampling/importance_sampling_ratio/min": 0.07447884231805801, + "sampling/importance_sampling_ratio/mean": 0.7824680209159851, + "sampling/importance_sampling_ratio/max": 1.8527640104293823, + "kl": 0.03539742121938616, + "entropy": 1.3018206283450127, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.78309725271538, + "epoch": 0.064, + "step": 32 + }, + { + "loss": 0.05625719577074051, + "grad_norm": 0.22016967833042145, + "learning_rate": 3e-05, + "num_tokens": 309889.0, + "completions/mean_length": 474.0, + "completions/min_length": 382.0, + "completions/max_length": 584.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 474.0, + "completions/min_terminated_length": 382.0, + "completions/max_terminated_length": 584.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027828343212604523, + "sampling/sampling_logp_difference/max": 0.38585615158081055, + "sampling/importance_sampling_ratio/min": 0.18006731569766998, + "sampling/importance_sampling_ratio/mean": 0.7832435369491577, + "sampling/importance_sampling_ratio/max": 2.9875097274780273, + "kl": 0.029592803912237287, + "entropy": 1.321175642311573, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 44.51360382232815, + "epoch": 0.066, + "step": 33 + }, + { + "loss": -0.037432070821523666, + "grad_norm": 0.2873040437698364, + "learning_rate": 3e-05, + "num_tokens": 318904.0, + "completions/mean_length": 459.9375, + "completions/min_length": 410.0, + "completions/max_length": 510.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 459.9375, + "completions/min_terminated_length": 410.0, + "completions/max_terminated_length": 510.0, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.1086779832839966, + "reward": 6.1875, + "reward_std": 1.1086779832839966, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029470665380358696, + "sampling/sampling_logp_difference/max": 0.41987133026123047, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5744763016700745, + "sampling/importance_sampling_ratio/max": 2.6647069454193115, + "kl": 0.02840927499346435, + "entropy": 1.2222414836287498, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 133.03877451224253, + "epoch": 0.068, + "step": 34 + }, + { + "loss": -0.20422951877117157, + "grad_norm": 0.2490653246641159, + "learning_rate": 3e-05, + "num_tokens": 328011.0, + "completions/mean_length": 462.1875, + "completions/min_length": 413.0, + "completions/max_length": 522.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 462.1875, + "completions/min_terminated_length": 413.0, + "completions/max_terminated_length": 522.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "reward": 6.9375, + "reward_std": 1.1814539432525635, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026844775304198265, + "sampling/sampling_logp_difference/max": 0.36430132389068604, + "sampling/importance_sampling_ratio/min": 0.020566223189234734, + "sampling/importance_sampling_ratio/mean": 0.7954420447349548, + "sampling/importance_sampling_ratio/max": 2.510730266571045, + "kl": 0.025841041060630232, + "entropy": 1.1665974482893944, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.5843787365593, + "epoch": 0.07, + "step": 35 + }, + { + "loss": -0.11188814043998718, + "grad_norm": 0.15268851816654205, + "learning_rate": 3e-05, + "num_tokens": 337731.0, + "completions/mean_length": 492.0, + "completions/min_length": 429.0, + "completions/max_length": 549.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 492.0, + "completions/min_terminated_length": 429.0, + "completions/max_terminated_length": 549.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.7274384498596191, + "reward": 6.5625, + "reward_std": 0.7274384498596191, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028961554169654846, + "sampling/sampling_logp_difference/max": 0.4023854732513428, + "sampling/importance_sampling_ratio/min": 0.10853960365056992, + "sampling/importance_sampling_ratio/mean": 0.614537239074707, + "sampling/importance_sampling_ratio/max": 1.6270908117294312, + "kl": 0.023660800594370812, + "entropy": 1.3467887043952942, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.843947287183255, + "epoch": 0.072, + "step": 36 + }, + { + "loss": 0.37223517894744873, + "grad_norm": 0.42548227310180664, + "learning_rate": 3e-05, + "num_tokens": 346815.0, + "completions/mean_length": 460.75, + "completions/min_length": 399.0, + "completions/max_length": 513.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 460.75, + "completions/min_terminated_length": 399.0, + "completions/max_terminated_length": 513.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028780322521924973, + "sampling/sampling_logp_difference/max": 0.4037543535232544, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7942180633544922, + "sampling/importance_sampling_ratio/max": 2.6624510288238525, + "kl": 0.022181478852871805, + "entropy": 1.2476315572857857, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 42.04662919091061, + "epoch": 0.074, + "step": 37 + }, + { + "loss": -0.09029137343168259, + "grad_norm": 0.16980381309986115, + "learning_rate": 3e-05, + "num_tokens": 355711.0, + "completions/mean_length": 452.0, + "completions/min_length": 363.0, + "completions/max_length": 611.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 452.0, + "completions/min_terminated_length": 363.0, + "completions/max_terminated_length": 611.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026646045967936516, + "sampling/sampling_logp_difference/max": 0.30433225631713867, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.721072793006897, + "sampling/importance_sampling_ratio/max": 2.3201518058776855, + "kl": 0.017483733594417572, + "entropy": 1.1745503433048725, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 38.63075139513239, + "epoch": 0.076, + "step": 38 + }, + { + "loss": -0.03154226019978523, + "grad_norm": 0.21305795013904572, + "learning_rate": 3e-05, + "num_tokens": 364860.0, + "completions/mean_length": 466.8125, + "completions/min_length": 429.0, + "completions/max_length": 521.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 466.8125, + "completions/min_terminated_length": 429.0, + "completions/max_terminated_length": 521.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028257746249437332, + "sampling/sampling_logp_difference/max": 0.37443917989730835, + "sampling/importance_sampling_ratio/min": 0.11245065927505493, + "sampling/importance_sampling_ratio/mean": 0.674609363079071, + "sampling/importance_sampling_ratio/max": 2.0297553539276123, + "kl": 0.021121050289366394, + "entropy": 1.2664988860487938, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.028073193039745, + "epoch": 0.078, + "step": 39 + }, + { + "loss": -0.035816628485918045, + "grad_norm": 0.12235487997531891, + "learning_rate": 3e-05, + "num_tokens": 374607.0, + "completions/mean_length": 485.1875, + "completions/min_length": 430.0, + "completions/max_length": 557.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 485.1875, + "completions/min_terminated_length": 430.0, + "completions/max_terminated_length": 557.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029600802809000015, + "sampling/sampling_logp_difference/max": 0.36985254287719727, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5344723463058472, + "sampling/importance_sampling_ratio/max": 2.242042303085327, + "kl": 0.018535695446189493, + "entropy": 1.3458357080817223, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.446288945619017, + "epoch": 0.08, + "step": 40 + }, + { + "loss": 0.0011727213859558105, + "grad_norm": 0.15642686188220978, + "learning_rate": 3e-05, + "num_tokens": 384317.0, + "completions/mean_length": 498.875, + "completions/min_length": 429.0, + "completions/max_length": 594.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 498.875, + "completions/min_terminated_length": 429.0, + "completions/max_terminated_length": 594.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8850612044334412, + "reward": 6.375, + "reward_std": 0.8850612044334412, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03018251620233059, + "sampling/sampling_logp_difference/max": 0.6487679481506348, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.3696203827857971, + "sampling/importance_sampling_ratio/max": 1.5691092014312744, + "kl": 0.013967045990284532, + "entropy": 1.3402792811393738, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.15720977121964, + "epoch": 0.082, + "step": 41 + }, + { + "loss": 0.10363321751356125, + "grad_norm": 0.3236794173717499, + "learning_rate": 3e-05, + "num_tokens": 393934.0, + "completions/mean_length": 482.0625, + "completions/min_length": 440.0, + "completions/max_length": 568.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 482.0625, + "completions/min_terminated_length": 440.0, + "completions/max_terminated_length": 568.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02936164103448391, + "sampling/sampling_logp_difference/max": 0.7769032716751099, + "sampling/importance_sampling_ratio/min": 0.07682990282773972, + "sampling/importance_sampling_ratio/mean": 0.6415404677391052, + "sampling/importance_sampling_ratio/max": 2.0568668842315674, + "kl": 0.01707366103073582, + "entropy": 1.3317564576864243, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.066733688581735, + "epoch": 0.084, + "step": 42 + }, + { + "loss": -0.027566466480493546, + "grad_norm": 0.1771102100610733, + "learning_rate": 3e-05, + "num_tokens": 403893.0, + "completions/mean_length": 510.9375, + "completions/min_length": 466.0, + "completions/max_length": 606.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 510.9375, + "completions/min_terminated_length": 466.0, + "completions/max_terminated_length": 606.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.209338665008545, + "reward": 6.4375, + "reward_std": 1.209338665008545, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028799494728446007, + "sampling/sampling_logp_difference/max": 0.36153650283813477, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7769261598587036, + "sampling/importance_sampling_ratio/max": 2.3430161476135254, + "kl": 0.01784718461567536, + "entropy": 1.2099780030548573, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.90863296529278, + "epoch": 0.086, + "step": 43 + }, + { + "loss": -0.04157561436295509, + "grad_norm": 0.3432070314884186, + "learning_rate": 3e-05, + "num_tokens": 413312.0, + "completions/mean_length": 494.1875, + "completions/min_length": 432.0, + "completions/max_length": 551.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 494.1875, + "completions/min_terminated_length": 432.0, + "completions/max_terminated_length": 551.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028122060000896454, + "sampling/sampling_logp_difference/max": 0.31444358825683594, + "sampling/importance_sampling_ratio/min": 0.07410597801208496, + "sampling/importance_sampling_ratio/mean": 0.8989821672439575, + "sampling/importance_sampling_ratio/max": 2.726816177368164, + "kl": 0.014529847539961338, + "entropy": 1.2924985438585281, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.7880685236305, + "epoch": 0.088, + "step": 44 + }, + { + "loss": 0.029176680371165276, + "grad_norm": 0.36233776807785034, + "learning_rate": 3e-05, + "num_tokens": 423624.0, + "completions/mean_length": 543.5, + "completions/min_length": 484.0, + "completions/max_length": 637.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 543.5, + "completions/min_terminated_length": 484.0, + "completions/max_terminated_length": 637.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02717999368906021, + "sampling/sampling_logp_difference/max": 0.31381869316101074, + "sampling/importance_sampling_ratio/min": 0.09822077304124832, + "sampling/importance_sampling_ratio/mean": 0.6537867188453674, + "sampling/importance_sampling_ratio/max": 1.677195429801941, + "kl": 0.015796773659531027, + "entropy": 1.206408604979515, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.35960763087496, + "epoch": 0.09, + "step": 45 + }, + { + "loss": 0.18850615620613098, + "grad_norm": 0.20219561457633972, + "learning_rate": 3e-05, + "num_tokens": 433817.0, + "completions/mean_length": 534.5625, + "completions/min_length": 453.0, + "completions/max_length": 643.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 534.5625, + "completions/min_terminated_length": 453.0, + "completions/max_terminated_length": 643.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02723248302936554, + "sampling/sampling_logp_difference/max": 0.35237741470336914, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.697495698928833, + "sampling/importance_sampling_ratio/max": 2.203894853591919, + "kl": 0.014481160265859216, + "entropy": 1.2577891275286674, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.56032704282552, + "epoch": 0.092, + "step": 46 + }, + { + "loss": 0.06984467804431915, + "grad_norm": 0.22262753546237946, + "learning_rate": 3e-05, + "num_tokens": 444045.0, + "completions/mean_length": 538.25, + "completions/min_length": 469.0, + "completions/max_length": 642.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 538.25, + "completions/min_terminated_length": 469.0, + "completions/max_terminated_length": 642.0, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.0626225471496582, + "reward": 6.0625, + "reward_std": 1.0626225471496582, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028838323429226875, + "sampling/sampling_logp_difference/max": 0.4496123790740967, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6712950468063354, + "sampling/importance_sampling_ratio/max": 2.2261502742767334, + "kl": 0.012554377142805606, + "entropy": 1.3271892964839935, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.226045075803995, + "epoch": 0.094, + "step": 47 + }, + { + "loss": 0.09024985134601593, + "grad_norm": 0.3244606554508209, + "learning_rate": 3e-05, + "num_tokens": 453945.0, + "completions/mean_length": 514.25, + "completions/min_length": 424.0, + "completions/max_length": 628.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 514.25, + "completions/min_terminated_length": 424.0, + "completions/max_terminated_length": 628.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026116060093045235, + "sampling/sampling_logp_difference/max": 0.3650559186935425, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7607913017272949, + "sampling/importance_sampling_ratio/max": 2.6745388507843018, + "kl": 0.0157591889728792, + "entropy": 1.1054812744259834, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.565261672716588, + "epoch": 0.096, + "step": 48 + }, + { + "loss": -0.2898017466068268, + "grad_norm": 0.4033137857913971, + "learning_rate": 3e-05, + "num_tokens": 463576.0, + "completions/mean_length": 502.4375, + "completions/min_length": 439.0, + "completions/max_length": 585.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 502.4375, + "completions/min_terminated_length": 439.0, + "completions/max_terminated_length": 585.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.2632629871368408, + "reward": 6.4375, + "reward_std": 1.2632629871368408, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028341906145215034, + "sampling/sampling_logp_difference/max": 0.3675417900085449, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7124314308166504, + "sampling/importance_sampling_ratio/max": 2.6974196434020996, + "kl": 0.015613102470524609, + "entropy": 1.2281213253736496, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 29.838082558009773, + "epoch": 0.098, + "step": 49 + }, + { + "loss": -0.19065965712070465, + "grad_norm": 0.23144562542438507, + "learning_rate": 3e-05, + "num_tokens": 473915.0, + "completions/mean_length": 530.1875, + "completions/min_length": 458.0, + "completions/max_length": 573.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 530.1875, + "completions/min_terminated_length": 458.0, + "completions/max_terminated_length": 573.0, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.1954776048660278, + "reward": 6.3125, + "reward_std": 1.1954776048660278, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02853373810648918, + "sampling/sampling_logp_difference/max": 0.2777421474456787, + "sampling/importance_sampling_ratio/min": 0.1628064066171646, + "sampling/importance_sampling_ratio/mean": 0.8162041902542114, + "sampling/importance_sampling_ratio/max": 2.54063081741333, + "kl": 0.01374751579714939, + "entropy": 1.1955150067806244, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.047010839451104, + "epoch": 0.1, + "step": 50 + }, + { + "loss": -0.1761355698108673, + "grad_norm": 0.4241364896297455, + "learning_rate": 3e-05, + "num_tokens": 484577.0, + "completions/mean_length": 549.875, + "completions/min_length": 463.0, + "completions/max_length": 619.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 549.875, + "completions/min_terminated_length": 463.0, + "completions/max_terminated_length": 619.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 1.0935417413711548, + "reward": 6.5625, + "reward_std": 1.0935417413711548, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028025619685649872, + "sampling/sampling_logp_difference/max": 0.5649824142456055, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6905896663665771, + "sampling/importance_sampling_ratio/max": 2.929507255554199, + "kl": 0.013923745544161648, + "entropy": 1.1870752647519112, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 44.488836522214115, + "epoch": 0.102, + "step": 51 + }, + { + "loss": -0.06503897905349731, + "grad_norm": 0.32851356267929077, + "learning_rate": 3e-05, + "num_tokens": 495015.0, + "completions/mean_length": 552.375, + "completions/min_length": 484.0, + "completions/max_length": 625.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 552.375, + "completions/min_terminated_length": 484.0, + "completions/max_terminated_length": 625.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "reward": 7.0625, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02687419019639492, + "sampling/sampling_logp_difference/max": 0.3454720973968506, + "sampling/importance_sampling_ratio/min": 0.040177375078201294, + "sampling/importance_sampling_ratio/mean": 0.9612224102020264, + "sampling/importance_sampling_ratio/max": 2.9652340412139893, + "kl": 0.01297539210645482, + "entropy": 1.098166175186634, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.21497478755191, + "epoch": 0.104, + "step": 52 + }, + { + "loss": -0.18975484371185303, + "grad_norm": 0.21921825408935547, + "learning_rate": 3e-05, + "num_tokens": 505909.0, + "completions/mean_length": 578.375, + "completions/min_length": 499.0, + "completions/max_length": 670.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 578.375, + "completions/min_terminated_length": 499.0, + "completions/max_terminated_length": 670.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028095029294490814, + "sampling/sampling_logp_difference/max": 0.39138758182525635, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5587533116340637, + "sampling/importance_sampling_ratio/max": 2.0094237327575684, + "kl": 0.017025968758389354, + "entropy": 1.2311968132853508, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 31.140278964303434, + "epoch": 0.106, + "step": 53 + }, + { + "loss": 0.14847250282764435, + "grad_norm": 0.27410373091697693, + "learning_rate": 3e-05, + "num_tokens": 515073.0, + "completions/mean_length": 473.25, + "completions/min_length": 308.0, + "completions/max_length": 661.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 473.25, + "completions/min_terminated_length": 308.0, + "completions/max_terminated_length": 661.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02810005284845829, + "sampling/sampling_logp_difference/max": 0.5433444976806641, + "sampling/importance_sampling_ratio/min": 0.09779425710439682, + "sampling/importance_sampling_ratio/mean": 0.7429271936416626, + "sampling/importance_sampling_ratio/max": 2.875750780105591, + "kl": 0.018500705540645868, + "entropy": 1.2682743221521378, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.365382733754814, + "epoch": 0.108, + "step": 54 + }, + { + "loss": 0.05967015400528908, + "grad_norm": 0.29323700070381165, + "learning_rate": 3e-05, + "num_tokens": 525377.0, + "completions/mean_length": 549.0, + "completions/min_length": 439.0, + "completions/max_length": 615.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 549.0, + "completions/min_terminated_length": 439.0, + "completions/max_terminated_length": 615.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02906947024166584, + "sampling/sampling_logp_difference/max": 0.39328718185424805, + "sampling/importance_sampling_ratio/min": 0.08072065562009811, + "sampling/importance_sampling_ratio/mean": 0.6094669103622437, + "sampling/importance_sampling_ratio/max": 1.4832638502120972, + "kl": 0.016703857632819563, + "entropy": 1.3073157891631126, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.47015379369259, + "epoch": 0.11, + "step": 55 + }, + { + "loss": -0.04828515648841858, + "grad_norm": 0.200644388794899, + "learning_rate": 3e-05, + "num_tokens": 536163.0, + "completions/mean_length": 576.625, + "completions/min_length": 500.0, + "completions/max_length": 659.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 576.625, + "completions/min_terminated_length": 500.0, + "completions/max_terminated_length": 659.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02962428703904152, + "sampling/sampling_logp_difference/max": 0.3664684295654297, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5633801221847534, + "sampling/importance_sampling_ratio/max": 2.3941831588745117, + "kl": 0.018819268501829356, + "entropy": 1.2820357605814934, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.172011949121952, + "epoch": 0.112, + "step": 56 + }, + { + "loss": -0.12029920518398285, + "grad_norm": 0.19633841514587402, + "learning_rate": 3e-05, + "num_tokens": 548143.0, + "completions/mean_length": 640.25, + "completions/min_length": 556.0, + "completions/max_length": 775.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 640.25, + "completions/min_terminated_length": 556.0, + "completions/max_terminated_length": 775.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028747636824846268, + "sampling/sampling_logp_difference/max": 0.33841991424560547, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6956661343574524, + "sampling/importance_sampling_ratio/max": 2.3877830505371094, + "kl": 0.02060725452611223, + "entropy": 1.4191219061613083, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.892430102452636, + "epoch": 0.114, + "step": 57 + }, + { + "loss": -0.02738652005791664, + "grad_norm": 0.24178451299667358, + "learning_rate": 3e-05, + "num_tokens": 558710.0, + "completions/mean_length": 560.4375, + "completions/min_length": 513.0, + "completions/max_length": 646.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 560.4375, + "completions/min_terminated_length": 513.0, + "completions/max_terminated_length": 646.0, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.1814539432525635, + "reward": 6.0625, + "reward_std": 1.1814539432525635, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029777564108371735, + "sampling/sampling_logp_difference/max": 0.3029825687408447, + "sampling/importance_sampling_ratio/min": 0.05691095441579819, + "sampling/importance_sampling_ratio/mean": 0.4813012480735779, + "sampling/importance_sampling_ratio/max": 1.4787031412124634, + "kl": 0.023009511292912066, + "entropy": 1.3337246850132942, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.878377372398973, + "epoch": 0.116, + "step": 58 + }, + { + "loss": 0.04775794595479965, + "grad_norm": 0.1262614130973816, + "learning_rate": 3e-05, + "num_tokens": 569826.0, + "completions/mean_length": 595.25, + "completions/min_length": 498.0, + "completions/max_length": 697.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 595.25, + "completions/min_terminated_length": 498.0, + "completions/max_terminated_length": 697.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 1.0307763814926147, + "reward": 6.4375, + "reward_std": 1.0307763814926147, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028494788333773613, + "sampling/sampling_logp_difference/max": 0.42272377014160156, + "sampling/importance_sampling_ratio/min": 0.026369251310825348, + "sampling/importance_sampling_ratio/mean": 0.5139275193214417, + "sampling/importance_sampling_ratio/max": 2.1841602325439453, + "kl": 0.022026430582627654, + "entropy": 1.3107040077447891, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.42572767334059, + "epoch": 0.118, + "step": 59 + }, + { + "loss": 0.2925710678100586, + "grad_norm": 0.588756799697876, + "learning_rate": 3e-05, + "num_tokens": 580229.0, + "completions/mean_length": 549.1875, + "completions/min_length": 489.0, + "completions/max_length": 630.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 549.1875, + "completions/min_terminated_length": 489.0, + "completions/max_terminated_length": 630.0, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 0.9979145526885986, + "reward": 6.0625, + "reward_std": 0.9979145526885986, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02885228767991066, + "sampling/sampling_logp_difference/max": 0.4628920555114746, + "sampling/importance_sampling_ratio/min": 0.1096419170498848, + "sampling/importance_sampling_ratio/mean": 1.1227651834487915, + "sampling/importance_sampling_ratio/max": 2.8465583324432373, + "kl": 0.025482238037511706, + "entropy": 1.1738965958356857, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.57787229306996, + "epoch": 0.12, + "step": 60 + }, + { + "loss": 0.12304374575614929, + "grad_norm": 0.2779391407966614, + "learning_rate": 3e-05, + "num_tokens": 591178.0, + "completions/mean_length": 577.3125, + "completions/min_length": 495.0, + "completions/max_length": 692.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 577.3125, + "completions/min_terminated_length": 495.0, + "completions/max_terminated_length": 692.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8139410614967346, + "reward": 6.5625, + "reward_std": 0.8139410614967346, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027112768962979317, + "sampling/sampling_logp_difference/max": 0.47726941108703613, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.650765061378479, + "sampling/importance_sampling_ratio/max": 2.1122686862945557, + "kl": 0.02629381464794278, + "entropy": 1.1913195550441742, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.71764762001112, + "epoch": 0.122, + "step": 61 + }, + { + "loss": 0.17419062554836273, + "grad_norm": 0.1645125448703766, + "learning_rate": 3e-05, + "num_tokens": 603055.0, + "completions/mean_length": 628.3125, + "completions/min_length": 527.0, + "completions/max_length": 722.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 628.3125, + "completions/min_terminated_length": 527.0, + "completions/max_terminated_length": 722.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "reward": 6.625, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028948483988642693, + "sampling/sampling_logp_difference/max": 0.4525270462036133, + "sampling/importance_sampling_ratio/min": 0.03490918502211571, + "sampling/importance_sampling_ratio/mean": 0.5809424519538879, + "sampling/importance_sampling_ratio/max": 2.1662254333496094, + "kl": 0.025764177553355694, + "entropy": 1.3224291801452637, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 35.74759274255484, + "epoch": 0.124, + "step": 62 + }, + { + "loss": -0.019145065918564796, + "grad_norm": 0.20707836747169495, + "learning_rate": 3e-05, + "num_tokens": 614341.0, + "completions/mean_length": 597.375, + "completions/min_length": 478.0, + "completions/max_length": 736.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 597.375, + "completions/min_terminated_length": 478.0, + "completions/max_terminated_length": 736.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "reward": 6.9375, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028000790625810623, + "sampling/sampling_logp_difference/max": 0.42907285690307617, + "sampling/importance_sampling_ratio/min": 0.1595209240913391, + "sampling/importance_sampling_ratio/mean": 0.7066210508346558, + "sampling/importance_sampling_ratio/max": 2.0268709659576416, + "kl": 0.026473846402950585, + "entropy": 1.1552416533231735, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 39.37250621803105, + "epoch": 0.126, + "step": 63 + }, + { + "loss": 0.037037670612335205, + "grad_norm": 0.21552008390426636, + "learning_rate": 3e-05, + "num_tokens": 625165.0, + "completions/mean_length": 568.5, + "completions/min_length": 509.0, + "completions/max_length": 631.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 568.5, + "completions/min_terminated_length": 509.0, + "completions/max_terminated_length": 631.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "reward": 7.0625, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02858484350144863, + "sampling/sampling_logp_difference/max": 0.374176025390625, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6408567428588867, + "sampling/importance_sampling_ratio/max": 2.1896748542785645, + "kl": 0.029041914734989405, + "entropy": 1.2810933291912079, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.69576471252367, + "epoch": 0.128, + "step": 64 + }, + { + "loss": 0.0731797143816948, + "grad_norm": 0.2287176102399826, + "learning_rate": 3e-05, + "num_tokens": 636633.0, + "completions/mean_length": 622.75, + "completions/min_length": 545.0, + "completions/max_length": 699.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 622.75, + "completions/min_terminated_length": 545.0, + "completions/max_terminated_length": 699.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.9574271440505981, + "reward": 6.375, + "reward_std": 0.9574271440505981, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02755960263311863, + "sampling/sampling_logp_difference/max": 0.4503474235534668, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5977773666381836, + "sampling/importance_sampling_ratio/max": 1.4288272857666016, + "kl": 0.023987084976397455, + "entropy": 1.32467532902956, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.502957582939416, + "epoch": 0.13, + "step": 65 + }, + { + "loss": 0.10324293375015259, + "grad_norm": 0.27425676584243774, + "learning_rate": 3e-05, + "num_tokens": 648597.0, + "completions/mean_length": 639.25, + "completions/min_length": 554.0, + "completions/max_length": 767.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 639.25, + "completions/min_terminated_length": 554.0, + "completions/max_terminated_length": 767.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "reward": 7.0, + "reward_std": 0.632455587387085, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029852069914340973, + "sampling/sampling_logp_difference/max": 0.4819211959838867, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6433250904083252, + "sampling/importance_sampling_ratio/max": 2.971261501312256, + "kl": 0.028104660217650235, + "entropy": 1.400998167693615, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.79405551031232, + "epoch": 0.132, + "step": 66 + }, + { + "loss": -0.11447598040103912, + "grad_norm": 0.13181555271148682, + "learning_rate": 3e-05, + "num_tokens": 658875.0, + "completions/mean_length": 541.375, + "completions/min_length": 455.0, + "completions/max_length": 624.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 541.375, + "completions/min_terminated_length": 455.0, + "completions/max_terminated_length": 624.0, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02911720983684063, + "sampling/sampling_logp_difference/max": 0.44372403621673584, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4192371368408203, + "sampling/importance_sampling_ratio/max": 1.1883972883224487, + "kl": 0.03373931790702045, + "entropy": 1.2635268718004227, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.6137525443919, + "epoch": 0.134, + "step": 67 + }, + { + "loss": -0.10130438208580017, + "grad_norm": 0.20560002326965332, + "learning_rate": 3e-05, + "num_tokens": 671690.0, + "completions/mean_length": 689.4375, + "completions/min_length": 544.0, + "completions/max_length": 842.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 689.4375, + "completions/min_terminated_length": 544.0, + "completions/max_terminated_length": 842.0, + "rewards/quality_reward/mean": 5.875, + "rewards/quality_reward/std": 2.0289571285247803, + "reward": 5.875, + "reward_std": 2.0289571285247803, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028364315629005432, + "sampling/sampling_logp_difference/max": 0.35082435607910156, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.9476768374443054, + "sampling/importance_sampling_ratio/max": 2.8383262157440186, + "kl": 0.026702777307946235, + "entropy": 1.2496536895632744, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 29.35345455724746, + "epoch": 0.136, + "step": 68 + }, + { + "loss": -0.0560678169131279, + "grad_norm": 0.20431844890117645, + "learning_rate": 3e-05, + "num_tokens": 683046.0, + "completions/mean_length": 614.25, + "completions/min_length": 493.0, + "completions/max_length": 701.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 614.25, + "completions/min_terminated_length": 493.0, + "completions/max_terminated_length": 701.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "reward": 7.0625, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028125843033194542, + "sampling/sampling_logp_difference/max": 0.5758893489837646, + "sampling/importance_sampling_ratio/min": 0.05678822472691536, + "sampling/importance_sampling_ratio/mean": 0.5678162574768066, + "sampling/importance_sampling_ratio/max": 2.162766933441162, + "kl": 0.03377161466050893, + "entropy": 1.1948458775877953, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.574916049838066, + "epoch": 0.138, + "step": 69 + }, + { + "loss": 0.17035090923309326, + "grad_norm": 0.26451998949050903, + "learning_rate": 3e-05, + "num_tokens": 694323.0, + "completions/mean_length": 588.3125, + "completions/min_length": 532.0, + "completions/max_length": 673.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 588.3125, + "completions/min_terminated_length": 532.0, + "completions/max_terminated_length": 673.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02863166108727455, + "sampling/sampling_logp_difference/max": 0.46894216537475586, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.45171743631362915, + "sampling/importance_sampling_ratio/max": 1.950176477432251, + "kl": 0.03907236340455711, + "entropy": 1.2782762721180916, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.817490340210497, + "epoch": 0.14, + "step": 70 + }, + { + "loss": 0.4998631179332733, + "grad_norm": 0.5549097657203674, + "learning_rate": 3e-05, + "num_tokens": 705773.0, + "completions/mean_length": 597.125, + "completions/min_length": 492.0, + "completions/max_length": 683.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 597.125, + "completions/min_terminated_length": 492.0, + "completions/max_terminated_length": 683.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028340937569737434, + "sampling/sampling_logp_difference/max": 0.4324514865875244, + "sampling/importance_sampling_ratio/min": 0.05456363409757614, + "sampling/importance_sampling_ratio/mean": 0.7522475123405457, + "sampling/importance_sampling_ratio/max": 2.9404170513153076, + "kl": 0.03670362115371972, + "entropy": 1.2051330730319023, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 41.66373607888818, + "epoch": 0.142, + "step": 71 + }, + { + "loss": -0.02419177070260048, + "grad_norm": 0.10015435516834259, + "learning_rate": 3e-05, + "num_tokens": 717159.0, + "completions/mean_length": 604.125, + "completions/min_length": 536.0, + "completions/max_length": 691.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 604.125, + "completions/min_terminated_length": 536.0, + "completions/max_terminated_length": 691.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028367817401885986, + "sampling/sampling_logp_difference/max": 0.6400742530822754, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4037884473800659, + "sampling/importance_sampling_ratio/max": 1.1231037378311157, + "kl": 0.03911226138006896, + "entropy": 1.18794547021389, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.86539705330506, + "epoch": 0.144, + "step": 72 + }, + { + "loss": -0.027635926380753517, + "grad_norm": 0.07918722927570343, + "learning_rate": 3e-05, + "num_tokens": 728402.0, + "completions/mean_length": 597.1875, + "completions/min_length": 506.0, + "completions/max_length": 721.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 597.1875, + "completions/min_terminated_length": 506.0, + "completions/max_terminated_length": 721.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030029678717255592, + "sampling/sampling_logp_difference/max": 0.6819734573364258, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4324396848678589, + "sampling/importance_sampling_ratio/max": 1.5195796489715576, + "kl": 0.03177848691120744, + "entropy": 1.2641174346208572, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 29.29490938829258, + "epoch": 0.146, + "step": 73 + }, + { + "loss": -0.031142480671405792, + "grad_norm": 0.3853444755077362, + "learning_rate": 3e-05, + "num_tokens": 739632.0, + "completions/mean_length": 574.375, + "completions/min_length": 527.0, + "completions/max_length": 640.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 574.375, + "completions/min_terminated_length": 527.0, + "completions/max_terminated_length": 640.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030795859172940254, + "sampling/sampling_logp_difference/max": 0.42021608352661133, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.9200767874717712, + "sampling/importance_sampling_ratio/max": 2.6853926181793213, + "kl": 0.03433372196741402, + "entropy": 1.3364054411649704, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 34.52008486771956, + "epoch": 0.148, + "step": 74 + }, + { + "loss": -0.2718795835971832, + "grad_norm": 0.18101376295089722, + "learning_rate": 3e-05, + "num_tokens": 751164.0, + "completions/mean_length": 618.25, + "completions/min_length": 539.0, + "completions/max_length": 670.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 618.25, + "completions/min_terminated_length": 539.0, + "completions/max_terminated_length": 670.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028895940631628036, + "sampling/sampling_logp_difference/max": 0.42606019973754883, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5582400560379028, + "sampling/importance_sampling_ratio/max": 1.8397568464279175, + "kl": 0.02779634529724717, + "entropy": 1.365300178527832, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.76476171752438, + "epoch": 0.15, + "step": 75 + }, + { + "loss": -0.10660596191883087, + "grad_norm": 0.1852622777223587, + "learning_rate": 3e-05, + "num_tokens": 762370.0, + "completions/mean_length": 603.375, + "completions/min_length": 520.0, + "completions/max_length": 799.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 603.375, + "completions/min_terminated_length": 520.0, + "completions/max_terminated_length": 799.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028557566925883293, + "sampling/sampling_logp_difference/max": 0.481325626373291, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.550554633140564, + "sampling/importance_sampling_ratio/max": 2.317336082458496, + "kl": 0.03176840906962752, + "entropy": 1.2181015871465206, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.090129756834358, + "epoch": 0.152, + "step": 76 + }, + { + "loss": -0.20856647193431854, + "grad_norm": 0.1531844586133957, + "learning_rate": 3e-05, + "num_tokens": 773169.0, + "completions/mean_length": 572.9375, + "completions/min_length": 502.0, + "completions/max_length": 661.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 572.9375, + "completions/min_terminated_length": 502.0, + "completions/max_terminated_length": 661.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029906686395406723, + "sampling/sampling_logp_difference/max": 0.4692528247833252, + "sampling/importance_sampling_ratio/min": 0.1521405428647995, + "sampling/importance_sampling_ratio/mean": 0.8239375352859497, + "sampling/importance_sampling_ratio/max": 2.855010986328125, + "kl": 0.03523328877054155, + "entropy": 1.2986655682325363, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.004970545414835, + "epoch": 0.154, + "step": 77 + }, + { + "loss": -0.10802068561315536, + "grad_norm": 0.14113759994506836, + "learning_rate": 3e-05, + "num_tokens": 784724.0, + "completions/mean_length": 607.1875, + "completions/min_length": 540.0, + "completions/max_length": 705.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 607.1875, + "completions/min_terminated_length": 540.0, + "completions/max_terminated_length": 705.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027884263545274734, + "sampling/sampling_logp_difference/max": 0.4082927703857422, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.613497257232666, + "sampling/importance_sampling_ratio/max": 2.0096027851104736, + "kl": 0.025135049945674837, + "entropy": 1.1003647185862064, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 29.614154959097505, + "epoch": 0.156, + "step": 78 + }, + { + "loss": -0.0687609314918518, + "grad_norm": 0.18073000013828278, + "learning_rate": 3e-05, + "num_tokens": 795517.0, + "completions/mean_length": 568.0625, + "completions/min_length": 471.0, + "completions/max_length": 649.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 568.0625, + "completions/min_terminated_length": 471.0, + "completions/max_terminated_length": 649.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.9639329314231873, + "reward": 6.5625, + "reward_std": 0.9639329314231873, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029626762494444847, + "sampling/sampling_logp_difference/max": 0.4658241271972656, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.3485238552093506, + "sampling/importance_sampling_ratio/max": 1.277299404144287, + "kl": 0.028119354974478483, + "entropy": 1.266264721751213, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.5287338164635, + "epoch": 0.158, + "step": 79 + }, + { + "loss": 0.059164684265851974, + "grad_norm": 0.31362995505332947, + "learning_rate": 3e-05, + "num_tokens": 806258.0, + "completions/mean_length": 565.8125, + "completions/min_length": 499.0, + "completions/max_length": 674.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 565.8125, + "completions/min_terminated_length": 499.0, + "completions/max_terminated_length": 674.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02946357987821102, + "sampling/sampling_logp_difference/max": 0.4123659133911133, + "sampling/importance_sampling_ratio/min": 0.06356429308652878, + "sampling/importance_sampling_ratio/mean": 0.7027873396873474, + "sampling/importance_sampling_ratio/max": 2.5347814559936523, + "kl": 0.028471783734858036, + "entropy": 1.2773499339818954, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.251087154261768, + "epoch": 0.16, + "step": 80 + }, + { + "loss": 0.14755703508853912, + "grad_norm": 0.1959177404642105, + "learning_rate": 3e-05, + "num_tokens": 816717.0, + "completions/mean_length": 536.1875, + "completions/min_length": 464.0, + "completions/max_length": 600.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 536.1875, + "completions/min_terminated_length": 464.0, + "completions/max_terminated_length": 600.0, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.095445156097412, + "reward": 6.0, + "reward_std": 1.095445156097412, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02814806066453457, + "sampling/sampling_logp_difference/max": 0.28648805618286133, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8167816400527954, + "sampling/importance_sampling_ratio/max": 2.764387369155884, + "kl": 0.02152467134874314, + "entropy": 1.211024284362793, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.752198832575232, + "epoch": 0.162, + "step": 81 + }, + { + "loss": -0.1586945354938507, + "grad_norm": 0.16224367916584015, + "learning_rate": 3e-05, + "num_tokens": 827591.0, + "completions/mean_length": 568.625, + "completions/min_length": 514.0, + "completions/max_length": 673.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 568.625, + "completions/min_terminated_length": 514.0, + "completions/max_terminated_length": 673.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.26491117477417, + "reward": 6.5, + "reward_std": 1.26491117477417, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02937215007841587, + "sampling/sampling_logp_difference/max": 0.3966444730758667, + "sampling/importance_sampling_ratio/min": 0.1125984862446785, + "sampling/importance_sampling_ratio/mean": 0.7026975154876709, + "sampling/importance_sampling_ratio/max": 2.4172537326812744, + "kl": 0.02812566957436502, + "entropy": 1.2584010139107704, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.57465209439397, + "epoch": 0.164, + "step": 82 + }, + { + "loss": 0.09868354350328445, + "grad_norm": 0.28551891446113586, + "learning_rate": 3e-05, + "num_tokens": 838760.0, + "completions/mean_length": 583.5625, + "completions/min_length": 492.0, + "completions/max_length": 783.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 583.5625, + "completions/min_terminated_length": 492.0, + "completions/max_terminated_length": 783.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02833949774503708, + "sampling/sampling_logp_difference/max": 0.3390723466873169, + "sampling/importance_sampling_ratio/min": 0.2032935619354248, + "sampling/importance_sampling_ratio/mean": 0.7391272783279419, + "sampling/importance_sampling_ratio/max": 1.9430233240127563, + "kl": 0.023335880250670016, + "entropy": 1.2498536258935928, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.9633763525635, + "epoch": 0.166, + "step": 83 + }, + { + "loss": -0.04368923604488373, + "grad_norm": 0.1089889332652092, + "learning_rate": 3e-05, + "num_tokens": 849945.0, + "completions/mean_length": 581.0625, + "completions/min_length": 519.0, + "completions/max_length": 656.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 581.0625, + "completions/min_terminated_length": 519.0, + "completions/max_terminated_length": 656.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02942320704460144, + "sampling/sampling_logp_difference/max": 0.48818397521972656, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4864083528518677, + "sampling/importance_sampling_ratio/max": 1.6800583600997925, + "kl": 0.02546319324756041, + "entropy": 1.1582137942314148, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.7196712391451, + "epoch": 0.168, + "step": 84 + }, + { + "loss": 0.055040232837200165, + "grad_norm": 0.18219847977161407, + "learning_rate": 3e-05, + "num_tokens": 860734.0, + "completions/mean_length": 553.3125, + "completions/min_length": 492.0, + "completions/max_length": 647.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 553.3125, + "completions/min_terminated_length": 492.0, + "completions/max_terminated_length": 647.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028103860095143318, + "sampling/sampling_logp_difference/max": 0.3728243112564087, + "sampling/importance_sampling_ratio/min": 0.08803392946720123, + "sampling/importance_sampling_ratio/mean": 0.6381184458732605, + "sampling/importance_sampling_ratio/max": 2.2940757274627686, + "kl": 0.023275951389223337, + "entropy": 1.1762890554964542, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 28.569310082122684, + "epoch": 0.17, + "step": 85 + }, + { + "loss": 0.019699495285749435, + "grad_norm": 0.2631295323371887, + "learning_rate": 3e-05, + "num_tokens": 871182.0, + "completions/mean_length": 538.5, + "completions/min_length": 471.0, + "completions/max_length": 603.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 538.5, + "completions/min_terminated_length": 471.0, + "completions/max_terminated_length": 603.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02922222763299942, + "sampling/sampling_logp_difference/max": 0.358644962310791, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5496660470962524, + "sampling/importance_sampling_ratio/max": 2.950981378555298, + "kl": 0.027657793601974845, + "entropy": 1.1561524420976639, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.95468496438116, + "epoch": 0.172, + "step": 86 + }, + { + "loss": -0.0012568621896207333, + "grad_norm": 0.25233277678489685, + "learning_rate": 3e-05, + "num_tokens": 881272.0, + "completions/mean_length": 528.625, + "completions/min_length": 418.0, + "completions/max_length": 688.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 528.625, + "completions/min_terminated_length": 418.0, + "completions/max_terminated_length": 688.0, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.704154372215271, + "reward": 6.3125, + "reward_std": 0.704154372215271, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030555889010429382, + "sampling/sampling_logp_difference/max": 0.3433331251144409, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.44232380390167236, + "sampling/importance_sampling_ratio/max": 1.940340280532837, + "kl": 0.025461523793637753, + "entropy": 1.382395550608635, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 29.044239778537303, + "epoch": 0.174, + "step": 87 + }, + { + "loss": -0.0040507810190320015, + "grad_norm": 0.1609290987253189, + "learning_rate": 3e-05, + "num_tokens": 891608.0, + "completions/mean_length": 537.0, + "completions/min_length": 469.0, + "completions/max_length": 627.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 537.0, + "completions/min_terminated_length": 469.0, + "completions/max_terminated_length": 627.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028627343475818634, + "sampling/sampling_logp_difference/max": 0.3994784355163574, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4413543939590454, + "sampling/importance_sampling_ratio/max": 1.8592076301574707, + "kl": 0.023582924506627023, + "entropy": 1.1740282103419304, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.642031190916896, + "epoch": 0.176, + "step": 88 + }, + { + "loss": -0.10014888644218445, + "grad_norm": 0.25136521458625793, + "learning_rate": 3e-05, + "num_tokens": 902472.0, + "completions/mean_length": 562.0, + "completions/min_length": 490.0, + "completions/max_length": 625.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 562.0, + "completions/min_terminated_length": 490.0, + "completions/max_terminated_length": 625.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.154700517654419, + "reward": 6.5, + "reward_std": 1.154700517654419, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029525987803936005, + "sampling/sampling_logp_difference/max": 0.48168420791625977, + "sampling/importance_sampling_ratio/min": 0.031755149364471436, + "sampling/importance_sampling_ratio/mean": 0.5528109073638916, + "sampling/importance_sampling_ratio/max": 2.0172529220581055, + "kl": 0.03104234568309039, + "entropy": 1.3354259580373764, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.934129936154932, + "epoch": 0.178, + "step": 89 + }, + { + "loss": 0.02994484454393387, + "grad_norm": 0.2416294664144516, + "learning_rate": 3e-05, + "num_tokens": 913003.0, + "completions/mean_length": 543.6875, + "completions/min_length": 463.0, + "completions/max_length": 622.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 543.6875, + "completions/min_terminated_length": 463.0, + "completions/max_terminated_length": 622.0, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0246951580047607, + "reward": 6.125, + "reward_std": 1.0246951580047607, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030115650966763496, + "sampling/sampling_logp_difference/max": 0.36547136306762695, + "sampling/importance_sampling_ratio/min": 0.1312582641839981, + "sampling/importance_sampling_ratio/mean": 0.5672672390937805, + "sampling/importance_sampling_ratio/max": 1.4122473001480103, + "kl": 0.02474795945454389, + "entropy": 1.3072879239916801, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.719651044346392, + "epoch": 0.18, + "step": 90 + }, + { + "loss": 0.17740829288959503, + "grad_norm": 0.11401186883449554, + "learning_rate": 3e-05, + "num_tokens": 923971.0, + "completions/mean_length": 587.5, + "completions/min_length": 491.0, + "completions/max_length": 712.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 587.5, + "completions/min_terminated_length": 491.0, + "completions/max_terminated_length": 712.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027218280360102654, + "sampling/sampling_logp_difference/max": 0.43628501892089844, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4373893141746521, + "sampling/importance_sampling_ratio/max": 1.223915934562683, + "kl": 0.018764875654596835, + "entropy": 1.2642723061144352, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.256958127953112, + "epoch": 0.182, + "step": 91 + }, + { + "loss": -0.1092228814959526, + "grad_norm": 0.14000695943832397, + "learning_rate": 3e-05, + "num_tokens": 933880.0, + "completions/mean_length": 519.3125, + "completions/min_length": 461.0, + "completions/max_length": 589.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 519.3125, + "completions/min_terminated_length": 461.0, + "completions/max_terminated_length": 589.0, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.9574271440505981, + "reward": 7.125, + "reward_std": 0.9574271440505981, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02841360680758953, + "sampling/sampling_logp_difference/max": 0.3843420743942261, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6406391263008118, + "sampling/importance_sampling_ratio/max": 2.4079525470733643, + "kl": 0.017409664229489863, + "entropy": 1.2509336844086647, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.99564675288275, + "epoch": 0.184, + "step": 92 + }, + { + "loss": -0.15029624104499817, + "grad_norm": 0.16520461440086365, + "learning_rate": 3e-05, + "num_tokens": 945269.0, + "completions/mean_length": 587.8125, + "completions/min_length": 502.0, + "completions/max_length": 681.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 587.8125, + "completions/min_terminated_length": 502.0, + "completions/max_terminated_length": 681.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02762974239885807, + "sampling/sampling_logp_difference/max": 0.3571600914001465, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5902000069618225, + "sampling/importance_sampling_ratio/max": 1.7825064659118652, + "kl": 0.028165725176222622, + "entropy": 1.1215453520417213, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.922352592926472, + "epoch": 0.186, + "step": 93 + }, + { + "loss": 0.2850193381309509, + "grad_norm": 0.24750091135501862, + "learning_rate": 3e-05, + "num_tokens": 956021.0, + "completions/mean_length": 566.0, + "completions/min_length": 493.0, + "completions/max_length": 631.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 566.0, + "completions/min_terminated_length": 493.0, + "completions/max_terminated_length": 631.0, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029227914288640022, + "sampling/sampling_logp_difference/max": 0.4927506446838379, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6849822402000427, + "sampling/importance_sampling_ratio/max": 1.9146449565887451, + "kl": 0.02070689742686227, + "entropy": 1.2864234894514084, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.034203104209155, + "epoch": 0.188, + "step": 94 + }, + { + "loss": -0.032361116260290146, + "grad_norm": 0.11928244680166245, + "learning_rate": 3e-05, + "num_tokens": 965920.0, + "completions/mean_length": 516.6875, + "completions/min_length": 486.0, + "completions/max_length": 562.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 516.6875, + "completions/min_terminated_length": 486.0, + "completions/max_terminated_length": 562.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02843114361166954, + "sampling/sampling_logp_difference/max": 0.37001991271972656, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.32463955879211426, + "sampling/importance_sampling_ratio/max": 1.216193437576294, + "kl": 0.023086783126927912, + "entropy": 1.2404684573411942, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.103232022374868, + "epoch": 0.19, + "step": 95 + }, + { + "loss": 0.1640928089618683, + "grad_norm": 0.2920665442943573, + "learning_rate": 3e-05, + "num_tokens": 976695.0, + "completions/mean_length": 548.4375, + "completions/min_length": 484.0, + "completions/max_length": 650.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 548.4375, + "completions/min_terminated_length": 484.0, + "completions/max_terminated_length": 650.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02842729538679123, + "sampling/sampling_logp_difference/max": 0.31093156337738037, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.754618763923645, + "sampling/importance_sampling_ratio/max": 1.8954812288284302, + "kl": 0.017841250344645232, + "entropy": 1.2861761301755905, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 40.74571412149817, + "epoch": 0.192, + "step": 96 + }, + { + "loss": 0.05926981568336487, + "grad_norm": 0.29983460903167725, + "learning_rate": 3e-05, + "num_tokens": 987120.0, + "completions/mean_length": 553.5625, + "completions/min_length": 445.0, + "completions/max_length": 649.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 553.5625, + "completions/min_terminated_length": 445.0, + "completions/max_terminated_length": 649.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "reward": 6.5, + "reward_std": 0.632455587387085, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02854262851178646, + "sampling/sampling_logp_difference/max": 0.40996313095092773, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6645779609680176, + "sampling/importance_sampling_ratio/max": 2.0362496376037598, + "kl": 0.020173200638964772, + "entropy": 1.1928813084959984, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.38945102225989, + "epoch": 0.194, + "step": 97 + }, + { + "loss": -0.09927551448345184, + "grad_norm": 0.12385546416044235, + "learning_rate": 3e-05, + "num_tokens": 997395.0, + "completions/mean_length": 546.1875, + "completions/min_length": 475.0, + "completions/max_length": 628.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 546.1875, + "completions/min_terminated_length": 475.0, + "completions/max_terminated_length": 628.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029298899695277214, + "sampling/sampling_logp_difference/max": 0.35561084747314453, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5988417267799377, + "sampling/importance_sampling_ratio/max": 2.2717533111572266, + "kl": 0.02262102661188692, + "entropy": 1.365786962211132, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.35145698580891, + "epoch": 0.196, + "step": 98 + }, + { + "loss": 0.2574020326137543, + "grad_norm": 0.2662176787853241, + "learning_rate": 3e-05, + "num_tokens": 1008458.0, + "completions/mean_length": 593.4375, + "completions/min_length": 508.0, + "completions/max_length": 768.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 593.4375, + "completions/min_terminated_length": 508.0, + "completions/max_terminated_length": 768.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "reward": 6.875, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02820964716374874, + "sampling/sampling_logp_difference/max": 0.42550981044769287, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6420408487319946, + "sampling/importance_sampling_ratio/max": 2.1308085918426514, + "kl": 0.02589411090593785, + "entropy": 1.1754724085330963, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.02622760878876, + "epoch": 0.198, + "step": 99 + }, + { + "loss": 0.06747792661190033, + "grad_norm": 0.2549664378166199, + "learning_rate": 3e-05, + "num_tokens": 1018793.0, + "completions/mean_length": 538.4375, + "completions/min_length": 483.0, + "completions/max_length": 595.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 538.4375, + "completions/min_terminated_length": 483.0, + "completions/max_terminated_length": 595.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0299112256616354, + "sampling/sampling_logp_difference/max": 0.4304838180541992, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7165549993515015, + "sampling/importance_sampling_ratio/max": 2.3327062129974365, + "kl": 0.024644543533213437, + "entropy": 1.3136962801218033, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.768271412234753, + "epoch": 0.2, + "step": 100 + }, + { + "loss": -0.21041882038116455, + "grad_norm": 0.34537598490715027, + "learning_rate": 3e-05, + "num_tokens": 1029751.0, + "completions/mean_length": 567.875, + "completions/min_length": 493.0, + "completions/max_length": 661.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 567.875, + "completions/min_terminated_length": 493.0, + "completions/max_terminated_length": 661.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "reward": 7.0625, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02962569333612919, + "sampling/sampling_logp_difference/max": 0.3903813362121582, + "sampling/importance_sampling_ratio/min": 0.026465320959687233, + "sampling/importance_sampling_ratio/mean": 0.5392330288887024, + "sampling/importance_sampling_ratio/max": 2.282634973526001, + "kl": 0.025688456720672548, + "entropy": 1.15415108948946, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.715499105863273, + "epoch": 0.202, + "step": 101 + }, + { + "loss": 0.24403473734855652, + "grad_norm": 0.27241969108581543, + "learning_rate": 3e-05, + "num_tokens": 1040601.0, + "completions/mean_length": 574.125, + "completions/min_length": 495.0, + "completions/max_length": 662.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 574.125, + "completions/min_terminated_length": 495.0, + "completions/max_terminated_length": 662.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030232973396778107, + "sampling/sampling_logp_difference/max": 0.4158613681793213, + "sampling/importance_sampling_ratio/min": 0.07846305519342422, + "sampling/importance_sampling_ratio/mean": 0.6375491619110107, + "sampling/importance_sampling_ratio/max": 2.3365371227264404, + "kl": 0.025680337683297694, + "entropy": 1.3766441270709038, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.942134194541723, + "epoch": 0.204, + "step": 102 + }, + { + "loss": -0.07210355252027512, + "grad_norm": 0.2711203992366791, + "learning_rate": 3e-05, + "num_tokens": 1051229.0, + "completions/mean_length": 557.75, + "completions/min_length": 510.0, + "completions/max_length": 630.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 557.75, + "completions/min_terminated_length": 510.0, + "completions/max_terminated_length": 630.0, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.0144785642623901, + "reward": 6.3125, + "reward_std": 1.0144785642623901, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030201904475688934, + "sampling/sampling_logp_difference/max": 0.3629894256591797, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5626887083053589, + "sampling/importance_sampling_ratio/max": 1.8279200792312622, + "kl": 0.022622586810030043, + "entropy": 1.3887510225176811, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.44108156999573, + "epoch": 0.206, + "step": 103 + }, + { + "loss": 0.06065649166703224, + "grad_norm": 0.27351662516593933, + "learning_rate": 3e-05, + "num_tokens": 1061745.0, + "completions/mean_length": 544.75, + "completions/min_length": 484.0, + "completions/max_length": 622.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 544.75, + "completions/min_terminated_length": 484.0, + "completions/max_terminated_length": 622.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027608510106801987, + "sampling/sampling_logp_difference/max": 0.3885481357574463, + "sampling/importance_sampling_ratio/min": 0.05167346075177193, + "sampling/importance_sampling_ratio/mean": 0.8908482789993286, + "sampling/importance_sampling_ratio/max": 2.500911235809326, + "kl": 0.02198210428468883, + "entropy": 1.1656717136502266, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.46229960815981, + "epoch": 0.208, + "step": 104 + }, + { + "loss": 0.06463687866926193, + "grad_norm": 0.21975310146808624, + "learning_rate": 3e-05, + "num_tokens": 1072231.0, + "completions/mean_length": 562.875, + "completions/min_length": 499.0, + "completions/max_length": 616.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 562.875, + "completions/min_terminated_length": 499.0, + "completions/max_terminated_length": 616.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030092012137174606, + "sampling/sampling_logp_difference/max": 0.9359657764434814, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5429776906967163, + "sampling/importance_sampling_ratio/max": 1.7785983085632324, + "kl": 0.023721053614281118, + "entropy": 1.3012276738882065, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.80119998846203, + "epoch": 0.21, + "step": 105 + }, + { + "loss": -0.004262822680175304, + "grad_norm": 0.23242872953414917, + "learning_rate": 3e-05, + "num_tokens": 1083184.0, + "completions/mean_length": 580.0625, + "completions/min_length": 502.0, + "completions/max_length": 656.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 580.0625, + "completions/min_terminated_length": 502.0, + "completions/max_terminated_length": 656.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.3102163076400757, + "reward": 6.375, + "reward_std": 1.3102163076400757, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02952728420495987, + "sampling/sampling_logp_difference/max": 0.7246572971343994, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6959555745124817, + "sampling/importance_sampling_ratio/max": 2.0552361011505127, + "kl": 0.025716557982377708, + "entropy": 1.3428374752402306, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.27699494967237, + "epoch": 0.212, + "step": 106 + }, + { + "loss": 0.08491670340299606, + "grad_norm": 0.23287689685821533, + "learning_rate": 3e-05, + "num_tokens": 1094204.0, + "completions/mean_length": 589.75, + "completions/min_length": 547.0, + "completions/max_length": 655.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 589.75, + "completions/min_terminated_length": 547.0, + "completions/max_terminated_length": 655.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02880111336708069, + "sampling/sampling_logp_difference/max": 0.36995506286621094, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5767678022384644, + "sampling/importance_sampling_ratio/max": 2.1368408203125, + "kl": 0.025674917036667466, + "entropy": 1.471379242837429, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 40.17427978478372, + "epoch": 0.214, + "step": 107 + }, + { + "loss": 0.04861483350396156, + "grad_norm": 0.09948146343231201, + "learning_rate": 3e-05, + "num_tokens": 1105303.0, + "completions/mean_length": 595.6875, + "completions/min_length": 524.0, + "completions/max_length": 699.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 595.6875, + "completions/min_terminated_length": 524.0, + "completions/max_terminated_length": 699.0, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.0878112316131592, + "reward": 6.125, + "reward_std": 1.0878112316131592, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029913678765296936, + "sampling/sampling_logp_difference/max": 0.3477153778076172, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.3343955874443054, + "sampling/importance_sampling_ratio/max": 1.1923820972442627, + "kl": 0.022876911563798785, + "entropy": 1.2550728917121887, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.59066634438932, + "epoch": 0.216, + "step": 108 + }, + { + "loss": 0.10488568246364594, + "grad_norm": 0.1392563134431839, + "learning_rate": 3e-05, + "num_tokens": 1116591.0, + "completions/mean_length": 583.5, + "completions/min_length": 467.0, + "completions/max_length": 670.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 583.5, + "completions/min_terminated_length": 467.0, + "completions/max_terminated_length": 670.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0288787130266428, + "sampling/sampling_logp_difference/max": 0.31003570556640625, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4996475875377655, + "sampling/importance_sampling_ratio/max": 2.0463244915008545, + "kl": 0.0263087993953377, + "entropy": 1.2097205966711044, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.845282280817628, + "epoch": 0.218, + "step": 109 + }, + { + "loss": -0.23892748355865479, + "grad_norm": 0.5394540429115295, + "learning_rate": 3e-05, + "num_tokens": 1127493.0, + "completions/mean_length": 571.375, + "completions/min_length": 492.0, + "completions/max_length": 740.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 571.375, + "completions/min_terminated_length": 492.0, + "completions/max_terminated_length": 740.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028643080964684486, + "sampling/sampling_logp_difference/max": 0.3650541305541992, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 1.223832130432129, + "sampling/importance_sampling_ratio/max": 2.671478509902954, + "kl": 0.024339908617548645, + "entropy": 1.28146480768919, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.93386760680005, + "epoch": 0.22, + "step": 110 + }, + { + "loss": -0.21332937479019165, + "grad_norm": 0.4297163188457489, + "learning_rate": 3e-05, + "num_tokens": 1136979.0, + "completions/mean_length": 493.875, + "completions/min_length": 344.0, + "completions/max_length": 573.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 493.875, + "completions/min_terminated_length": 344.0, + "completions/max_terminated_length": 573.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.6800735592842102, + "reward": 6.9375, + "reward_std": 0.6800735592842102, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028577474877238274, + "sampling/sampling_logp_difference/max": 0.49600934982299805, + "sampling/importance_sampling_ratio/min": 0.02116413414478302, + "sampling/importance_sampling_ratio/mean": 0.7321407794952393, + "sampling/importance_sampling_ratio/max": 2.889394760131836, + "kl": 0.024493444827385247, + "entropy": 1.12203798443079, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.056686570402235, + "epoch": 0.222, + "step": 111 + }, + { + "loss": 0.0004943995736539364, + "grad_norm": 0.008436380885541439, + "learning_rate": 3e-05, + "num_tokens": 1148176.0, + "completions/mean_length": 583.8125, + "completions/min_length": 505.0, + "completions/max_length": 672.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 583.8125, + "completions/min_terminated_length": 505.0, + "completions/max_terminated_length": 672.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "reward": 7.0, + "reward_std": 0.0, + "frac_reward_zero_std": 1.0, + "sampling/sampling_logp_difference/mean": 0.03075096383690834, + "sampling/sampling_logp_difference/max": 0.572547435760498, + "sampling/importance_sampling_ratio/min": 0.006825839169323444, + "sampling/importance_sampling_ratio/mean": 0.7060814499855042, + "sampling/importance_sampling_ratio/max": 2.642366886138916, + "kl": 0.024869016953743994, + "entropy": 1.4159239530563354, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.1555822850205, + "epoch": 0.224, + "step": 112 + }, + { + "loss": -0.22745110094547272, + "grad_norm": 0.2439681738615036, + "learning_rate": 3e-05, + "num_tokens": 1159380.0, + "completions/mean_length": 582.75, + "completions/min_length": 502.0, + "completions/max_length": 638.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 582.75, + "completions/min_terminated_length": 502.0, + "completions/max_terminated_length": 638.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02884429693222046, + "sampling/sampling_logp_difference/max": 0.2513730525970459, + "sampling/importance_sampling_ratio/min": 0.021104907616972923, + "sampling/importance_sampling_ratio/mean": 1.0150926113128662, + "sampling/importance_sampling_ratio/max": 2.5284109115600586, + "kl": 0.025248280493542552, + "entropy": 1.2932439520955086, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.09852197067812, + "epoch": 0.226, + "step": 113 + }, + { + "loss": 0.3550976812839508, + "grad_norm": 0.23755788803100586, + "learning_rate": 3e-05, + "num_tokens": 1170628.0, + "completions/mean_length": 580.0, + "completions/min_length": 535.0, + "completions/max_length": 627.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 580.0, + "completions/min_terminated_length": 535.0, + "completions/max_terminated_length": 627.0, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 7.25, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03159492090344429, + "sampling/sampling_logp_difference/max": 0.5576918125152588, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 1.0749173164367676, + "sampling/importance_sampling_ratio/max": 2.750225305557251, + "kl": 0.02679906424600631, + "entropy": 1.246352232992649, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.629896679893136, + "epoch": 0.228, + "step": 114 + }, + { + "loss": -0.05118201673030853, + "grad_norm": 0.3102787733078003, + "learning_rate": 3e-05, + "num_tokens": 1181295.0, + "completions/mean_length": 562.6875, + "completions/min_length": 496.0, + "completions/max_length": 657.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 562.6875, + "completions/min_terminated_length": 496.0, + "completions/max_terminated_length": 657.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030200565233826637, + "sampling/sampling_logp_difference/max": 0.6224374771118164, + "sampling/importance_sampling_ratio/min": 0.09920533001422882, + "sampling/importance_sampling_ratio/mean": 0.8949281573295593, + "sampling/importance_sampling_ratio/max": 2.634671926498413, + "kl": 0.027133187628351152, + "entropy": 1.253239594399929, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 38.715506959706545, + "epoch": 0.23, + "step": 115 + }, + { + "loss": -0.05550215765833855, + "grad_norm": 0.14933602511882782, + "learning_rate": 3e-05, + "num_tokens": 1192750.0, + "completions/mean_length": 610.9375, + "completions/min_length": 538.0, + "completions/max_length": 706.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 610.9375, + "completions/min_terminated_length": 538.0, + "completions/max_terminated_length": 706.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "reward": 6.625, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03152918070554733, + "sampling/sampling_logp_difference/max": 0.5450258255004883, + "sampling/importance_sampling_ratio/min": 0.009796842001378536, + "sampling/importance_sampling_ratio/mean": 0.41488125920295715, + "sampling/importance_sampling_ratio/max": 1.6116093397140503, + "kl": 0.031228074803948402, + "entropy": 1.2940760999917984, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.821606623008847, + "epoch": 0.232, + "step": 116 + }, + { + "loss": -0.09421571344137192, + "grad_norm": 0.23778750002384186, + "learning_rate": 3e-05, + "num_tokens": 1203219.0, + "completions/mean_length": 560.8125, + "completions/min_length": 501.0, + "completions/max_length": 630.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 560.8125, + "completions/min_terminated_length": 501.0, + "completions/max_terminated_length": 630.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "reward": 6.8125, + "reward_std": 0.75, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030015992000699043, + "sampling/sampling_logp_difference/max": 0.8687701225280762, + "sampling/importance_sampling_ratio/min": 0.010288448072969913, + "sampling/importance_sampling_ratio/mean": 0.7292319536209106, + "sampling/importance_sampling_ratio/max": 2.1054162979125977, + "kl": 0.03231424337718636, + "entropy": 1.21239273250103, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.80020274501294, + "epoch": 0.234, + "step": 117 + }, + { + "loss": 0.010135526768863201, + "grad_norm": 0.177206888794899, + "learning_rate": 3e-05, + "num_tokens": 1214562.0, + "completions/mean_length": 583.4375, + "completions/min_length": 503.0, + "completions/max_length": 712.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 583.4375, + "completions/min_terminated_length": 503.0, + "completions/max_terminated_length": 712.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.0246951580047607, + "reward": 6.375, + "reward_std": 1.0246951580047607, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02755727432668209, + "sampling/sampling_logp_difference/max": 0.37839651107788086, + "sampling/importance_sampling_ratio/min": 0.14393718540668488, + "sampling/importance_sampling_ratio/mean": 0.5455202460289001, + "sampling/importance_sampling_ratio/max": 1.6413226127624512, + "kl": 0.027808396029286087, + "entropy": 1.0914121232926846, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 32.04508572584018, + "epoch": 0.236, + "step": 118 + }, + { + "loss": -0.05460066720843315, + "grad_norm": 0.10887355357408524, + "learning_rate": 3e-05, + "num_tokens": 1226136.0, + "completions/mean_length": 623.875, + "completions/min_length": 555.0, + "completions/max_length": 802.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 623.875, + "completions/min_terminated_length": 555.0, + "completions/max_terminated_length": 802.0, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.983263373374939, + "reward": 6.25, + "reward_std": 1.983263373374939, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03079008124768734, + "sampling/sampling_logp_difference/max": 0.5650186538696289, + "sampling/importance_sampling_ratio/min": 0.06713607162237167, + "sampling/importance_sampling_ratio/mean": 0.4625241756439209, + "sampling/importance_sampling_ratio/max": 1.5456031560897827, + "kl": 0.029803494922816753, + "entropy": 1.2099597677588463, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.80143166380003, + "epoch": 0.238, + "step": 119 + }, + { + "loss": -0.1489834189414978, + "grad_norm": 0.1758948117494583, + "learning_rate": 3e-05, + "num_tokens": 1237856.0, + "completions/mean_length": 621.0, + "completions/min_length": 536.0, + "completions/max_length": 722.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 621.0, + "completions/min_terminated_length": 536.0, + "completions/max_terminated_length": 722.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "reward": 6.5, + "reward_std": 0.632455587387085, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029741039499640465, + "sampling/sampling_logp_difference/max": 0.40829265117645264, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7058912515640259, + "sampling/importance_sampling_ratio/max": 2.4328413009643555, + "kl": 0.03042414935771376, + "entropy": 1.3042216151952744, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.90572352707386, + "epoch": 0.24, + "step": 120 + }, + { + "loss": 0.22666211426258087, + "grad_norm": 0.42553070187568665, + "learning_rate": 3e-05, + "num_tokens": 1249173.0, + "completions/mean_length": 593.8125, + "completions/min_length": 62.0, + "completions/max_length": 740.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 593.8125, + "completions/min_terminated_length": 62.0, + "completions/max_terminated_length": 740.0, + "rewards/quality_reward/mean": 5.625, + "rewards/quality_reward/std": 1.8574175834655762, + "reward": 5.625, + "reward_std": 1.8574175834655762, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030399736016988754, + "sampling/sampling_logp_difference/max": 0.6260476112365723, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6919515132904053, + "sampling/importance_sampling_ratio/max": 2.5172836780548096, + "kl": 0.04428348131477833, + "entropy": 1.132676463574171, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 38.02521731564775, + "epoch": 0.242, + "step": 121 + }, + { + "loss": 0.024570390582084656, + "grad_norm": 0.248436838388443, + "learning_rate": 3e-05, + "num_tokens": 1260463.0, + "completions/mean_length": 587.625, + "completions/min_length": 527.0, + "completions/max_length": 662.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 587.625, + "completions/min_terminated_length": 527.0, + "completions/max_terminated_length": 662.0, + "rewards/quality_reward/mean": 6.0625, + "rewards/quality_reward/std": 1.236594796180725, + "reward": 6.0625, + "reward_std": 1.236594796180725, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030399201437830925, + "sampling/sampling_logp_difference/max": 0.540553092956543, + "sampling/importance_sampling_ratio/min": 0.06326956301927567, + "sampling/importance_sampling_ratio/mean": 0.8825340867042542, + "sampling/importance_sampling_ratio/max": 2.833437442779541, + "kl": 0.03361149993725121, + "entropy": 1.2395975515246391, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.796182619407773, + "epoch": 0.244, + "step": 122 + }, + { + "loss": -0.053390923887491226, + "grad_norm": 0.16029156744480133, + "learning_rate": 3e-05, + "num_tokens": 1271644.0, + "completions/mean_length": 585.3125, + "completions/min_length": 506.0, + "completions/max_length": 648.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 585.3125, + "completions/min_terminated_length": 506.0, + "completions/max_terminated_length": 648.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.9639329314231873, + "reward": 6.4375, + "reward_std": 0.9639329314231873, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.031593482941389084, + "sampling/sampling_logp_difference/max": 0.7028732299804688, + "sampling/importance_sampling_ratio/min": 0.08802779763936996, + "sampling/importance_sampling_ratio/mean": 0.5607253909111023, + "sampling/importance_sampling_ratio/max": 2.6618704795837402, + "kl": 0.03495740657672286, + "entropy": 1.375852882862091, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.86260191956535, + "epoch": 0.246, + "step": 123 + }, + { + "loss": -0.010514559224247932, + "grad_norm": 0.19288285076618195, + "learning_rate": 3e-05, + "num_tokens": 1283351.0, + "completions/mean_length": 630.1875, + "completions/min_length": 571.0, + "completions/max_length": 676.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 630.1875, + "completions/min_terminated_length": 571.0, + "completions/max_terminated_length": 676.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 1.0246951580047607, + "reward": 6.625, + "reward_std": 1.0246951580047607, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030591927468776703, + "sampling/sampling_logp_difference/max": 0.6886825561523438, + "sampling/importance_sampling_ratio/min": 0.007580960635095835, + "sampling/importance_sampling_ratio/mean": 0.5407211184501648, + "sampling/importance_sampling_ratio/max": 1.4858638048171997, + "kl": 0.033586084260605276, + "entropy": 1.3353190049529076, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.809663326013833, + "epoch": 0.248, + "step": 124 + }, + { + "loss": 0.10287950932979584, + "grad_norm": 0.15600983798503876, + "learning_rate": 3e-05, + "num_tokens": 1294997.0, + "completions/mean_length": 599.875, + "completions/min_length": 484.0, + "completions/max_length": 683.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 599.875, + "completions/min_terminated_length": 484.0, + "completions/max_terminated_length": 683.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.730296790599823, + "reward": 7.0, + "reward_std": 0.730296790599823, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030487919226288795, + "sampling/sampling_logp_difference/max": 0.5988303422927856, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6814589500427246, + "sampling/importance_sampling_ratio/max": 1.832617163658142, + "kl": 0.03213575447443873, + "entropy": 1.168940719217062, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 36.94939920771867, + "epoch": 0.25, + "step": 125 + }, + { + "loss": 0.16848862171173096, + "grad_norm": 0.3560245931148529, + "learning_rate": 3e-05, + "num_tokens": 1305993.0, + "completions/mean_length": 582.25, + "completions/min_length": 527.0, + "completions/max_length": 634.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 582.25, + "completions/min_terminated_length": 527.0, + "completions/max_terminated_length": 634.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029648227617144585, + "sampling/sampling_logp_difference/max": 0.7731144428253174, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7458471059799194, + "sampling/importance_sampling_ratio/max": 2.17899489402771, + "kl": 0.025753034162335098, + "entropy": 1.0670793130993843, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.284966757521033, + "epoch": 0.252, + "step": 126 + }, + { + "loss": 0.16780534386634827, + "grad_norm": 0.18122251331806183, + "learning_rate": 3e-05, + "num_tokens": 1318054.0, + "completions/mean_length": 650.8125, + "completions/min_length": 571.0, + "completions/max_length": 717.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 650.8125, + "completions/min_terminated_length": 571.0, + "completions/max_terminated_length": 717.0, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03218457102775574, + "sampling/sampling_logp_difference/max": 1.0368115901947021, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5485143065452576, + "sampling/importance_sampling_ratio/max": 1.937699556350708, + "kl": 0.03214431612286717, + "entropy": 1.2732752412557602, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.58785921242088, + "epoch": 0.254, + "step": 127 + }, + { + "loss": 0.06286599487066269, + "grad_norm": 0.1356409788131714, + "learning_rate": 3e-05, + "num_tokens": 1329500.0, + "completions/mean_length": 617.375, + "completions/min_length": 530.0, + "completions/max_length": 743.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 617.375, + "completions/min_terminated_length": 530.0, + "completions/max_terminated_length": 743.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.032271042466163635, + "sampling/sampling_logp_difference/max": 0.676828145980835, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4681059420108795, + "sampling/importance_sampling_ratio/max": 2.106013298034668, + "kl": 0.026579287368804216, + "entropy": 1.306506209075451, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.268729500938207, + "epoch": 0.256, + "step": 128 + }, + { + "loss": -0.059055861085653305, + "grad_norm": 0.2950517237186432, + "learning_rate": 3e-05, + "num_tokens": 1341020.0, + "completions/mean_length": 599.0, + "completions/min_length": 519.0, + "completions/max_length": 722.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 599.0, + "completions/min_terminated_length": 519.0, + "completions/max_terminated_length": 722.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.031067587435245514, + "sampling/sampling_logp_difference/max": 0.5858409404754639, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6402126550674438, + "sampling/importance_sampling_ratio/max": 2.7381844520568848, + "kl": 0.02614310395438224, + "entropy": 1.3493447452783585, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 28.869210730306804, + "epoch": 0.258, + "step": 129 + }, + { + "loss": 0.003650778206065297, + "grad_norm": 0.42696425318717957, + "learning_rate": 3e-05, + "num_tokens": 1351928.0, + "completions/mean_length": 585.75, + "completions/min_length": 520.0, + "completions/max_length": 710.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 585.75, + "completions/min_terminated_length": 520.0, + "completions/max_terminated_length": 710.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.031333789229393005, + "sampling/sampling_logp_difference/max": 0.4930081367492676, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8535536527633667, + "sampling/importance_sampling_ratio/max": 2.9060652256011963, + "kl": 0.02683100081048906, + "entropy": 1.2956265732645988, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.11609491892159, + "epoch": 0.26, + "step": 130 + }, + { + "loss": -0.01662549562752247, + "grad_norm": 0.20076203346252441, + "learning_rate": 3e-05, + "num_tokens": 1362825.0, + "completions/mean_length": 568.5625, + "completions/min_length": 470.0, + "completions/max_length": 653.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 568.5625, + "completions/min_terminated_length": 470.0, + "completions/max_terminated_length": 653.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "reward": 6.9375, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0319429412484169, + "sampling/sampling_logp_difference/max": 0.963031530380249, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6302506327629089, + "sampling/importance_sampling_ratio/max": 2.6266331672668457, + "kl": 0.024291134322993457, + "entropy": 1.2097233161330223, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.060474771540612, + "epoch": 0.262, + "step": 131 + }, + { + "loss": -0.0666906088590622, + "grad_norm": 0.184434711933136, + "learning_rate": 3e-05, + "num_tokens": 1374296.0, + "completions/mean_length": 605.9375, + "completions/min_length": 505.0, + "completions/max_length": 741.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 605.9375, + "completions/min_terminated_length": 505.0, + "completions/max_terminated_length": 741.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.6191391944885254, + "reward": 6.375, + "reward_std": 0.6191391944885254, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030500290915369987, + "sampling/sampling_logp_difference/max": 0.671515941619873, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5425443649291992, + "sampling/importance_sampling_ratio/max": 1.831458568572998, + "kl": 0.02043789450544864, + "entropy": 1.2823583781719208, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.10482985060662, + "epoch": 0.264, + "step": 132 + }, + { + "loss": 0.6353421211242676, + "grad_norm": 0.4572683274745941, + "learning_rate": 3e-05, + "num_tokens": 1385318.0, + "completions/mean_length": 591.375, + "completions/min_length": 513.0, + "completions/max_length": 665.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 591.375, + "completions/min_terminated_length": 513.0, + "completions/max_terminated_length": 665.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028536029160022736, + "sampling/sampling_logp_difference/max": 0.3869748115539551, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7995439767837524, + "sampling/importance_sampling_ratio/max": 2.3584084510803223, + "kl": 0.025262096198275685, + "entropy": 1.1699804589152336, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 36.004622367210686, + "epoch": 0.266, + "step": 133 + }, + { + "loss": 0.026529084891080856, + "grad_norm": 0.256655752658844, + "learning_rate": 3e-05, + "num_tokens": 1396234.0, + "completions/mean_length": 554.25, + "completions/min_length": 463.0, + "completions/max_length": 652.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 554.25, + "completions/min_terminated_length": 463.0, + "completions/max_terminated_length": 652.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.7745966911315918, + "reward": 6.75, + "reward_std": 0.7745966911315918, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02867746911942959, + "sampling/sampling_logp_difference/max": 1.145315170288086, + "sampling/importance_sampling_ratio/min": 0.022647401317954063, + "sampling/importance_sampling_ratio/mean": 0.8202446699142456, + "sampling/importance_sampling_ratio/max": 1.9517148733139038, + "kl": 0.022568197746295482, + "entropy": 1.1336797252297401, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.057327402289957, + "epoch": 0.268, + "step": 134 + }, + { + "loss": 0.2014756053686142, + "grad_norm": 0.30302342772483826, + "learning_rate": 3e-05, + "num_tokens": 1407322.0, + "completions/mean_length": 573.5, + "completions/min_length": 499.0, + "completions/max_length": 641.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 573.5, + "completions/min_terminated_length": 499.0, + "completions/max_terminated_length": 641.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "reward": 6.9375, + "reward_std": 0.57373046875, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03068450093269348, + "sampling/sampling_logp_difference/max": 0.4479391574859619, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6178270578384399, + "sampling/importance_sampling_ratio/max": 2.9701545238494873, + "kl": 0.02298387698829174, + "entropy": 1.1878332123160362, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 36.01238542608917, + "epoch": 0.27, + "step": 135 + }, + { + "loss": 0.3642374873161316, + "grad_norm": 0.39115583896636963, + "learning_rate": 3e-05, + "num_tokens": 1418010.0, + "completions/mean_length": 534.0, + "completions/min_length": 466.0, + "completions/max_length": 647.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 534.0, + "completions/min_terminated_length": 466.0, + "completions/max_terminated_length": 647.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.7187952995300293, + "reward": 6.375, + "reward_std": 0.7187952995300293, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0298798568546772, + "sampling/sampling_logp_difference/max": 0.5613884925842285, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7149391770362854, + "sampling/importance_sampling_ratio/max": 2.0828306674957275, + "kl": 0.020954113570041955, + "entropy": 1.2640416622161865, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.20259432308376, + "epoch": 0.272, + "step": 136 + }, + { + "loss": -0.15938539803028107, + "grad_norm": 0.4104433059692383, + "learning_rate": 3e-05, + "num_tokens": 1428335.0, + "completions/mean_length": 538.8125, + "completions/min_length": 451.0, + "completions/max_length": 621.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 538.8125, + "completions/min_terminated_length": 451.0, + "completions/max_terminated_length": 621.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.7187952995300293, + "reward": 6.875, + "reward_std": 0.7187952995300293, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030781995505094528, + "sampling/sampling_logp_difference/max": 0.46263813972473145, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5245123505592346, + "sampling/importance_sampling_ratio/max": 2.013305425643921, + "kl": 0.02069689182098955, + "entropy": 1.2438515722751617, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.362532567232847, + "epoch": 0.274, + "step": 137 + }, + { + "loss": 0.3297041654586792, + "grad_norm": 0.41347458958625793, + "learning_rate": 3e-05, + "num_tokens": 1438734.0, + "completions/mean_length": 542.4375, + "completions/min_length": 459.0, + "completions/max_length": 618.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 542.4375, + "completions/min_terminated_length": 459.0, + "completions/max_terminated_length": 618.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02844768762588501, + "sampling/sampling_logp_difference/max": 0.40987062454223633, + "sampling/importance_sampling_ratio/min": 0.016625043004751205, + "sampling/importance_sampling_ratio/mean": 0.822363555431366, + "sampling/importance_sampling_ratio/max": 2.6671712398529053, + "kl": 0.020388772361911833, + "entropy": 1.2741251289844513, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 34.91616539563984, + "epoch": 0.276, + "step": 138 + }, + { + "loss": -0.25778308510780334, + "grad_norm": 0.3896540701389313, + "learning_rate": 3e-05, + "num_tokens": 1448611.0, + "completions/mean_length": 516.3125, + "completions/min_length": 440.0, + "completions/max_length": 627.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 516.3125, + "completions/min_terminated_length": 440.0, + "completions/max_terminated_length": 627.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027655858546495438, + "sampling/sampling_logp_difference/max": 0.2820701599121094, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6815162897109985, + "sampling/importance_sampling_ratio/max": 1.9001679420471191, + "kl": 0.022598200594075024, + "entropy": 1.2030403539538383, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.81112790806219, + "epoch": 0.278, + "step": 139 + }, + { + "loss": 0.09288515895605087, + "grad_norm": 0.2338583916425705, + "learning_rate": 3e-05, + "num_tokens": 1458456.0, + "completions/mean_length": 512.8125, + "completions/min_length": 450.0, + "completions/max_length": 569.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 512.8125, + "completions/min_terminated_length": 450.0, + "completions/max_terminated_length": 569.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.375, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02669401653110981, + "sampling/sampling_logp_difference/max": 0.4399615526199341, + "sampling/importance_sampling_ratio/min": 0.13776090741157532, + "sampling/importance_sampling_ratio/mean": 0.7086957693099976, + "sampling/importance_sampling_ratio/max": 2.7245709896087646, + "kl": 0.022081921808421612, + "entropy": 1.124063789844513, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.541061893571168, + "epoch": 0.28, + "step": 140 + }, + { + "loss": 0.08992868661880493, + "grad_norm": 0.1792808622121811, + "learning_rate": 3e-05, + "num_tokens": 1468623.0, + "completions/mean_length": 521.9375, + "completions/min_length": 460.0, + "completions/max_length": 621.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 521.9375, + "completions/min_terminated_length": 460.0, + "completions/max_terminated_length": 621.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.6191391944885254, + "reward": 6.625, + "reward_std": 0.6191391944885254, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028398238122463226, + "sampling/sampling_logp_difference/max": 0.42975759506225586, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5165826082229614, + "sampling/importance_sampling_ratio/max": 1.6533762216567993, + "kl": 0.02498150523751974, + "entropy": 1.3581550270318985, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.935550182592124, + "epoch": 0.282, + "step": 141 + }, + { + "loss": 0.24750135838985443, + "grad_norm": 0.2593871057033539, + "learning_rate": 3e-05, + "num_tokens": 1477449.0, + "completions/mean_length": 454.625, + "completions/min_length": 274.0, + "completions/max_length": 511.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 454.625, + "completions/min_terminated_length": 274.0, + "completions/max_terminated_length": 511.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "reward": 7.0, + "reward_std": 0.632455587387085, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029166901484131813, + "sampling/sampling_logp_difference/max": 0.3056577444076538, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6540807485580444, + "sampling/importance_sampling_ratio/max": 1.5698224306106567, + "kl": 0.024267837987281382, + "entropy": 1.2165675312280655, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.03540184069425, + "epoch": 0.284, + "step": 142 + }, + { + "loss": 0.330167293548584, + "grad_norm": 0.37285444140434265, + "learning_rate": 3e-05, + "num_tokens": 1486931.0, + "completions/mean_length": 490.625, + "completions/min_length": 446.0, + "completions/max_length": 526.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 490.625, + "completions/min_terminated_length": 446.0, + "completions/max_terminated_length": 526.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02719968557357788, + "sampling/sampling_logp_difference/max": 0.42168426513671875, + "sampling/importance_sampling_ratio/min": 0.07088703662157059, + "sampling/importance_sampling_ratio/mean": 0.7264441251754761, + "sampling/importance_sampling_ratio/max": 2.156266927719116, + "kl": 0.019997276307549328, + "entropy": 1.1259984448552132, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.82226228993386, + "epoch": 0.286, + "step": 143 + }, + { + "loss": 0.07405021786689758, + "grad_norm": 0.11882677674293518, + "learning_rate": 3e-05, + "num_tokens": 1496916.0, + "completions/mean_length": 529.0625, + "completions/min_length": 421.0, + "completions/max_length": 1024.0, + "completions/clipped_ratio": 0.0625, + "completions/mean_terminated_length": 496.0666809082031, + "completions/min_terminated_length": 421.0, + "completions/max_terminated_length": 631.0, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 1.732050895690918, + "reward": 5.75, + "reward_std": 1.732050895690918, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02569635957479477, + "sampling/sampling_logp_difference/max": 0.6632819175720215, + "sampling/importance_sampling_ratio/min": 0.11731626838445663, + "sampling/importance_sampling_ratio/mean": 0.39511895179748535, + "sampling/importance_sampling_ratio/max": 0.8768613934516907, + "kl": 0.027060870255809277, + "entropy": 1.0426596216857433, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.100794151891023, + "epoch": 0.288, + "step": 144 + }, + { + "loss": -0.18078479170799255, + "grad_norm": 0.25876912474632263, + "learning_rate": 3e-05, + "num_tokens": 1507157.0, + "completions/mean_length": 502.5625, + "completions/min_length": 452.0, + "completions/max_length": 544.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 502.5625, + "completions/min_terminated_length": 452.0, + "completions/max_terminated_length": 544.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02768835797905922, + "sampling/sampling_logp_difference/max": 0.3683091402053833, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7513852119445801, + "sampling/importance_sampling_ratio/max": 1.90565824508667, + "kl": 0.028287828317843378, + "entropy": 1.2609772458672523, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.17377450503409, + "epoch": 0.29, + "step": 145 + }, + { + "loss": -0.07925756275653839, + "grad_norm": 0.4150021970272064, + "learning_rate": 3e-05, + "num_tokens": 1516833.0, + "completions/mean_length": 478.25, + "completions/min_length": 443.0, + "completions/max_length": 586.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 478.25, + "completions/min_terminated_length": 443.0, + "completions/max_terminated_length": 586.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.095445156097412, + "reward": 6.5, + "reward_std": 1.095445156097412, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028425993397831917, + "sampling/sampling_logp_difference/max": 0.40656137466430664, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6499220132827759, + "sampling/importance_sampling_ratio/max": 2.0522961616516113, + "kl": 0.027647150680422783, + "entropy": 1.2076954543590546, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.426104408223182, + "epoch": 0.292, + "step": 146 + }, + { + "loss": 0.04165569692850113, + "grad_norm": 0.33721745014190674, + "learning_rate": 3e-05, + "num_tokens": 1526028.0, + "completions/mean_length": 467.1875, + "completions/min_length": 396.0, + "completions/max_length": 512.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 467.1875, + "completions/min_terminated_length": 396.0, + "completions/max_terminated_length": 512.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028098180890083313, + "sampling/sampling_logp_difference/max": 0.4148428440093994, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8091086149215698, + "sampling/importance_sampling_ratio/max": 2.507693290710449, + "kl": 0.02999569766689092, + "entropy": 1.1012553870677948, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 32.705999514088035, + "epoch": 0.294, + "step": 147 + }, + { + "loss": -0.018139198422431946, + "grad_norm": 0.26061347126960754, + "learning_rate": 3e-05, + "num_tokens": 1535348.0, + "completions/mean_length": 467.0, + "completions/min_length": 385.0, + "completions/max_length": 536.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 467.0, + "completions/min_terminated_length": 385.0, + "completions/max_terminated_length": 536.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.6020797491073608, + "reward": 6.6875, + "reward_std": 0.6020797491073608, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028018539771437645, + "sampling/sampling_logp_difference/max": 0.35097575187683105, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6952720880508423, + "sampling/importance_sampling_ratio/max": 2.5010173320770264, + "kl": 0.0399768726201728, + "entropy": 1.2554677203297615, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 34.92355508869514, + "epoch": 0.296, + "step": 148 + }, + { + "loss": -0.01242863480001688, + "grad_norm": 0.2081325203180313, + "learning_rate": 3e-05, + "num_tokens": 1544912.0, + "completions/mean_length": 486.75, + "completions/min_length": 435.0, + "completions/max_length": 563.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 486.75, + "completions/min_terminated_length": 435.0, + "completions/max_terminated_length": 563.0, + "rewards/quality_reward/mean": 5.9375, + "rewards/quality_reward/std": 1.1814539432525635, + "reward": 5.9375, + "reward_std": 1.1814539432525635, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030079778283834457, + "sampling/sampling_logp_difference/max": 0.45126640796661377, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4945816695690155, + "sampling/importance_sampling_ratio/max": 1.0925099849700928, + "kl": 0.03278218396008015, + "entropy": 1.2563373371958733, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.784944237209857, + "epoch": 0.298, + "step": 149 + }, + { + "loss": -0.1471974402666092, + "grad_norm": 0.40016695857048035, + "learning_rate": 3e-05, + "num_tokens": 1554417.0, + "completions/mean_length": 483.0625, + "completions/min_length": 366.0, + "completions/max_length": 571.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 483.0625, + "completions/min_terminated_length": 366.0, + "completions/max_terminated_length": 571.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "reward": 7.0, + "reward_std": 0.632455587387085, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027187082916498184, + "sampling/sampling_logp_difference/max": 0.438828706741333, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8288668394088745, + "sampling/importance_sampling_ratio/max": 2.876359701156616, + "kl": 0.032314015785232186, + "entropy": 1.1289120316505432, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.687996607273817, + "epoch": 0.3, + "step": 150 + }, + { + "loss": -0.3493230938911438, + "grad_norm": 0.3958420157432556, + "learning_rate": 3e-05, + "num_tokens": 1564101.0, + "completions/mean_length": 504.25, + "completions/min_length": 436.0, + "completions/max_length": 618.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 504.25, + "completions/min_terminated_length": 436.0, + "completions/max_terminated_length": 618.0, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 0.9105859398841858, + "reward": 5.8125, + "reward_std": 0.9105859398841858, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02742597460746765, + "sampling/sampling_logp_difference/max": 0.48229074478149414, + "sampling/importance_sampling_ratio/min": 0.06443088501691818, + "sampling/importance_sampling_ratio/mean": 0.7897872924804688, + "sampling/importance_sampling_ratio/max": 2.8077433109283447, + "kl": 0.02723738143686205, + "entropy": 1.1718142479658127, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 34.508475522045046, + "epoch": 0.302, + "step": 151 + }, + { + "loss": -0.2918842136859894, + "grad_norm": 0.1944788247346878, + "learning_rate": 3e-05, + "num_tokens": 1570429.0, + "completions/mean_length": 296.0, + "completions/min_length": 106.0, + "completions/max_length": 519.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 296.0, + "completions/min_terminated_length": 106.0, + "completions/max_terminated_length": 519.0, + "rewards/quality_reward/mean": 3.4375, + "rewards/quality_reward/std": 3.558440685272217, + "reward": 3.4375, + "reward_std": 3.558440685272217, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02710951678454876, + "sampling/sampling_logp_difference/max": 0.4975461959838867, + "sampling/importance_sampling_ratio/min": 0.1504185050725937, + "sampling/importance_sampling_ratio/mean": 1.0381181240081787, + "sampling/importance_sampling_ratio/max": 1.85885751247406, + "kl": 0.030508540105074644, + "entropy": 1.0032543204724789, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.075861463323236, + "epoch": 0.304, + "step": 152 + }, + { + "loss": 0.33224302530288696, + "grad_norm": 0.4307408928871155, + "learning_rate": 3e-05, + "num_tokens": 1580372.0, + "completions/mean_length": 514.4375, + "completions/min_length": 459.0, + "completions/max_length": 610.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 514.4375, + "completions/min_terminated_length": 459.0, + "completions/max_terminated_length": 610.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027627088129520416, + "sampling/sampling_logp_difference/max": 0.48117589950561523, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8224437832832336, + "sampling/importance_sampling_ratio/max": 2.5702013969421387, + "kl": 0.02628148818621412, + "entropy": 1.2780758067965508, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.07258096849546, + "epoch": 0.306, + "step": 153 + }, + { + "loss": 0.05762449651956558, + "grad_norm": 0.1959461271762848, + "learning_rate": 3e-05, + "num_tokens": 1589438.0, + "completions/mean_length": 467.125, + "completions/min_length": 428.0, + "completions/max_length": 529.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 467.125, + "completions/min_terminated_length": 428.0, + "completions/max_terminated_length": 529.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "reward": 7.0625, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02732112817466259, + "sampling/sampling_logp_difference/max": 0.43965983390808105, + "sampling/importance_sampling_ratio/min": 0.11490790545940399, + "sampling/importance_sampling_ratio/mean": 0.7094592452049255, + "sampling/importance_sampling_ratio/max": 2.2318003177642822, + "kl": 0.026262085768394172, + "entropy": 1.161174800246954, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.3326059714891, + "epoch": 0.308, + "step": 154 + }, + { + "loss": -0.19043315947055817, + "grad_norm": 0.4554482400417328, + "learning_rate": 3e-05, + "num_tokens": 1597588.0, + "completions/mean_length": 396.875, + "completions/min_length": 294.0, + "completions/max_length": 503.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 396.875, + "completions/min_terminated_length": 294.0, + "completions/max_terminated_length": 503.0, + "rewards/quality_reward/mean": 5.5625, + "rewards/quality_reward/std": 1.3149778842926025, + "reward": 5.5625, + "reward_std": 1.3149778842926025, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029761571437120438, + "sampling/sampling_logp_difference/max": 0.5363807678222656, + "sampling/importance_sampling_ratio/min": 0.10123267024755478, + "sampling/importance_sampling_ratio/mean": 0.7672010064125061, + "sampling/importance_sampling_ratio/max": 2.1980347633361816, + "kl": 0.03402461623772979, + "entropy": 1.1771309785544872, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.508908156771213, + "epoch": 0.31, + "step": 155 + }, + { + "loss": 0.07504862546920776, + "grad_norm": 0.20946863293647766, + "learning_rate": 3e-05, + "num_tokens": 1607508.0, + "completions/mean_length": 520.5, + "completions/min_length": 422.0, + "completions/max_length": 673.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 520.5, + "completions/min_terminated_length": 422.0, + "completions/max_terminated_length": 673.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028051164001226425, + "sampling/sampling_logp_difference/max": 0.40722954273223877, + "sampling/importance_sampling_ratio/min": 0.016474967822432518, + "sampling/importance_sampling_ratio/mean": 0.631747305393219, + "sampling/importance_sampling_ratio/max": 1.429632306098938, + "kl": 0.02810170315206051, + "entropy": 1.123583823442459, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.622006124351174, + "epoch": 0.312, + "step": 156 + }, + { + "loss": 0.29842275381088257, + "grad_norm": 0.3838019073009491, + "learning_rate": 3e-05, + "num_tokens": 1617121.0, + "completions/mean_length": 471.8125, + "completions/min_length": 374.0, + "completions/max_length": 623.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 471.8125, + "completions/min_terminated_length": 374.0, + "completions/max_terminated_length": 623.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.5773502588272095, + "reward": 6.75, + "reward_std": 0.5773502588272095, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02675374038517475, + "sampling/sampling_logp_difference/max": 0.48647427558898926, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5510131120681763, + "sampling/importance_sampling_ratio/max": 2.64119291305542, + "kl": 0.029524097801186144, + "entropy": 1.0345656536519527, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 34.35223956173286, + "epoch": 0.314, + "step": 157 + }, + { + "loss": 0.07440078258514404, + "grad_norm": 0.14107273519039154, + "learning_rate": 3e-05, + "num_tokens": 1626462.0, + "completions/mean_length": 479.3125, + "completions/min_length": 415.0, + "completions/max_length": 536.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 479.3125, + "completions/min_terminated_length": 415.0, + "completions/max_terminated_length": 536.0, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.8341662883758545, + "reward": 6.1875, + "reward_std": 0.8341662883758545, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02727513015270233, + "sampling/sampling_logp_difference/max": 0.37659645080566406, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.515890896320343, + "sampling/importance_sampling_ratio/max": 1.719329833984375, + "kl": 0.02758750319480896, + "entropy": 1.197593629360199, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.49356387415901, + "epoch": 0.316, + "step": 158 + }, + { + "loss": 0.0027256053872406483, + "grad_norm": 0.22129040956497192, + "learning_rate": 3e-05, + "num_tokens": 1635613.0, + "completions/mean_length": 465.4375, + "completions/min_length": 392.0, + "completions/max_length": 567.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 465.4375, + "completions/min_terminated_length": 392.0, + "completions/max_terminated_length": 567.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 6.4375, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027347734197974205, + "sampling/sampling_logp_difference/max": 0.3532288074493408, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4655284583568573, + "sampling/importance_sampling_ratio/max": 2.2551939487457275, + "kl": 0.02928700065240264, + "entropy": 1.182745985686779, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.563114238902926, + "epoch": 0.318, + "step": 159 + }, + { + "loss": -0.1119004413485527, + "grad_norm": 0.22958238422870636, + "learning_rate": 3e-05, + "num_tokens": 1645060.0, + "completions/mean_length": 480.9375, + "completions/min_length": 378.0, + "completions/max_length": 618.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 480.9375, + "completions/min_terminated_length": 378.0, + "completions/max_terminated_length": 618.0, + "rewards/quality_reward/mean": 5.8125, + "rewards/quality_reward/std": 2.9033026695251465, + "reward": 5.8125, + "reward_std": 2.9033026695251465, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02425791323184967, + "sampling/sampling_logp_difference/max": 0.3485531806945801, + "sampling/importance_sampling_ratio/min": 0.16755303740501404, + "sampling/importance_sampling_ratio/mean": 0.6600984334945679, + "sampling/importance_sampling_ratio/max": 2.3590943813323975, + "kl": 0.025641236337833107, + "entropy": 1.0412059053778648, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.564059282653034, + "epoch": 0.32, + "step": 160 + }, + { + "loss": -0.24258574843406677, + "grad_norm": 0.2202390879392624, + "learning_rate": 3e-05, + "num_tokens": 1654901.0, + "completions/mean_length": 511.5625, + "completions/min_length": 440.0, + "completions/max_length": 625.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 511.5625, + "completions/min_terminated_length": 440.0, + "completions/max_terminated_length": 625.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.9660918116569519, + "reward": 6.5, + "reward_std": 0.9660918116569519, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02693682350218296, + "sampling/sampling_logp_difference/max": 0.37401676177978516, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8924014568328857, + "sampling/importance_sampling_ratio/max": 2.616337776184082, + "kl": 0.03178418125025928, + "entropy": 1.0806752033531666, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.71096785319969, + "epoch": 0.322, + "step": 161 + }, + { + "loss": -0.0895138755440712, + "grad_norm": 0.2978271245956421, + "learning_rate": 3e-05, + "num_tokens": 1664568.0, + "completions/mean_length": 484.1875, + "completions/min_length": 431.0, + "completions/max_length": 590.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 484.1875, + "completions/min_terminated_length": 431.0, + "completions/max_terminated_length": 590.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02701719105243683, + "sampling/sampling_logp_difference/max": 0.3564906120300293, + "sampling/importance_sampling_ratio/min": 0.20267778635025024, + "sampling/importance_sampling_ratio/mean": 0.9070306420326233, + "sampling/importance_sampling_ratio/max": 2.0902533531188965, + "kl": 0.026724245748482645, + "entropy": 1.2182030156254768, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.52080715773627, + "epoch": 0.324, + "step": 162 + }, + { + "loss": 0.016086462885141373, + "grad_norm": 0.13530702888965607, + "learning_rate": 3e-05, + "num_tokens": 1674423.0, + "completions/mean_length": 506.9375, + "completions/min_length": 452.0, + "completions/max_length": 543.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 506.9375, + "completions/min_terminated_length": 452.0, + "completions/max_terminated_length": 543.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.5123475790023804, + "reward": 6.5625, + "reward_std": 0.5123475790023804, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028113799169659615, + "sampling/sampling_logp_difference/max": 1.1959445476531982, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.583191990852356, + "sampling/importance_sampling_ratio/max": 2.1556015014648438, + "kl": 0.03180140187032521, + "entropy": 1.2944460734724998, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.659780140966177, + "epoch": 0.326, + "step": 163 + }, + { + "loss": 0.2851857542991638, + "grad_norm": 0.5087841153144836, + "learning_rate": 3e-05, + "num_tokens": 1683864.0, + "completions/mean_length": 490.5625, + "completions/min_length": 447.0, + "completions/max_length": 525.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 490.5625, + "completions/min_terminated_length": 447.0, + "completions/max_terminated_length": 525.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.0327955484390259, + "reward": 6.5, + "reward_std": 1.0327955484390259, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028977636247873306, + "sampling/sampling_logp_difference/max": 0.38115930557250977, + "sampling/importance_sampling_ratio/min": 0.09362189471721649, + "sampling/importance_sampling_ratio/mean": 0.8939677476882935, + "sampling/importance_sampling_ratio/max": 2.531486988067627, + "kl": 0.026518055819906294, + "entropy": 1.289131723344326, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.67840038659051, + "epoch": 0.328, + "step": 164 + }, + { + "loss": 0.5024052858352661, + "grad_norm": 0.5895075798034668, + "learning_rate": 3e-05, + "num_tokens": 1693592.0, + "completions/mean_length": 508.5, + "completions/min_length": 465.0, + "completions/max_length": 570.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 508.5, + "completions/min_terminated_length": 465.0, + "completions/max_terminated_length": 570.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027791393920779228, + "sampling/sampling_logp_difference/max": 0.25212621688842773, + "sampling/importance_sampling_ratio/min": 0.13951139152050018, + "sampling/importance_sampling_ratio/mean": 0.8359720706939697, + "sampling/importance_sampling_ratio/max": 2.5420279502868652, + "kl": 0.03340678755193949, + "entropy": 1.2583990097045898, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.054075544700027, + "epoch": 0.33, + "step": 165 + }, + { + "loss": 0.43730953335762024, + "grad_norm": 0.4107528030872345, + "learning_rate": 3e-05, + "num_tokens": 1703722.0, + "completions/mean_length": 522.625, + "completions/min_length": 445.0, + "completions/max_length": 573.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 522.625, + "completions/min_terminated_length": 445.0, + "completions/max_terminated_length": 573.0, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.5773502588272095, + "reward": 7.25, + "reward_std": 0.5773502588272095, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028407979756593704, + "sampling/sampling_logp_difference/max": 0.33945512771606445, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5325981378555298, + "sampling/importance_sampling_ratio/max": 1.8613929748535156, + "kl": 0.028830039431340992, + "entropy": 1.2723611742258072, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.466566514223814, + "epoch": 0.332, + "step": 166 + }, + { + "loss": -0.17192532122135162, + "grad_norm": 0.23576849699020386, + "learning_rate": 3e-05, + "num_tokens": 1713129.0, + "completions/mean_length": 485.4375, + "completions/min_length": 430.0, + "completions/max_length": 544.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 485.4375, + "completions/min_terminated_length": 430.0, + "completions/max_terminated_length": 544.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.8539125919342041, + "reward": 6.9375, + "reward_std": 0.8539125919342041, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027760744094848633, + "sampling/sampling_logp_difference/max": 0.4001603126525879, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.835480809211731, + "sampling/importance_sampling_ratio/max": 2.364237070083618, + "kl": 0.03369407169520855, + "entropy": 1.159641794860363, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.427942908834666, + "epoch": 0.334, + "step": 167 + }, + { + "loss": -0.10911832749843597, + "grad_norm": 0.14523518085479736, + "learning_rate": 3e-05, + "num_tokens": 1723371.0, + "completions/mean_length": 531.125, + "completions/min_length": 439.0, + "completions/max_length": 653.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 531.125, + "completions/min_terminated_length": 439.0, + "completions/max_terminated_length": 653.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026400625705718994, + "sampling/sampling_logp_difference/max": 0.42547130584716797, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5449534058570862, + "sampling/importance_sampling_ratio/max": 2.233501672744751, + "kl": 0.032465216936543584, + "entropy": 1.1074568964540958, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.74156094249338, + "epoch": 0.336, + "step": 168 + }, + { + "loss": 0.14656513929367065, + "grad_norm": 0.3119359314441681, + "learning_rate": 3e-05, + "num_tokens": 1733279.0, + "completions/mean_length": 518.25, + "completions/min_length": 447.0, + "completions/max_length": 629.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 518.25, + "completions/min_terminated_length": 447.0, + "completions/max_terminated_length": 629.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027896685525774956, + "sampling/sampling_logp_difference/max": 0.34301328659057617, + "sampling/importance_sampling_ratio/min": 0.061056625097990036, + "sampling/importance_sampling_ratio/mean": 0.8524343967437744, + "sampling/importance_sampling_ratio/max": 2.882887601852417, + "kl": 0.03448521322570741, + "entropy": 1.2571639120578766, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.03652939805761, + "epoch": 0.338, + "step": 169 + }, + { + "loss": -0.11407067626714706, + "grad_norm": 0.23340070247650146, + "learning_rate": 3e-05, + "num_tokens": 1743740.0, + "completions/mean_length": 552.8125, + "completions/min_length": 454.0, + "completions/max_length": 637.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 552.8125, + "completions/min_terminated_length": 454.0, + "completions/max_terminated_length": 637.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.6551081538200378, + "reward": 6.8125, + "reward_std": 0.6551081538200378, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03021315485239029, + "sampling/sampling_logp_difference/max": 0.314577579498291, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5497220754623413, + "sampling/importance_sampling_ratio/max": 1.6628351211547852, + "kl": 0.03460722556337714, + "entropy": 1.2771715074777603, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.62061845092103, + "epoch": 0.34, + "step": 170 + }, + { + "loss": -0.012545892037451267, + "grad_norm": 0.07674646377563477, + "learning_rate": 3e-05, + "num_tokens": 1753231.0, + "completions/mean_length": 495.1875, + "completions/min_length": 434.0, + "completions/max_length": 564.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 495.1875, + "completions/min_terminated_length": 434.0, + "completions/max_terminated_length": 564.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.7274384498596191, + "reward": 6.4375, + "reward_std": 0.7274384498596191, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027837729081511497, + "sampling/sampling_logp_difference/max": 0.5454483032226562, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4176323413848877, + "sampling/importance_sampling_ratio/max": 1.37643563747406, + "kl": 0.0316173325991258, + "entropy": 1.1425480283796787, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 35.69278695946559, + "epoch": 0.342, + "step": 171 + }, + { + "loss": 0.2167063057422638, + "grad_norm": 0.5406383275985718, + "learning_rate": 3e-05, + "num_tokens": 1763511.0, + "completions/mean_length": 536.5, + "completions/min_length": 475.0, + "completions/max_length": 614.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 536.5, + "completions/min_terminated_length": 475.0, + "completions/max_terminated_length": 614.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "reward": 6.875, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028938323259353638, + "sampling/sampling_logp_difference/max": 0.5733523368835449, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 1.1003804206848145, + "sampling/importance_sampling_ratio/max": 2.74458646774292, + "kl": 0.03873496490996331, + "entropy": 1.3311709240078926, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.092736863531172, + "epoch": 0.344, + "step": 172 + }, + { + "loss": 0.07203174382448196, + "grad_norm": 0.11082098633050919, + "learning_rate": 3e-05, + "num_tokens": 1773304.0, + "completions/mean_length": 508.0625, + "completions/min_length": 437.0, + "completions/max_length": 614.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 508.0625, + "completions/min_terminated_length": 437.0, + "completions/max_terminated_length": 614.0, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.5, + "reward": 7.125, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028249088674783707, + "sampling/sampling_logp_difference/max": 0.595893383026123, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5755537748336792, + "sampling/importance_sampling_ratio/max": 1.8160909414291382, + "kl": 0.03200511261820793, + "entropy": 1.22428647428751, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 36.1855756030418, + "epoch": 0.346, + "step": 173 + }, + { + "loss": 0.2996499538421631, + "grad_norm": 0.3865050971508026, + "learning_rate": 3e-05, + "num_tokens": 1784039.0, + "completions/mean_length": 556.9375, + "completions/min_length": 487.0, + "completions/max_length": 720.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 556.9375, + "completions/min_terminated_length": 487.0, + "completions/max_terminated_length": 720.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027585921809077263, + "sampling/sampling_logp_difference/max": 0.41143274307250977, + "sampling/importance_sampling_ratio/min": 0.09306051582098007, + "sampling/importance_sampling_ratio/mean": 0.5800511240959167, + "sampling/importance_sampling_ratio/max": 2.0816619396209717, + "kl": 0.03591357555706054, + "entropy": 1.1718761064112186, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.269209342077374, + "epoch": 0.348, + "step": 174 + }, + { + "loss": 0.07849398255348206, + "grad_norm": 0.11888998746871948, + "learning_rate": 3e-05, + "num_tokens": 1793682.0, + "completions/mean_length": 494.6875, + "completions/min_length": 442.0, + "completions/max_length": 596.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 494.6875, + "completions/min_terminated_length": 442.0, + "completions/max_terminated_length": 596.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "reward": 6.875, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.031242625787854195, + "sampling/sampling_logp_difference/max": 0.4447822570800781, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4274219870567322, + "sampling/importance_sampling_ratio/max": 1.3145290613174438, + "kl": 0.03842530632391572, + "entropy": 1.3337711468338966, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.293509372044355, + "epoch": 0.35, + "step": 175 + }, + { + "loss": 0.11461115628480911, + "grad_norm": 0.2991865277290344, + "learning_rate": 3e-05, + "num_tokens": 1804039.0, + "completions/mean_length": 544.8125, + "completions/min_length": 462.0, + "completions/max_length": 706.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 544.8125, + "completions/min_terminated_length": 462.0, + "completions/max_terminated_length": 706.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8850612044334412, + "reward": 6.625, + "reward_std": 0.8850612044334412, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026338135823607445, + "sampling/sampling_logp_difference/max": 0.3649592399597168, + "sampling/importance_sampling_ratio/min": 0.07339605689048767, + "sampling/importance_sampling_ratio/mean": 0.5072454810142517, + "sampling/importance_sampling_ratio/max": 1.5985729694366455, + "kl": 0.036185722798109055, + "entropy": 1.19626072794199, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 40.05168053135276, + "epoch": 0.352, + "step": 176 + }, + { + "loss": 0.11777876317501068, + "grad_norm": 0.21207627654075623, + "learning_rate": 3e-05, + "num_tokens": 1813440.0, + "completions/mean_length": 488.0625, + "completions/min_length": 419.0, + "completions/max_length": 546.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 488.0625, + "completions/min_terminated_length": 419.0, + "completions/max_terminated_length": 546.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.5163977742195129, + "reward": 7.0, + "reward_std": 0.5163977742195129, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027743803337216377, + "sampling/sampling_logp_difference/max": 0.35143423080444336, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6533275246620178, + "sampling/importance_sampling_ratio/max": 2.3352530002593994, + "kl": 0.036609378294087946, + "entropy": 1.2391329184174538, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 33.59382761735469, + "epoch": 0.354, + "step": 177 + }, + { + "loss": -0.20823253691196442, + "grad_norm": 0.2076549232006073, + "learning_rate": 3e-05, + "num_tokens": 1823018.0, + "completions/mean_length": 495.125, + "completions/min_length": 416.0, + "completions/max_length": 570.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 495.125, + "completions/min_terminated_length": 416.0, + "completions/max_terminated_length": 570.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 1.095445156097412, + "reward": 7.0, + "reward_std": 1.095445156097412, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02992408722639084, + "sampling/sampling_logp_difference/max": 0.3826625347137451, + "sampling/importance_sampling_ratio/min": 0.08218635618686676, + "sampling/importance_sampling_ratio/mean": 0.8220031261444092, + "sampling/importance_sampling_ratio/max": 2.9768388271331787, + "kl": 0.03601150680333376, + "entropy": 1.3244052603840828, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.288791641127318, + "epoch": 0.356, + "step": 178 + }, + { + "loss": 0.06042468547821045, + "grad_norm": 0.5341953039169312, + "learning_rate": 3e-05, + "num_tokens": 1832736.0, + "completions/mean_length": 497.875, + "completions/min_length": 445.0, + "completions/max_length": 567.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 497.875, + "completions/min_terminated_length": 445.0, + "completions/max_terminated_length": 567.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8944272398948669, + "reward": 6.5, + "reward_std": 0.8944272398948669, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029129499569535255, + "sampling/sampling_logp_difference/max": 0.5151598453521729, + "sampling/importance_sampling_ratio/min": 0.037978146225214005, + "sampling/importance_sampling_ratio/mean": 1.0708422660827637, + "sampling/importance_sampling_ratio/max": 2.1878349781036377, + "kl": 0.0400471081957221, + "entropy": 1.2238815650343895, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.113372664432973, + "epoch": 0.358, + "step": 179 + }, + { + "loss": -0.10235662013292313, + "grad_norm": 0.21081708371639252, + "learning_rate": 3e-05, + "num_tokens": 1843203.0, + "completions/mean_length": 541.6875, + "completions/min_length": 487.0, + "completions/max_length": 614.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 541.6875, + "completions/min_terminated_length": 487.0, + "completions/max_terminated_length": 614.0, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.981070876121521, + "reward": 6.1875, + "reward_std": 0.981070876121521, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02854372188448906, + "sampling/sampling_logp_difference/max": 0.31549549102783203, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4484874904155731, + "sampling/importance_sampling_ratio/max": 1.1632962226867676, + "kl": 0.03922082995995879, + "entropy": 1.4256544262170792, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 38.8069160906598, + "epoch": 0.36, + "step": 180 + }, + { + "loss": -0.1838480830192566, + "grad_norm": 0.28472819924354553, + "learning_rate": 3e-05, + "num_tokens": 1853555.0, + "completions/mean_length": 541.0, + "completions/min_length": 487.0, + "completions/max_length": 610.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 541.0, + "completions/min_terminated_length": 487.0, + "completions/max_terminated_length": 610.0, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 0.6551081538200378, + "reward": 6.1875, + "reward_std": 0.6551081538200378, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028904814273118973, + "sampling/sampling_logp_difference/max": 0.4628629684448242, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5436820983886719, + "sampling/importance_sampling_ratio/max": 1.6971478462219238, + "kl": 0.039078159374184906, + "entropy": 1.2127383947372437, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.927347922697663, + "epoch": 0.362, + "step": 181 + }, + { + "loss": 0.11645574867725372, + "grad_norm": 0.40580859780311584, + "learning_rate": 3e-05, + "num_tokens": 1863670.0, + "completions/mean_length": 521.1875, + "completions/min_length": 481.0, + "completions/max_length": 590.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 521.1875, + "completions/min_terminated_length": 481.0, + "completions/max_terminated_length": 590.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02727590501308441, + "sampling/sampling_logp_difference/max": 0.42972230911254883, + "sampling/importance_sampling_ratio/min": 0.0672435387969017, + "sampling/importance_sampling_ratio/mean": 0.7627977132797241, + "sampling/importance_sampling_ratio/max": 2.6977338790893555, + "kl": 0.03652556415181607, + "entropy": 1.205168604850769, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.11851307330653, + "epoch": 0.364, + "step": 182 + }, + { + "loss": 0.07254824787378311, + "grad_norm": 0.30436721444129944, + "learning_rate": 3e-05, + "num_tokens": 1873353.0, + "completions/mean_length": 501.6875, + "completions/min_length": 429.0, + "completions/max_length": 563.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 501.6875, + "completions/min_terminated_length": 429.0, + "completions/max_terminated_length": 563.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.5163977742195129, + "reward": 7.0, + "reward_std": 0.5163977742195129, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028735067695379257, + "sampling/sampling_logp_difference/max": 0.3192565441131592, + "sampling/importance_sampling_ratio/min": 0.05660989508032799, + "sampling/importance_sampling_ratio/mean": 0.6635246276855469, + "sampling/importance_sampling_ratio/max": 1.7811214923858643, + "kl": 0.04035243031103164, + "entropy": 1.2342532575130463, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.842315018642694, + "epoch": 0.366, + "step": 183 + }, + { + "loss": -0.020260833203792572, + "grad_norm": 0.15960462391376495, + "learning_rate": 3e-05, + "num_tokens": 1883907.0, + "completions/mean_length": 540.625, + "completions/min_length": 488.0, + "completions/max_length": 621.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 540.625, + "completions/min_terminated_length": 488.0, + "completions/max_terminated_length": 621.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029474109411239624, + "sampling/sampling_logp_difference/max": 0.9810755252838135, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6561183929443359, + "sampling/importance_sampling_ratio/max": 1.9560747146606445, + "kl": 0.04095173126552254, + "entropy": 1.3501724749803543, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.30614952277392, + "epoch": 0.368, + "step": 184 + }, + { + "loss": 0.07327698916196823, + "grad_norm": 0.19153976440429688, + "learning_rate": 3e-05, + "num_tokens": 1894262.0, + "completions/mean_length": 532.6875, + "completions/min_length": 434.0, + "completions/max_length": 594.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 532.6875, + "completions/min_terminated_length": 434.0, + "completions/max_terminated_length": 594.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02841799519956112, + "sampling/sampling_logp_difference/max": 0.378201961517334, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7404133677482605, + "sampling/importance_sampling_ratio/max": 2.3196122646331787, + "kl": 0.045232257107272744, + "entropy": 1.3397118523716927, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.66303364513442, + "epoch": 0.37, + "step": 185 + }, + { + "loss": -0.1904258131980896, + "grad_norm": 0.1921214759349823, + "learning_rate": 3e-05, + "num_tokens": 1904748.0, + "completions/mean_length": 547.875, + "completions/min_length": 498.0, + "completions/max_length": 626.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 547.875, + "completions/min_terminated_length": 498.0, + "completions/max_terminated_length": 626.0, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.0468206405639648, + "reward": 6.1875, + "reward_std": 1.0468206405639648, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030228231102228165, + "sampling/sampling_logp_difference/max": 0.43239259719848633, + "sampling/importance_sampling_ratio/min": 0.1473371982574463, + "sampling/importance_sampling_ratio/mean": 0.8759132027626038, + "sampling/importance_sampling_ratio/max": 2.1734814643859863, + "kl": 0.04523745132610202, + "entropy": 1.4480287805199623, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.321936973370612, + "epoch": 0.372, + "step": 186 + }, + { + "loss": -0.005613957531750202, + "grad_norm": 0.1446281522512436, + "learning_rate": 3e-05, + "num_tokens": 1915044.0, + "completions/mean_length": 522.5, + "completions/min_length": 464.0, + "completions/max_length": 595.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 522.5, + "completions/min_terminated_length": 464.0, + "completions/max_terminated_length": 595.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028958076611161232, + "sampling/sampling_logp_difference/max": 0.3580789566040039, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.49870407581329346, + "sampling/importance_sampling_ratio/max": 1.1428344249725342, + "kl": 0.04069687286391854, + "entropy": 1.1799098625779152, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.772348938975483, + "epoch": 0.374, + "step": 187 + }, + { + "loss": -0.09858276695013046, + "grad_norm": 0.09093533456325531, + "learning_rate": 3e-05, + "num_tokens": 1925055.0, + "completions/mean_length": 526.6875, + "completions/min_length": 428.0, + "completions/max_length": 590.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 526.6875, + "completions/min_terminated_length": 428.0, + "completions/max_terminated_length": 590.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02906900830566883, + "sampling/sampling_logp_difference/max": 0.455765962600708, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.47519102692604065, + "sampling/importance_sampling_ratio/max": 1.5353080034255981, + "kl": 0.048393386183306575, + "entropy": 1.1814791783690453, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.42380119021982, + "epoch": 0.376, + "step": 188 + }, + { + "loss": -0.16722381114959717, + "grad_norm": 0.20003275573253632, + "learning_rate": 3e-05, + "num_tokens": 1935415.0, + "completions/mean_length": 538.5, + "completions/min_length": 447.0, + "completions/max_length": 663.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 538.5, + "completions/min_terminated_length": 447.0, + "completions/max_terminated_length": 663.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028748787939548492, + "sampling/sampling_logp_difference/max": 0.4760274887084961, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.680183470249176, + "sampling/importance_sampling_ratio/max": 2.9816064834594727, + "kl": 0.04744360945187509, + "entropy": 1.2316770479083061, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.73181858472526, + "epoch": 0.378, + "step": 189 + }, + { + "loss": 0.04805057868361473, + "grad_norm": 0.13213400542736053, + "learning_rate": 3e-05, + "num_tokens": 1945985.0, + "completions/mean_length": 555.125, + "completions/min_length": 463.0, + "completions/max_length": 659.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 555.125, + "completions/min_terminated_length": 463.0, + "completions/max_terminated_length": 659.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.9574271440505981, + "reward": 6.625, + "reward_std": 0.9574271440505981, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0274125337600708, + "sampling/sampling_logp_difference/max": 0.3399984836578369, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.41058292984962463, + "sampling/importance_sampling_ratio/max": 1.563953161239624, + "kl": 0.04299823543988168, + "entropy": 1.198552466928959, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.826407154556364, + "epoch": 0.38, + "step": 190 + }, + { + "loss": -0.3787975311279297, + "grad_norm": 0.2910110354423523, + "learning_rate": 3e-05, + "num_tokens": 1956445.0, + "completions/mean_length": 539.25, + "completions/min_length": 469.0, + "completions/max_length": 684.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 539.25, + "completions/min_terminated_length": 469.0, + "completions/max_terminated_length": 684.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029267774894833565, + "sampling/sampling_logp_difference/max": 0.3755226135253906, + "sampling/importance_sampling_ratio/min": 0.1316290944814682, + "sampling/importance_sampling_ratio/mean": 0.8701735734939575, + "sampling/importance_sampling_ratio/max": 2.4134271144866943, + "kl": 0.046097030164673924, + "entropy": 1.179109387099743, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.135659996420145, + "epoch": 0.382, + "step": 191 + }, + { + "loss": 0.013828473165631294, + "grad_norm": 0.20911987125873566, + "learning_rate": 3e-05, + "num_tokens": 1966693.0, + "completions/mean_length": 540.0, + "completions/min_length": 496.0, + "completions/max_length": 615.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 540.0, + "completions/min_terminated_length": 496.0, + "completions/max_terminated_length": 615.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "reward": 6.9375, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02878478728234768, + "sampling/sampling_logp_difference/max": 0.4313013553619385, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5034524202346802, + "sampling/importance_sampling_ratio/max": 1.9282022714614868, + "kl": 0.04525213362649083, + "entropy": 1.2047618329524994, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.677200393751264, + "epoch": 0.384, + "step": 192 + }, + { + "loss": -0.06190801039338112, + "grad_norm": 0.3986797034740448, + "learning_rate": 3e-05, + "num_tokens": 1977311.0, + "completions/mean_length": 553.125, + "completions/min_length": 501.0, + "completions/max_length": 597.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 553.125, + "completions/min_terminated_length": 501.0, + "completions/max_terminated_length": 597.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.632455587387085, + "reward": 7.0, + "reward_std": 0.632455587387085, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02950127050280571, + "sampling/sampling_logp_difference/max": 0.7719376087188721, + "sampling/importance_sampling_ratio/min": 0.02624017745256424, + "sampling/importance_sampling_ratio/mean": 0.9780403971672058, + "sampling/importance_sampling_ratio/max": 2.877324104309082, + "kl": 0.04699963750317693, + "entropy": 1.1857876032590866, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.189579842612147, + "epoch": 0.386, + "step": 193 + }, + { + "loss": -0.15963155031204224, + "grad_norm": 0.22821271419525146, + "learning_rate": 3e-05, + "num_tokens": 1987680.0, + "completions/mean_length": 531.5625, + "completions/min_length": 465.0, + "completions/max_length": 590.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 531.5625, + "completions/min_terminated_length": 465.0, + "completions/max_terminated_length": 590.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.8164966106414795, + "reward": 6.5, + "reward_std": 0.8164966106414795, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029790451750159264, + "sampling/sampling_logp_difference/max": 0.45777153968811035, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7382668256759644, + "sampling/importance_sampling_ratio/max": 2.790942907333374, + "kl": 0.04149021068587899, + "entropy": 1.3334204703569412, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.228061076253653, + "epoch": 0.388, + "step": 194 + }, + { + "loss": -0.2362610548734665, + "grad_norm": 0.3617555797100067, + "learning_rate": 3e-05, + "num_tokens": 1997481.0, + "completions/mean_length": 505.0625, + "completions/min_length": 455.0, + "completions/max_length": 575.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 505.0625, + "completions/min_terminated_length": 455.0, + "completions/max_terminated_length": 575.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.7187952995300293, + "reward": 6.625, + "reward_std": 0.7187952995300293, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029685894027352333, + "sampling/sampling_logp_difference/max": 0.6245463490486145, + "sampling/importance_sampling_ratio/min": 0.09889467060565948, + "sampling/importance_sampling_ratio/mean": 0.9099248647689819, + "sampling/importance_sampling_ratio/max": 2.3454530239105225, + "kl": 0.042452862951904535, + "entropy": 1.2570307329297066, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.880568680819124, + "epoch": 0.39, + "step": 195 + }, + { + "loss": 0.026877164840698242, + "grad_norm": 0.35804808139801025, + "learning_rate": 3e-05, + "num_tokens": 2007860.0, + "completions/mean_length": 544.6875, + "completions/min_length": 458.0, + "completions/max_length": 631.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 544.6875, + "completions/min_terminated_length": 458.0, + "completions/max_terminated_length": 631.0, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02899312786757946, + "sampling/sampling_logp_difference/max": 0.4234185218811035, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6493271589279175, + "sampling/importance_sampling_ratio/max": 2.670585870742798, + "kl": 0.03823408088646829, + "entropy": 1.208221659064293, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.456977635622025, + "epoch": 0.392, + "step": 196 + }, + { + "loss": -0.019679736346006393, + "grad_norm": 0.05589874088764191, + "learning_rate": 3e-05, + "num_tokens": 2018946.0, + "completions/mean_length": 582.875, + "completions/min_length": 517.0, + "completions/max_length": 676.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 582.875, + "completions/min_terminated_length": 517.0, + "completions/max_terminated_length": 676.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029303058981895447, + "sampling/sampling_logp_difference/max": 0.33884429931640625, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.35759687423706055, + "sampling/importance_sampling_ratio/max": 1.1570472717285156, + "kl": 0.03486072865780443, + "entropy": 1.3732100576162338, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.522779263556004, + "epoch": 0.394, + "step": 197 + }, + { + "loss": 0.0447416789829731, + "grad_norm": 0.23638494312763214, + "learning_rate": 3e-05, + "num_tokens": 2029178.0, + "completions/mean_length": 533.5, + "completions/min_length": 479.0, + "completions/max_length": 625.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 533.5, + "completions/min_terminated_length": 479.0, + "completions/max_terminated_length": 625.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029459550976753235, + "sampling/sampling_logp_difference/max": 0.37561988830566406, + "sampling/importance_sampling_ratio/min": 0.09831889718770981, + "sampling/importance_sampling_ratio/mean": 0.539449155330658, + "sampling/importance_sampling_ratio/max": 1.484117865562439, + "kl": 0.037777561345137656, + "entropy": 1.1664377599954605, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.065524043980986, + "epoch": 0.396, + "step": 198 + }, + { + "loss": -0.09766081720590591, + "grad_norm": 0.1923029124736786, + "learning_rate": 3e-05, + "num_tokens": 2039347.0, + "completions/mean_length": 536.0625, + "completions/min_length": 472.0, + "completions/max_length": 661.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 536.0625, + "completions/min_terminated_length": 472.0, + "completions/max_terminated_length": 661.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "reward": 6.8125, + "reward_std": 0.75, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027488065883517265, + "sampling/sampling_logp_difference/max": 0.35316991806030273, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5547572374343872, + "sampling/importance_sampling_ratio/max": 2.6362762451171875, + "kl": 0.03572200902272016, + "entropy": 1.254080481827259, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.96936017088592, + "epoch": 0.398, + "step": 199 + }, + { + "loss": 0.014221633784472942, + "grad_norm": 0.22953365743160248, + "learning_rate": 3e-05, + "num_tokens": 2049795.0, + "completions/mean_length": 551.5, + "completions/min_length": 505.0, + "completions/max_length": 610.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 551.5, + "completions/min_terminated_length": 505.0, + "completions/max_terminated_length": 610.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8920949101448059, + "reward": 6.5625, + "reward_std": 0.8920949101448059, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0300765261054039, + "sampling/sampling_logp_difference/max": 0.7396450042724609, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6678089499473572, + "sampling/importance_sampling_ratio/max": 2.4539027214050293, + "kl": 0.036497756373137236, + "entropy": 1.2724409252405167, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.165886579081416, + "epoch": 0.4, + "step": 200 + }, + { + "loss": -0.13335926830768585, + "grad_norm": 0.0991974025964737, + "learning_rate": 3e-05, + "num_tokens": 2059963.0, + "completions/mean_length": 528.0, + "completions/min_length": 474.0, + "completions/max_length": 582.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 528.0, + "completions/min_terminated_length": 474.0, + "completions/max_terminated_length": 582.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.7841898202896118, + "reward": 6.375, + "reward_std": 1.7841898202896118, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027531839907169342, + "sampling/sampling_logp_difference/max": 0.5111579895019531, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6451135277748108, + "sampling/importance_sampling_ratio/max": 2.3520584106445312, + "kl": 0.03168696933425963, + "entropy": 1.120336215943098, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 38.76227374607697, + "epoch": 0.402, + "step": 201 + }, + { + "loss": 0.04318992793560028, + "grad_norm": 0.07962016016244888, + "learning_rate": 3e-05, + "num_tokens": 2070626.0, + "completions/mean_length": 549.9375, + "completions/min_length": 471.0, + "completions/max_length": 635.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 549.9375, + "completions/min_terminated_length": 471.0, + "completions/max_terminated_length": 635.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.0287276990711689, + "sampling/sampling_logp_difference/max": 0.42658185958862305, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.265199214220047, + "sampling/importance_sampling_ratio/max": 2.161137580871582, + "kl": 0.040914483717642725, + "entropy": 1.2026560828089714, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.940971142146736, + "epoch": 0.404, + "step": 202 + }, + { + "loss": -0.14832699298858643, + "grad_norm": 0.1376999020576477, + "learning_rate": 3e-05, + "num_tokens": 2080861.0, + "completions/mean_length": 511.1875, + "completions/min_length": 460.0, + "completions/max_length": 566.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 511.1875, + "completions/min_terminated_length": 460.0, + "completions/max_terminated_length": 566.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02949603646993637, + "sampling/sampling_logp_difference/max": 0.3160414695739746, + "sampling/importance_sampling_ratio/min": 0.18541352450847626, + "sampling/importance_sampling_ratio/mean": 0.7174543142318726, + "sampling/importance_sampling_ratio/max": 2.544222831726074, + "kl": 0.03862898051738739, + "entropy": 1.2832268327474594, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.54771270370111, + "epoch": 0.406, + "step": 203 + }, + { + "loss": -0.010008644312620163, + "grad_norm": 0.12836101651191711, + "learning_rate": 3e-05, + "num_tokens": 2090953.0, + "completions/mean_length": 524.25, + "completions/min_length": 445.0, + "completions/max_length": 605.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 524.25, + "completions/min_terminated_length": 445.0, + "completions/max_terminated_length": 605.0, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 7.125, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028712285682559013, + "sampling/sampling_logp_difference/max": 0.29064249992370605, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5054515600204468, + "sampling/importance_sampling_ratio/max": 2.204956531524658, + "kl": 0.032396848779171705, + "entropy": 1.2392274662852287, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.479188771452755, + "epoch": 0.408, + "step": 204 + }, + { + "loss": 0.053824737668037415, + "grad_norm": 0.2566336691379547, + "learning_rate": 3e-05, + "num_tokens": 2101795.0, + "completions/mean_length": 566.625, + "completions/min_length": 482.0, + "completions/max_length": 708.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 566.625, + "completions/min_terminated_length": 482.0, + "completions/max_terminated_length": 708.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027578134089708328, + "sampling/sampling_logp_difference/max": 0.4542531967163086, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6769458651542664, + "sampling/importance_sampling_ratio/max": 2.2913551330566406, + "kl": 0.03499211696907878, + "entropy": 1.3167504370212555, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.109014553949237, + "epoch": 0.41, + "step": 205 + }, + { + "loss": -0.2610609829425812, + "grad_norm": 0.3224847912788391, + "learning_rate": 3e-05, + "num_tokens": 2111719.0, + "completions/mean_length": 506.25, + "completions/min_length": 443.0, + "completions/max_length": 588.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 506.25, + "completions/min_terminated_length": 443.0, + "completions/max_terminated_length": 588.0, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.7187952995300293, + "reward": 7.125, + "reward_std": 0.7187952995300293, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02760414592921734, + "sampling/sampling_logp_difference/max": 0.5845310688018799, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5962464809417725, + "sampling/importance_sampling_ratio/max": 2.0974948406219482, + "kl": 0.03367950115352869, + "entropy": 1.244155466556549, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 12.972559538204223, + "epoch": 0.412, + "step": 206 + }, + { + "loss": 0.08153954148292542, + "grad_norm": 0.22504572570323944, + "learning_rate": 3e-05, + "num_tokens": 2121913.0, + "completions/mean_length": 515.125, + "completions/min_length": 431.0, + "completions/max_length": 623.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 515.125, + "completions/min_terminated_length": 431.0, + "completions/max_terminated_length": 623.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029437636956572533, + "sampling/sampling_logp_difference/max": 0.3475990295410156, + "sampling/importance_sampling_ratio/min": 0.16781684756278992, + "sampling/importance_sampling_ratio/mean": 0.4843714237213135, + "sampling/importance_sampling_ratio/max": 1.6359574794769287, + "kl": 0.039928025915287435, + "entropy": 1.2021623700857162, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 43.64637131197378, + "epoch": 0.414, + "step": 207 + }, + { + "loss": 0.34191110730171204, + "grad_norm": 0.3592093884944916, + "learning_rate": 3e-05, + "num_tokens": 2131806.0, + "completions/mean_length": 512.3125, + "completions/min_length": 421.0, + "completions/max_length": 613.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 512.3125, + "completions/min_terminated_length": 421.0, + "completions/max_terminated_length": 613.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.4787135720252991, + "reward": 6.6875, + "reward_std": 0.4787135720252991, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027714602649211884, + "sampling/sampling_logp_difference/max": 0.3398885726928711, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6557403206825256, + "sampling/importance_sampling_ratio/max": 2.192653179168701, + "kl": 0.036205250886268914, + "entropy": 1.2001312859356403, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.901265816297382, + "epoch": 0.416, + "step": 208 + }, + { + "loss": 0.2796367406845093, + "grad_norm": 0.47909337282180786, + "learning_rate": 3e-05, + "num_tokens": 2141849.0, + "completions/mean_length": 513.1875, + "completions/min_length": 429.0, + "completions/max_length": 588.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 513.1875, + "completions/min_terminated_length": 429.0, + "completions/max_terminated_length": 588.0, + "rewards/quality_reward/mean": 6.1875, + "rewards/quality_reward/std": 1.0468206405639648, + "reward": 6.1875, + "reward_std": 1.0468206405639648, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029922205954790115, + "sampling/sampling_logp_difference/max": 0.3722946047782898, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.48602545261383057, + "sampling/importance_sampling_ratio/max": 2.964437484741211, + "kl": 0.02966410096269101, + "entropy": 1.2848069965839386, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.673682311549783, + "epoch": 0.418, + "step": 209 + }, + { + "loss": 0.07772707939147949, + "grad_norm": 0.3124609887599945, + "learning_rate": 3e-05, + "num_tokens": 2151864.0, + "completions/mean_length": 507.9375, + "completions/min_length": 422.0, + "completions/max_length": 574.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 507.9375, + "completions/min_terminated_length": 422.0, + "completions/max_terminated_length": 574.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027899259701371193, + "sampling/sampling_logp_difference/max": 0.36536455154418945, + "sampling/importance_sampling_ratio/min": 0.052471309900283813, + "sampling/importance_sampling_ratio/mean": 0.9012110233306885, + "sampling/importance_sampling_ratio/max": 2.737311840057373, + "kl": 0.038097753771580756, + "entropy": 1.2064250856637955, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.180102336220443, + "epoch": 0.42, + "step": 210 + }, + { + "loss": 0.28776273131370544, + "grad_norm": 0.44815266132354736, + "learning_rate": 3e-05, + "num_tokens": 2161039.0, + "completions/mean_length": 480.9375, + "completions/min_length": 436.0, + "completions/max_length": 525.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 480.9375, + "completions/min_terminated_length": 436.0, + "completions/max_terminated_length": 525.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02677021361887455, + "sampling/sampling_logp_difference/max": 0.32479190826416016, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8898900151252747, + "sampling/importance_sampling_ratio/max": 2.063380479812622, + "kl": 0.027290108264423907, + "entropy": 1.2063737213611603, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.177836938295513, + "epoch": 0.422, + "step": 211 + }, + { + "loss": 0.17483392357826233, + "grad_norm": 0.35702478885650635, + "learning_rate": 3e-05, + "num_tokens": 2170961.0, + "completions/mean_length": 504.625, + "completions/min_length": 431.0, + "completions/max_length": 599.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 504.625, + "completions/min_terminated_length": 431.0, + "completions/max_terminated_length": 599.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027991417795419693, + "sampling/sampling_logp_difference/max": 0.35749173164367676, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7680925130844116, + "sampling/importance_sampling_ratio/max": 2.4756224155426025, + "kl": 0.031013125786557794, + "entropy": 1.3087149783968925, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.408967671450227, + "epoch": 0.424, + "step": 212 + }, + { + "loss": 0.09740053862333298, + "grad_norm": 0.29086264967918396, + "learning_rate": 3e-05, + "num_tokens": 2182910.0, + "completions/mean_length": 640.8125, + "completions/min_length": 457.0, + "completions/max_length": 797.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 640.8125, + "completions/min_terminated_length": 457.0, + "completions/max_terminated_length": 797.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "reward": 6.9375, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028743820264935493, + "sampling/sampling_logp_difference/max": 0.3868746757507324, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7117372155189514, + "sampling/importance_sampling_ratio/max": 1.893927812576294, + "kl": 0.024261576938442886, + "entropy": 1.3978670835494995, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.464720248244703, + "epoch": 0.426, + "step": 213 + }, + { + "loss": 0.18954241275787354, + "grad_norm": 0.2767268419265747, + "learning_rate": 3e-05, + "num_tokens": 2192242.0, + "completions/mean_length": 486.75, + "completions/min_length": 404.0, + "completions/max_length": 595.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 486.75, + "completions/min_terminated_length": 404.0, + "completions/max_terminated_length": 595.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.57373046875, + "reward": 7.0625, + "reward_std": 0.57373046875, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02797355316579342, + "sampling/sampling_logp_difference/max": 0.3631856441497803, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8847656846046448, + "sampling/importance_sampling_ratio/max": 2.0872268676757812, + "kl": 0.02811512805055827, + "entropy": 1.2355968467891216, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.314252337440848, + "epoch": 0.428, + "step": 214 + }, + { + "loss": -0.01179824024438858, + "grad_norm": 0.1873309463262558, + "learning_rate": 3e-05, + "num_tokens": 2202045.0, + "completions/mean_length": 503.1875, + "completions/min_length": 473.0, + "completions/max_length": 575.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 503.1875, + "completions/min_terminated_length": 473.0, + "completions/max_terminated_length": 575.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028071925044059753, + "sampling/sampling_logp_difference/max": 0.6166388988494873, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7044014930725098, + "sampling/importance_sampling_ratio/max": 2.573594808578491, + "kl": 0.03220478829462081, + "entropy": 1.2383001297712326, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 43.65747703285888, + "epoch": 0.43, + "step": 215 + }, + { + "loss": 0.03834616392850876, + "grad_norm": 0.09634320437908173, + "learning_rate": 3e-05, + "num_tokens": 2212188.0, + "completions/mean_length": 503.9375, + "completions/min_length": 464.0, + "completions/max_length": 565.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 503.9375, + "completions/min_terminated_length": 464.0, + "completions/max_terminated_length": 565.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027339771389961243, + "sampling/sampling_logp_difference/max": 0.35487866401672363, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5414069890975952, + "sampling/importance_sampling_ratio/max": 1.9774657487869263, + "kl": 0.03325538453646004, + "entropy": 1.1677803546190262, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.274593455251306, + "epoch": 0.432, + "step": 216 + }, + { + "loss": 0.11844412982463837, + "grad_norm": 0.3425586521625519, + "learning_rate": 3e-05, + "num_tokens": 2219440.0, + "completions/mean_length": 343.75, + "completions/min_length": 7.0, + "completions/max_length": 650.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 343.75, + "completions/min_terminated_length": 7.0, + "completions/max_terminated_length": 650.0, + "rewards/quality_reward/mean": 3.5, + "rewards/quality_reward/std": 3.265986442565918, + "reward": 3.5, + "reward_std": 3.265986442565918, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02829045243561268, + "sampling/sampling_logp_difference/max": 0.3881380558013916, + "sampling/importance_sampling_ratio/min": 0.08477991074323654, + "sampling/importance_sampling_ratio/mean": 0.781898021697998, + "sampling/importance_sampling_ratio/max": 1.4673620462417603, + "kl": 0.03517636051401496, + "entropy": 1.0582842603325844, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 13.36990327714011, + "epoch": 0.434, + "step": 217 + }, + { + "loss": 0.006120521575212479, + "grad_norm": 0.33612945675849915, + "learning_rate": 3e-05, + "num_tokens": 2228975.0, + "completions/mean_length": 471.4375, + "completions/min_length": 416.0, + "completions/max_length": 555.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 471.4375, + "completions/min_terminated_length": 416.0, + "completions/max_terminated_length": 555.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 0.632455587387085, + "reward": 6.5, + "reward_std": 0.632455587387085, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02783289924263954, + "sampling/sampling_logp_difference/max": 0.32509779930114746, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7267376780509949, + "sampling/importance_sampling_ratio/max": 2.1587650775909424, + "kl": 0.030886436812579632, + "entropy": 1.1179756224155426, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.99441510764882, + "epoch": 0.436, + "step": 218 + }, + { + "loss": 0.1648026406764984, + "grad_norm": 0.2976357340812683, + "learning_rate": 3e-05, + "num_tokens": 2238953.0, + "completions/mean_length": 520.125, + "completions/min_length": 462.0, + "completions/max_length": 611.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 520.125, + "completions/min_terminated_length": 462.0, + "completions/max_terminated_length": 611.0, + "rewards/quality_reward/mean": 5.3125, + "rewards/quality_reward/std": 1.078192949295044, + "reward": 5.3125, + "reward_std": 1.078192949295044, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028889676555991173, + "sampling/sampling_logp_difference/max": 0.3546750545501709, + "sampling/importance_sampling_ratio/min": 0.06810324639081955, + "sampling/importance_sampling_ratio/mean": 0.7682108879089355, + "sampling/importance_sampling_ratio/max": 1.6686924695968628, + "kl": 0.03020153450779617, + "entropy": 1.2453451082110405, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.669593635946512, + "epoch": 0.438, + "step": 219 + }, + { + "loss": -0.133737251162529, + "grad_norm": 0.13799847662448883, + "learning_rate": 3e-05, + "num_tokens": 2248199.0, + "completions/mean_length": 465.375, + "completions/min_length": 356.0, + "completions/max_length": 552.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 465.375, + "completions/min_terminated_length": 356.0, + "completions/max_terminated_length": 552.0, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 0.9660918116569519, + "reward": 6.0, + "reward_std": 0.9660918116569519, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027748603373765945, + "sampling/sampling_logp_difference/max": 0.35057830810546875, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5178577899932861, + "sampling/importance_sampling_ratio/max": 2.8963325023651123, + "kl": 0.03191920532844961, + "entropy": 1.3331433907151222, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.38788841944188, + "epoch": 0.44, + "step": 220 + }, + { + "loss": 0.2634640634059906, + "grad_norm": 0.3648691773414612, + "learning_rate": 3e-05, + "num_tokens": 2258327.0, + "completions/mean_length": 507.5, + "completions/min_length": 474.0, + "completions/max_length": 562.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 507.5, + "completions/min_terminated_length": 474.0, + "completions/max_terminated_length": 562.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "reward": 6.8125, + "reward_std": 0.75, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02875763736665249, + "sampling/sampling_logp_difference/max": 0.3389136791229248, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6709499359130859, + "sampling/importance_sampling_ratio/max": 2.1063547134399414, + "kl": 0.03521239408291876, + "entropy": 1.241542100906372, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.28709344472736, + "epoch": 0.442, + "step": 221 + }, + { + "loss": 0.13035088777542114, + "grad_norm": 0.1954081803560257, + "learning_rate": 3e-05, + "num_tokens": 2267381.0, + "completions/mean_length": 459.375, + "completions/min_length": 405.0, + "completions/max_length": 508.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 459.375, + "completions/min_terminated_length": 405.0, + "completions/max_terminated_length": 508.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.8920949101448059, + "reward": 6.5625, + "reward_std": 0.8920949101448059, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02665363810956478, + "sampling/sampling_logp_difference/max": 0.31617891788482666, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6949984431266785, + "sampling/importance_sampling_ratio/max": 2.681690216064453, + "kl": 0.03689368430059403, + "entropy": 1.141789611428976, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 37.694539529737085, + "epoch": 0.444, + "step": 222 + }, + { + "loss": 0.017291374504566193, + "grad_norm": 0.27568548917770386, + "learning_rate": 3e-05, + "num_tokens": 2277446.0, + "completions/mean_length": 515.5625, + "completions/min_length": 459.0, + "completions/max_length": 571.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 515.5625, + "completions/min_terminated_length": 459.0, + "completions/max_terminated_length": 571.0, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 7.125, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027980424463748932, + "sampling/sampling_logp_difference/max": 0.49445104598999023, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7807494401931763, + "sampling/importance_sampling_ratio/max": 2.847236156463623, + "kl": 0.032993816188536584, + "entropy": 1.2330311760306358, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.57380611775443, + "epoch": 0.446, + "step": 223 + }, + { + "loss": 0.05642539635300636, + "grad_norm": 0.10739865154027939, + "learning_rate": 3e-05, + "num_tokens": 2287345.0, + "completions/mean_length": 523.1875, + "completions/min_length": 451.0, + "completions/max_length": 634.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 523.1875, + "completions/min_terminated_length": 451.0, + "completions/max_terminated_length": 634.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027437299489974976, + "sampling/sampling_logp_difference/max": 0.40827369689941406, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6011937856674194, + "sampling/importance_sampling_ratio/max": 1.9240283966064453, + "kl": 0.02909247507341206, + "entropy": 1.0975877195596695, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 39.087660535704345, + "epoch": 0.448, + "step": 224 + }, + { + "loss": -0.02710402011871338, + "grad_norm": 0.22121335566043854, + "learning_rate": 3e-05, + "num_tokens": 2296741.0, + "completions/mean_length": 478.25, + "completions/min_length": 431.0, + "completions/max_length": 533.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 478.25, + "completions/min_terminated_length": 431.0, + "completions/max_terminated_length": 533.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.7274384498596191, + "reward": 6.4375, + "reward_std": 0.7274384498596191, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02776467800140381, + "sampling/sampling_logp_difference/max": 0.4316587448120117, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6062434315681458, + "sampling/importance_sampling_ratio/max": 1.8824238777160645, + "kl": 0.03540586424060166, + "entropy": 1.3173525258898735, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.200199087150395, + "epoch": 0.45, + "step": 225 + }, + { + "loss": 0.19160562753677368, + "grad_norm": 0.3865528702735901, + "learning_rate": 3e-05, + "num_tokens": 2306676.0, + "completions/mean_length": 499.9375, + "completions/min_length": 448.0, + "completions/max_length": 577.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 499.9375, + "completions/min_terminated_length": 448.0, + "completions/max_terminated_length": 577.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028557972982525826, + "sampling/sampling_logp_difference/max": 0.35040283203125, + "sampling/importance_sampling_ratio/min": 0.0957244485616684, + "sampling/importance_sampling_ratio/mean": 0.9618603587150574, + "sampling/importance_sampling_ratio/max": 2.6609690189361572, + "kl": 0.03646970179397613, + "entropy": 1.1501530036330223, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.752630488947034, + "epoch": 0.452, + "step": 226 + }, + { + "loss": 0.04028765484690666, + "grad_norm": 0.1407802700996399, + "learning_rate": 3e-05, + "num_tokens": 2316696.0, + "completions/mean_length": 510.75, + "completions/min_length": 419.0, + "completions/max_length": 610.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 510.75, + "completions/min_terminated_length": 419.0, + "completions/max_terminated_length": 610.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 6.5625, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027024609968066216, + "sampling/sampling_logp_difference/max": 0.3711676597595215, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6032290458679199, + "sampling/importance_sampling_ratio/max": 1.2131233215332031, + "kl": 0.030792692443355918, + "entropy": 1.190872348845005, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.143527323380113, + "epoch": 0.454, + "step": 227 + }, + { + "loss": -0.26227492094039917, + "grad_norm": 0.18564435839653015, + "learning_rate": 3e-05, + "num_tokens": 2326322.0, + "completions/mean_length": 495.625, + "completions/min_length": 419.0, + "completions/max_length": 575.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 495.625, + "completions/min_terminated_length": 419.0, + "completions/max_terminated_length": 575.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027370886877179146, + "sampling/sampling_logp_difference/max": 0.34752869606018066, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6159635782241821, + "sampling/importance_sampling_ratio/max": 2.640001058578491, + "kl": 0.025341857108287513, + "entropy": 1.2335442155599594, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.369792928919196, + "epoch": 0.456, + "step": 228 + }, + { + "loss": -0.031097467988729477, + "grad_norm": 0.2313818782567978, + "learning_rate": 3e-05, + "num_tokens": 2335973.0, + "completions/mean_length": 487.6875, + "completions/min_length": 449.0, + "completions/max_length": 528.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 487.6875, + "completions/min_terminated_length": 449.0, + "completions/max_terminated_length": 528.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.8539125919342041, + "reward": 6.9375, + "reward_std": 0.8539125919342041, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02758944220840931, + "sampling/sampling_logp_difference/max": 0.3627934455871582, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7488094568252563, + "sampling/importance_sampling_ratio/max": 2.227858543395996, + "kl": 0.03635518567170948, + "entropy": 1.2502131089568138, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.443659604527056, + "epoch": 0.458, + "step": 229 + }, + { + "loss": 0.5464498996734619, + "grad_norm": 0.5150085687637329, + "learning_rate": 3e-05, + "num_tokens": 2345579.0, + "completions/mean_length": 476.375, + "completions/min_length": 386.0, + "completions/max_length": 602.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 476.375, + "completions/min_terminated_length": 386.0, + "completions/max_terminated_length": 602.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.6551081538200378, + "reward": 6.8125, + "reward_std": 0.6551081538200378, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026741618290543556, + "sampling/sampling_logp_difference/max": 0.42708492279052734, + "sampling/importance_sampling_ratio/min": 0.1553211510181427, + "sampling/importance_sampling_ratio/mean": 0.7721551060676575, + "sampling/importance_sampling_ratio/max": 2.376497268676758, + "kl": 0.032692725653760135, + "entropy": 1.2348124124109745, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.71390812145546, + "epoch": 0.46, + "step": 230 + }, + { + "loss": -0.1646902710199356, + "grad_norm": 0.11509153991937637, + "learning_rate": 3e-05, + "num_tokens": 2355499.0, + "completions/mean_length": 517.5, + "completions/min_length": 424.0, + "completions/max_length": 593.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 517.5, + "completions/min_terminated_length": 424.0, + "completions/max_terminated_length": 593.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027583574876189232, + "sampling/sampling_logp_difference/max": 0.4672989845275879, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6012319326400757, + "sampling/importance_sampling_ratio/max": 2.2603800296783447, + "kl": 0.03900455019902438, + "entropy": 1.1868174076080322, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.276649605948478, + "epoch": 0.462, + "step": 231 + }, + { + "loss": 0.00344286416657269, + "grad_norm": 0.17417575418949127, + "learning_rate": 3e-05, + "num_tokens": 2366049.0, + "completions/mean_length": 560.875, + "completions/min_length": 494.0, + "completions/max_length": 691.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 560.875, + "completions/min_terminated_length": 494.0, + "completions/max_terminated_length": 691.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028019659221172333, + "sampling/sampling_logp_difference/max": 0.4202132225036621, + "sampling/importance_sampling_ratio/min": 0.13027621805667877, + "sampling/importance_sampling_ratio/mean": 0.740157961845398, + "sampling/importance_sampling_ratio/max": 2.206153154373169, + "kl": 0.031763071776367724, + "entropy": 1.1533633023500443, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.686087283305824, + "epoch": 0.464, + "step": 232 + }, + { + "loss": 0.03899282589554787, + "grad_norm": 0.22023369371891022, + "learning_rate": 3e-05, + "num_tokens": 2375582.0, + "completions/mean_length": 500.8125, + "completions/min_length": 410.0, + "completions/max_length": 594.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 500.8125, + "completions/min_terminated_length": 410.0, + "completions/max_terminated_length": 594.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "reward": 6.9375, + "reward_std": 0.57373046875, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027327092364430428, + "sampling/sampling_logp_difference/max": 0.4571092128753662, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6224058270454407, + "sampling/importance_sampling_ratio/max": 2.0323734283447266, + "kl": 0.03284288931172341, + "entropy": 1.2008480802178383, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.555491346865892, + "epoch": 0.466, + "step": 233 + }, + { + "loss": -0.06800927221775055, + "grad_norm": 0.13053849339485168, + "learning_rate": 3e-05, + "num_tokens": 2386702.0, + "completions/mean_length": 587.5, + "completions/min_length": 440.0, + "completions/max_length": 845.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 587.5, + "completions/min_terminated_length": 440.0, + "completions/max_terminated_length": 845.0, + "rewards/quality_reward/mean": 6.0, + "rewards/quality_reward/std": 1.7511900663375854, + "reward": 6.0, + "reward_std": 1.7511900663375854, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02704041451215744, + "sampling/sampling_logp_difference/max": 0.5877337455749512, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4879741370677948, + "sampling/importance_sampling_ratio/max": 1.4020758867263794, + "kl": 0.03163444856181741, + "entropy": 1.2540696933865547, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 34.3772664074786, + "epoch": 0.468, + "step": 234 + }, + { + "loss": -0.1597842425107956, + "grad_norm": 0.1838671714067459, + "learning_rate": 3e-05, + "num_tokens": 2396564.0, + "completions/mean_length": 506.375, + "completions/min_length": 422.0, + "completions/max_length": 584.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 506.375, + "completions/min_terminated_length": 422.0, + "completions/max_terminated_length": 584.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.7932003140449524, + "reward": 6.6875, + "reward_std": 0.7932003140449524, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029480738565325737, + "sampling/sampling_logp_difference/max": 0.37027931213378906, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7693536281585693, + "sampling/importance_sampling_ratio/max": 2.9093716144561768, + "kl": 0.03747367626056075, + "entropy": 1.3376594334840775, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 14.542957273777574, + "epoch": 0.47, + "step": 235 + }, + { + "loss": 0.24737706780433655, + "grad_norm": 0.44443222880363464, + "learning_rate": 3e-05, + "num_tokens": 2406551.0, + "completions/mean_length": 509.6875, + "completions/min_length": 443.0, + "completions/max_length": 605.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 509.6875, + "completions/min_terminated_length": 443.0, + "completions/max_terminated_length": 605.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028956102207303047, + "sampling/sampling_logp_difference/max": 0.29999852180480957, + "sampling/importance_sampling_ratio/min": 0.04889443516731262, + "sampling/importance_sampling_ratio/mean": 0.7888213396072388, + "sampling/importance_sampling_ratio/max": 2.7392280101776123, + "kl": 0.03565627557691187, + "entropy": 1.3714017421007156, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.966124589089304, + "epoch": 0.472, + "step": 236 + }, + { + "loss": -0.006456274073570967, + "grad_norm": 0.1285122185945511, + "learning_rate": 3e-05, + "num_tokens": 2416754.0, + "completions/mean_length": 541.1875, + "completions/min_length": 461.0, + "completions/max_length": 647.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 541.1875, + "completions/min_terminated_length": 461.0, + "completions/max_terminated_length": 647.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "reward": 6.8125, + "reward_std": 0.75, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027147701010107994, + "sampling/sampling_logp_difference/max": 0.4087851047515869, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5822510719299316, + "sampling/importance_sampling_ratio/max": 1.395982265472412, + "kl": 0.03105375764425844, + "entropy": 1.199029415845871, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 31.801921805832535, + "epoch": 0.474, + "step": 237 + }, + { + "loss": 0.3960018455982208, + "grad_norm": 0.3568721413612366, + "learning_rate": 3e-05, + "num_tokens": 2428005.0, + "completions/mean_length": 598.1875, + "completions/min_length": 496.0, + "completions/max_length": 772.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 598.1875, + "completions/min_terminated_length": 496.0, + "completions/max_terminated_length": 772.0, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.1254628896713257, + "reward": 6.25, + "reward_std": 1.1254628896713257, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027511969208717346, + "sampling/sampling_logp_difference/max": 0.42554450035095215, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5545582175254822, + "sampling/importance_sampling_ratio/max": 1.9572224617004395, + "kl": 0.03257777914404869, + "entropy": 1.2089053243398666, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.979049060028046, + "epoch": 0.476, + "step": 238 + }, + { + "loss": 0.025357680395245552, + "grad_norm": 0.07262199372053146, + "learning_rate": 3e-05, + "num_tokens": 2437779.0, + "completions/mean_length": 508.875, + "completions/min_length": 473.0, + "completions/max_length": 580.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 508.875, + "completions/min_terminated_length": 473.0, + "completions/max_terminated_length": 580.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029029743745923042, + "sampling/sampling_logp_difference/max": 0.37055206298828125, + "sampling/importance_sampling_ratio/min": 0.07304152101278305, + "sampling/importance_sampling_ratio/mean": 0.8696970343589783, + "sampling/importance_sampling_ratio/max": 2.7364466190338135, + "kl": 0.03327966062352061, + "entropy": 1.4715757966041565, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.994311626069248, + "epoch": 0.478, + "step": 239 + }, + { + "loss": 0.12445695698261261, + "grad_norm": 0.28445860743522644, + "learning_rate": 3e-05, + "num_tokens": 2448202.0, + "completions/mean_length": 527.9375, + "completions/min_length": 435.0, + "completions/max_length": 580.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 527.9375, + "completions/min_terminated_length": 435.0, + "completions/max_terminated_length": 580.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02976268343627453, + "sampling/sampling_logp_difference/max": 0.5368318557739258, + "sampling/importance_sampling_ratio/min": 0.052414167672395706, + "sampling/importance_sampling_ratio/mean": 0.5472592711448669, + "sampling/importance_sampling_ratio/max": 1.700259804725647, + "kl": 0.03817834367509931, + "entropy": 1.311545416712761, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.84421144844964, + "epoch": 0.48, + "step": 240 + }, + { + "loss": 0.04229091852903366, + "grad_norm": 0.16590596735477448, + "learning_rate": 3e-05, + "num_tokens": 2458743.0, + "completions/mean_length": 543.3125, + "completions/min_length": 471.0, + "completions/max_length": 598.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 543.3125, + "completions/min_terminated_length": 471.0, + "completions/max_terminated_length": 598.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02782438136637211, + "sampling/sampling_logp_difference/max": 0.40506887435913086, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.9457916021347046, + "sampling/importance_sampling_ratio/max": 2.728398323059082, + "kl": 0.03646332467906177, + "entropy": 1.180833749473095, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.389156353194267, + "epoch": 0.482, + "step": 241 + }, + { + "loss": -0.017443601042032242, + "grad_norm": 0.25544801354408264, + "learning_rate": 3e-05, + "num_tokens": 2468492.0, + "completions/mean_length": 502.3125, + "completions/min_length": 454.0, + "completions/max_length": 579.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 502.3125, + "completions/min_terminated_length": 454.0, + "completions/max_terminated_length": 579.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "reward": 6.9375, + "reward_std": 0.57373046875, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027156969532370567, + "sampling/sampling_logp_difference/max": 0.6171557903289795, + "sampling/importance_sampling_ratio/min": 0.08132059127092361, + "sampling/importance_sampling_ratio/mean": 0.8106446266174316, + "sampling/importance_sampling_ratio/max": 1.8124310970306396, + "kl": 0.03445987973827869, + "entropy": 1.1694707348942757, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.112746777944267, + "epoch": 0.484, + "step": 242 + }, + { + "loss": -0.030910439789295197, + "grad_norm": 0.17728154361248016, + "learning_rate": 3e-05, + "num_tokens": 2478878.0, + "completions/mean_length": 538.125, + "completions/min_length": 448.0, + "completions/max_length": 654.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 538.125, + "completions/min_terminated_length": 448.0, + "completions/max_terminated_length": 654.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 6.8125, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02802959457039833, + "sampling/sampling_logp_difference/max": 0.3394327163696289, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5417827367782593, + "sampling/importance_sampling_ratio/max": 1.7654427289962769, + "kl": 0.03608768491540104, + "entropy": 1.297831729054451, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 39.02764433948323, + "epoch": 0.486, + "step": 243 + }, + { + "loss": 0.010581104084849358, + "grad_norm": 0.1783561259508133, + "learning_rate": 3e-05, + "num_tokens": 2489938.0, + "completions/mean_length": 564.75, + "completions/min_length": 449.0, + "completions/max_length": 703.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 564.75, + "completions/min_terminated_length": 449.0, + "completions/max_terminated_length": 703.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028042398393154144, + "sampling/sampling_logp_difference/max": 0.8026522397994995, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6222037672996521, + "sampling/importance_sampling_ratio/max": 1.9127941131591797, + "kl": 0.04178670607507229, + "entropy": 1.2894479781389236, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.52857542503625, + "epoch": 0.488, + "step": 244 + }, + { + "loss": 0.06864061206579208, + "grad_norm": 0.34433481097221375, + "learning_rate": 3e-05, + "num_tokens": 2499760.0, + "completions/mean_length": 511.875, + "completions/min_length": 455.0, + "completions/max_length": 599.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 511.875, + "completions/min_terminated_length": 455.0, + "completions/max_terminated_length": 599.0, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 7.1875, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027506975457072258, + "sampling/sampling_logp_difference/max": 0.3724935054779053, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.9394024014472961, + "sampling/importance_sampling_ratio/max": 2.993313789367676, + "kl": 0.04671380412764847, + "entropy": 1.1273594908416271, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.86254589399323, + "epoch": 0.49, + "step": 245 + }, + { + "loss": -0.00970650464296341, + "grad_norm": 0.2073342353105545, + "learning_rate": 3e-05, + "num_tokens": 2510176.0, + "completions/mean_length": 533.0, + "completions/min_length": 467.0, + "completions/max_length": 684.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 533.0, + "completions/min_terminated_length": 467.0, + "completions/max_terminated_length": 684.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.75, + "reward": 6.8125, + "reward_std": 0.75, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028828633949160576, + "sampling/sampling_logp_difference/max": 0.5199446678161621, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6703793406486511, + "sampling/importance_sampling_ratio/max": 2.6556100845336914, + "kl": 0.03919998917263001, + "entropy": 1.2337471172213554, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.641962222289294, + "epoch": 0.492, + "step": 246 + }, + { + "loss": -0.060149889439344406, + "grad_norm": 0.10448901355266571, + "learning_rate": 3e-05, + "num_tokens": 2520981.0, + "completions/mean_length": 555.8125, + "completions/min_length": 493.0, + "completions/max_length": 636.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 555.8125, + "completions/min_terminated_length": 493.0, + "completions/max_terminated_length": 636.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.6831300854682922, + "reward": 6.75, + "reward_std": 0.6831300854682922, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028163518756628036, + "sampling/sampling_logp_difference/max": 0.36969757080078125, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4310106039047241, + "sampling/importance_sampling_ratio/max": 1.087956428527832, + "kl": 0.043822019128128886, + "entropy": 1.1313174478709698, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.198071955237538, + "epoch": 0.494, + "step": 247 + }, + { + "loss": -0.15169084072113037, + "grad_norm": 0.21327541768550873, + "learning_rate": 3e-05, + "num_tokens": 2531056.0, + "completions/mean_length": 533.1875, + "completions/min_length": 461.0, + "completions/max_length": 569.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 533.1875, + "completions/min_terminated_length": 461.0, + "completions/max_terminated_length": 569.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8139410614967346, + "reward": 6.4375, + "reward_std": 0.8139410614967346, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028493624180555344, + "sampling/sampling_logp_difference/max": 0.5983643531799316, + "sampling/importance_sampling_ratio/min": 0.042581744492053986, + "sampling/importance_sampling_ratio/mean": 0.924071729183197, + "sampling/importance_sampling_ratio/max": 2.145639657974243, + "kl": 0.0436656444799155, + "entropy": 1.2226731404662132, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 46.37140509998426, + "epoch": 0.496, + "step": 248 + }, + { + "loss": 0.024922871962189674, + "grad_norm": 0.1687636375427246, + "learning_rate": 3e-05, + "num_tokens": 2542129.0, + "completions/mean_length": 580.5625, + "completions/min_length": 514.0, + "completions/max_length": 675.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 580.5625, + "completions/min_terminated_length": 514.0, + "completions/max_terminated_length": 675.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 6.9375, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029055560007691383, + "sampling/sampling_logp_difference/max": 0.35921406745910645, + "sampling/importance_sampling_ratio/min": 0.11655592173337936, + "sampling/importance_sampling_ratio/mean": 0.6349776983261108, + "sampling/importance_sampling_ratio/max": 2.1956820487976074, + "kl": 0.03778462728951126, + "entropy": 1.2647478878498077, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 34.322586783673614, + "epoch": 0.498, + "step": 249 + }, + { + "loss": -0.24095430970191956, + "grad_norm": 0.3467749357223511, + "learning_rate": 3e-05, + "num_tokens": 2552412.0, + "completions/mean_length": 543.6875, + "completions/min_length": 451.0, + "completions/max_length": 694.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 543.6875, + "completions/min_terminated_length": 451.0, + "completions/max_terminated_length": 694.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.8062257766723633, + "reward": 6.625, + "reward_std": 0.8062257766723633, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028353629633784294, + "sampling/sampling_logp_difference/max": 0.3446807861328125, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.9581880569458008, + "sampling/importance_sampling_ratio/max": 2.5754284858703613, + "kl": 0.03919053066056222, + "entropy": 1.2015386000275612, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.97987106954679, + "epoch": 0.5, + "step": 250 + }, + { + "loss": -0.1708906590938568, + "grad_norm": 0.3573401868343353, + "learning_rate": 3e-05, + "num_tokens": 2563580.0, + "completions/mean_length": 584.5, + "completions/min_length": 492.0, + "completions/max_length": 749.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 584.5, + "completions/min_terminated_length": 492.0, + "completions/max_terminated_length": 749.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.704154372215271, + "reward": 6.6875, + "reward_std": 0.704154372215271, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02920239232480526, + "sampling/sampling_logp_difference/max": 0.35509490966796875, + "sampling/importance_sampling_ratio/min": 0.02091093361377716, + "sampling/importance_sampling_ratio/mean": 0.8432164788246155, + "sampling/importance_sampling_ratio/max": 2.4531548023223877, + "kl": 0.03822207520715892, + "entropy": 1.1965860426425934, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.99323159083724, + "epoch": 0.502, + "step": 251 + }, + { + "loss": -0.07006160914897919, + "grad_norm": 0.12554219365119934, + "learning_rate": 3e-05, + "num_tokens": 2573370.0, + "completions/mean_length": 503.375, + "completions/min_length": 438.0, + "completions/max_length": 565.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 503.375, + "completions/min_terminated_length": 438.0, + "completions/max_terminated_length": 565.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.6800735592842102, + "reward": 6.9375, + "reward_std": 0.6800735592842102, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029184387996792793, + "sampling/sampling_logp_difference/max": 0.37473583221435547, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7688348293304443, + "sampling/importance_sampling_ratio/max": 2.634645462036133, + "kl": 0.04616628657095134, + "entropy": 1.2068623006343842, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 43.27480686130002, + "epoch": 0.504, + "step": 252 + }, + { + "loss": 0.06332479417324066, + "grad_norm": 0.19425006210803986, + "learning_rate": 3e-05, + "num_tokens": 2583780.0, + "completions/mean_length": 540.625, + "completions/min_length": 484.0, + "completions/max_length": 645.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 540.625, + "completions/min_terminated_length": 484.0, + "completions/max_terminated_length": 645.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 6.75, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028582781553268433, + "sampling/sampling_logp_difference/max": 0.4301719665527344, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5209584832191467, + "sampling/importance_sampling_ratio/max": 1.2259461879730225, + "kl": 0.040457896422594786, + "entropy": 1.2164383009076118, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.769911186769605, + "epoch": 0.506, + "step": 253 + }, + { + "loss": -0.03424276039004326, + "grad_norm": 0.20388974249362946, + "learning_rate": 3e-05, + "num_tokens": 2592624.0, + "completions/mean_length": 452.25, + "completions/min_length": 275.0, + "completions/max_length": 626.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 452.25, + "completions/min_terminated_length": 275.0, + "completions/max_terminated_length": 626.0, + "rewards/quality_reward/mean": 7.375, + "rewards/quality_reward/std": 0.7187952995300293, + "reward": 7.375, + "reward_std": 0.7187952995300293, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028338422998785973, + "sampling/sampling_logp_difference/max": 0.2922825813293457, + "sampling/importance_sampling_ratio/min": 0.06710651516914368, + "sampling/importance_sampling_ratio/mean": 0.6690866947174072, + "sampling/importance_sampling_ratio/max": 2.8529531955718994, + "kl": 0.043233372969552875, + "entropy": 1.2369564026594162, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.368686333298683, + "epoch": 0.508, + "step": 254 + }, + { + "loss": -0.26449277997016907, + "grad_norm": 0.3238426446914673, + "learning_rate": 3e-05, + "num_tokens": 2603772.0, + "completions/mean_length": 593.25, + "completions/min_length": 494.0, + "completions/max_length": 678.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 593.25, + "completions/min_terminated_length": 494.0, + "completions/max_terminated_length": 678.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.57373046875, + "reward": 6.9375, + "reward_std": 0.57373046875, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02828345075249672, + "sampling/sampling_logp_difference/max": 0.5587451457977295, + "sampling/importance_sampling_ratio/min": 0.024249335750937462, + "sampling/importance_sampling_ratio/mean": 0.544727087020874, + "sampling/importance_sampling_ratio/max": 1.9910115003585815, + "kl": 0.040533376624807715, + "entropy": 1.2358134537935257, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 28.741963400039822, + "epoch": 0.51, + "step": 255 + }, + { + "loss": 0.1972789317369461, + "grad_norm": 0.24497906863689423, + "learning_rate": 3e-05, + "num_tokens": 2614015.0, + "completions/mean_length": 541.6875, + "completions/min_length": 388.0, + "completions/max_length": 639.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 541.6875, + "completions/min_terminated_length": 388.0, + "completions/max_terminated_length": 639.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.9287087917327881, + "reward": 6.9375, + "reward_std": 0.9287087917327881, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028692016378045082, + "sampling/sampling_logp_difference/max": 0.4399089813232422, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6314806342124939, + "sampling/importance_sampling_ratio/max": 2.615037679672241, + "kl": 0.035602600779384375, + "entropy": 1.2090466022491455, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.5418023574166, + "epoch": 0.512, + "step": 256 + }, + { + "loss": 0.08096523582935333, + "grad_norm": 0.23706457018852234, + "learning_rate": 3e-05, + "num_tokens": 2625568.0, + "completions/mean_length": 608.0625, + "completions/min_length": 465.0, + "completions/max_length": 765.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 608.0625, + "completions/min_terminated_length": 465.0, + "completions/max_terminated_length": 765.0, + "rewards/quality_reward/mean": 6.9375, + "rewards/quality_reward/std": 0.25, + "reward": 6.9375, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029812760651111603, + "sampling/sampling_logp_difference/max": 0.40811336040496826, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4297117590904236, + "sampling/importance_sampling_ratio/max": 2.320277214050293, + "kl": 0.0430363982450217, + "entropy": 1.4103579595685005, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 30.523871788289398, + "epoch": 0.514, + "step": 257 + }, + { + "loss": 0.02335459366440773, + "grad_norm": 0.3034888505935669, + "learning_rate": 3e-05, + "num_tokens": 2636443.0, + "completions/mean_length": 573.6875, + "completions/min_length": 508.0, + "completions/max_length": 660.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 573.6875, + "completions/min_terminated_length": 508.0, + "completions/max_terminated_length": 660.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8139410614967346, + "reward": 6.4375, + "reward_std": 0.8139410614967346, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028646407648921013, + "sampling/sampling_logp_difference/max": 0.3945488929748535, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6644032597541809, + "sampling/importance_sampling_ratio/max": 1.9408409595489502, + "kl": 0.044887167401611805, + "entropy": 1.2737382426857948, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.04052680823952, + "epoch": 0.516, + "step": 258 + }, + { + "loss": -0.16838416457176208, + "grad_norm": 0.15292863547801971, + "learning_rate": 3e-05, + "num_tokens": 2647171.0, + "completions/mean_length": 574.0, + "completions/min_length": 498.0, + "completions/max_length": 692.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 574.0, + "completions/min_terminated_length": 498.0, + "completions/max_terminated_length": 692.0, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 7.1875, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028790012001991272, + "sampling/sampling_logp_difference/max": 0.3294107913970947, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4699386954307556, + "sampling/importance_sampling_ratio/max": 1.4874011278152466, + "kl": 0.03639746748376638, + "entropy": 1.1968051716685295, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.238027889747173, + "epoch": 0.518, + "step": 259 + }, + { + "loss": 0.18645404279232025, + "grad_norm": 0.303418904542923, + "learning_rate": 3e-05, + "num_tokens": 2657583.0, + "completions/mean_length": 546.25, + "completions/min_length": 464.0, + "completions/max_length": 630.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 546.25, + "completions/min_terminated_length": 464.0, + "completions/max_terminated_length": 630.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027895493432879448, + "sampling/sampling_logp_difference/max": 0.3931598663330078, + "sampling/importance_sampling_ratio/min": 0.03731733188033104, + "sampling/importance_sampling_ratio/mean": 0.6766756772994995, + "sampling/importance_sampling_ratio/max": 2.91849422454834, + "kl": 0.04550225310958922, + "entropy": 1.1241988725960255, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.580054661724716, + "epoch": 0.52, + "step": 260 + }, + { + "loss": -0.01714285835623741, + "grad_norm": 0.20534871518611908, + "learning_rate": 3e-05, + "num_tokens": 2668042.0, + "completions/mean_length": 559.1875, + "completions/min_length": 526.0, + "completions/max_length": 604.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 559.1875, + "completions/min_terminated_length": 526.0, + "completions/max_terminated_length": 604.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029195168986916542, + "sampling/sampling_logp_difference/max": 0.3395042419433594, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8631129264831543, + "sampling/importance_sampling_ratio/max": 2.1291964054107666, + "kl": 0.05870963633060455, + "entropy": 1.2270803824067116, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.93122593825683, + "epoch": 0.522, + "step": 261 + }, + { + "loss": 0.10568767786026001, + "grad_norm": 0.3122904300689697, + "learning_rate": 3e-05, + "num_tokens": 2678318.0, + "completions/mean_length": 515.25, + "completions/min_length": 328.0, + "completions/max_length": 592.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 515.25, + "completions/min_terminated_length": 328.0, + "completions/max_terminated_length": 592.0, + "rewards/quality_reward/mean": 6.25, + "rewards/quality_reward/std": 1.5705626010894775, + "reward": 6.25, + "reward_std": 1.5705626010894775, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030075252056121826, + "sampling/sampling_logp_difference/max": 0.6918299198150635, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7784014940261841, + "sampling/importance_sampling_ratio/max": 2.7710750102996826, + "kl": 0.044736934592947364, + "entropy": 1.372651383280754, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 37.408678135834634, + "epoch": 0.524, + "step": 262 + }, + { + "loss": -0.14915889501571655, + "grad_norm": 0.14924532175064087, + "learning_rate": 3e-05, + "num_tokens": 2688798.0, + "completions/mean_length": 557.5, + "completions/min_length": 426.0, + "completions/max_length": 623.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 557.5, + "completions/min_terminated_length": 426.0, + "completions/max_terminated_length": 623.0, + "rewards/quality_reward/mean": 6.125, + "rewards/quality_reward/std": 1.1474609375, + "reward": 6.125, + "reward_std": 1.1474609375, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029747022315859795, + "sampling/sampling_logp_difference/max": 0.36669039726257324, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.46721577644348145, + "sampling/importance_sampling_ratio/max": 1.2867430448532104, + "kl": 0.04041226860135794, + "entropy": 1.4175518676638603, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.848117691930383, + "epoch": 0.526, + "step": 263 + }, + { + "loss": 0.23506437242031097, + "grad_norm": 0.2876143157482147, + "learning_rate": 3e-05, + "num_tokens": 2699949.0, + "completions/mean_length": 593.9375, + "completions/min_length": 525.0, + "completions/max_length": 741.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 593.9375, + "completions/min_terminated_length": 525.0, + "completions/max_terminated_length": 741.0, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 0.8732125163078308, + "reward": 6.3125, + "reward_std": 0.8732125163078308, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028063040226697922, + "sampling/sampling_logp_difference/max": 0.5987787246704102, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6758359670639038, + "sampling/importance_sampling_ratio/max": 2.0369813442230225, + "kl": 0.0446856344351545, + "entropy": 1.1905437037348747, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 36.39752811612561, + "epoch": 0.528, + "step": 264 + }, + { + "loss": -0.13722549378871918, + "grad_norm": 0.10611388832330704, + "learning_rate": 3e-05, + "num_tokens": 2710444.0, + "completions/mean_length": 542.4375, + "completions/min_length": 444.0, + "completions/max_length": 622.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 542.4375, + "completions/min_terminated_length": 444.0, + "completions/max_terminated_length": 622.0, + "rewards/quality_reward/mean": 7.4375, + "rewards/quality_reward/std": 0.6291528940200806, + "reward": 7.4375, + "reward_std": 0.6291528940200806, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026776142418384552, + "sampling/sampling_logp_difference/max": 0.49287891387939453, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4690064787864685, + "sampling/importance_sampling_ratio/max": 2.752124309539795, + "kl": 0.04348581004887819, + "entropy": 1.0300748609006405, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.34836289891973, + "epoch": 0.53, + "step": 265 + }, + { + "loss": 0.3793664872646332, + "grad_norm": 0.3129880428314209, + "learning_rate": 3e-05, + "num_tokens": 2721859.0, + "completions/mean_length": 604.9375, + "completions/min_length": 477.0, + "completions/max_length": 773.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 604.9375, + "completions/min_terminated_length": 477.0, + "completions/max_terminated_length": 773.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.57373046875, + "reward": 7.0625, + "reward_std": 0.57373046875, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029427004978060722, + "sampling/sampling_logp_difference/max": 0.48268747329711914, + "sampling/importance_sampling_ratio/min": 0.09143810719251633, + "sampling/importance_sampling_ratio/mean": 1.0858334302902222, + "sampling/importance_sampling_ratio/max": 2.9960079193115234, + "kl": 0.05965391919016838, + "entropy": 1.2693111822009087, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.01661626342684, + "epoch": 0.532, + "step": 266 + }, + { + "loss": 0.004537707194685936, + "grad_norm": 0.0628323182463646, + "learning_rate": 3e-05, + "num_tokens": 2732788.0, + "completions/mean_length": 580.0625, + "completions/min_length": 478.0, + "completions/max_length": 652.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 580.0625, + "completions/min_terminated_length": 478.0, + "completions/max_terminated_length": 652.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.3651483952999115, + "reward": 7.0, + "reward_std": 0.3651483952999115, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030082745477557182, + "sampling/sampling_logp_difference/max": 0.4802044630050659, + "sampling/importance_sampling_ratio/min": 0.05455247685313225, + "sampling/importance_sampling_ratio/mean": 0.5058171153068542, + "sampling/importance_sampling_ratio/max": 2.5967259407043457, + "kl": 0.0641073645092547, + "entropy": 1.193462796509266, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.015314052347094, + "epoch": 0.534, + "step": 267 + }, + { + "loss": 0.10845151543617249, + "grad_norm": 0.18749572336673737, + "learning_rate": 3e-05, + "num_tokens": 2747027.0, + "completions/mean_length": 757.9375, + "completions/min_length": 557.0, + "completions/max_length": 1024.0, + "completions/clipped_ratio": 0.125, + "completions/mean_terminated_length": 719.9285888671875, + "completions/min_terminated_length": 557.0, + "completions/max_terminated_length": 1021.0, + "rewards/quality_reward/mean": 5.75, + "rewards/quality_reward/std": 2.9325757026672363, + "reward": 5.75, + "reward_std": 2.9325757026672363, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02539048343896866, + "sampling/sampling_logp_difference/max": 0.6482839584350586, + "sampling/importance_sampling_ratio/min": 0.038960449397563934, + "sampling/importance_sampling_ratio/mean": 0.6503843069076538, + "sampling/importance_sampling_ratio/max": 1.6593483686447144, + "kl": 0.04782780213281512, + "entropy": 1.0249068401753902, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.50977089582011, + "epoch": 0.536, + "step": 268 + }, + { + "loss": -0.12464538961648941, + "grad_norm": 0.40060457587242126, + "learning_rate": 3e-05, + "num_tokens": 2758653.0, + "completions/mean_length": 625.125, + "completions/min_length": 548.0, + "completions/max_length": 720.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 625.125, + "completions/min_terminated_length": 548.0, + "completions/max_terminated_length": 720.0, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 7.125, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028948824852705002, + "sampling/sampling_logp_difference/max": 0.3937206268310547, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.7976686358451843, + "sampling/importance_sampling_ratio/max": 2.5361881256103516, + "kl": 0.07021735375747085, + "entropy": 1.3341968581080437, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 29.484100938774645, + "epoch": 0.538, + "step": 269 + }, + { + "loss": 0.1288338303565979, + "grad_norm": 0.2377263456583023, + "learning_rate": 3e-05, + "num_tokens": 2769927.0, + "completions/mean_length": 591.625, + "completions/min_length": 485.0, + "completions/max_length": 744.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 591.625, + "completions/min_terminated_length": 485.0, + "completions/max_terminated_length": 744.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 1.0878112316131592, + "reward": 6.875, + "reward_std": 1.0878112316131592, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029194451868534088, + "sampling/sampling_logp_difference/max": 0.4420192837715149, + "sampling/importance_sampling_ratio/min": 0.046515896916389465, + "sampling/importance_sampling_ratio/mean": 0.6740471124649048, + "sampling/importance_sampling_ratio/max": 1.932815670967102, + "kl": 0.07358641736209393, + "entropy": 1.2878348901867867, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 31.247754107695073, + "epoch": 0.54, + "step": 270 + }, + { + "loss": -0.19648313522338867, + "grad_norm": 0.19190217554569244, + "learning_rate": 3e-05, + "num_tokens": 2781162.0, + "completions/mean_length": 593.1875, + "completions/min_length": 495.0, + "completions/max_length": 788.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 593.1875, + "completions/min_terminated_length": 495.0, + "completions/max_terminated_length": 788.0, + "rewards/quality_reward/mean": 6.6875, + "rewards/quality_reward/std": 0.8732125163078308, + "reward": 6.6875, + "reward_std": 0.8732125163078308, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.030100077390670776, + "sampling/sampling_logp_difference/max": 0.3736577033996582, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 1.090332269668579, + "sampling/importance_sampling_ratio/max": 2.6719114780426025, + "kl": 0.06670599663630128, + "entropy": 1.2879671305418015, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.30791286379099, + "epoch": 0.542, + "step": 271 + }, + { + "loss": 0.07189144194126129, + "grad_norm": 0.20936760306358337, + "learning_rate": 3e-05, + "num_tokens": 2792824.0, + "completions/mean_length": 627.375, + "completions/min_length": 545.0, + "completions/max_length": 776.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 627.375, + "completions/min_terminated_length": 545.0, + "completions/max_terminated_length": 776.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "reward": 6.625, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029183225706219673, + "sampling/sampling_logp_difference/max": 0.6159329414367676, + "sampling/importance_sampling_ratio/min": 0.0627136304974556, + "sampling/importance_sampling_ratio/mean": 0.5998578667640686, + "sampling/importance_sampling_ratio/max": 2.1611435413360596, + "kl": 0.06371736479923129, + "entropy": 1.147661603987217, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 24.871985231991857, + "epoch": 0.544, + "step": 272 + }, + { + "loss": 0.14990417659282684, + "grad_norm": 0.1734493374824524, + "learning_rate": 3e-05, + "num_tokens": 2804816.0, + "completions/mean_length": 635.0, + "completions/min_length": 525.0, + "completions/max_length": 827.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 635.0, + "completions/min_terminated_length": 525.0, + "completions/max_terminated_length": 827.0, + "rewards/quality_reward/mean": 3.625, + "rewards/quality_reward/std": 3.7572154998779297, + "reward": 3.625, + "reward_std": 3.7572154998779297, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02763475477695465, + "sampling/sampling_logp_difference/max": 0.4317042827606201, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5959733128547668, + "sampling/importance_sampling_ratio/max": 2.1873209476470947, + "kl": 0.05470140045508742, + "entropy": 1.2339624986052513, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.013169853948057, + "epoch": 0.546, + "step": 273 + }, + { + "loss": -0.18302924931049347, + "grad_norm": 0.22998619079589844, + "learning_rate": 3e-05, + "num_tokens": 2816328.0, + "completions/mean_length": 609.5, + "completions/min_length": 509.0, + "completions/max_length": 703.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 609.5, + "completions/min_terminated_length": 509.0, + "completions/max_terminated_length": 703.0, + "rewards/quality_reward/mean": 7.375, + "rewards/quality_reward/std": 0.5, + "reward": 7.375, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028044190257787704, + "sampling/sampling_logp_difference/max": 0.3631110191345215, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6655144691467285, + "sampling/importance_sampling_ratio/max": 2.3319060802459717, + "kl": 0.0687105453107506, + "entropy": 1.1710311695933342, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 39.533187630586326, + "epoch": 0.548, + "step": 274 + }, + { + "loss": -0.034880224615335464, + "grad_norm": 0.07465694099664688, + "learning_rate": 3e-05, + "num_tokens": 2828313.0, + "completions/mean_length": 646.0625, + "completions/min_length": 537.0, + "completions/max_length": 926.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 646.0625, + "completions/min_terminated_length": 537.0, + "completions/max_terminated_length": 926.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 1.879716396331787, + "reward": 6.75, + "reward_std": 1.879716396331787, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029086390510201454, + "sampling/sampling_logp_difference/max": 0.7433638572692871, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5912219882011414, + "sampling/importance_sampling_ratio/max": 2.2781238555908203, + "kl": 0.08889741986058652, + "entropy": 1.16922065615654, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.293008426669985, + "epoch": 0.55, + "step": 275 + }, + { + "loss": 0.0015373103087767959, + "grad_norm": 0.01674834080040455, + "learning_rate": 3e-05, + "num_tokens": 2840084.0, + "completions/mean_length": 637.6875, + "completions/min_length": 512.0, + "completions/max_length": 760.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 637.6875, + "completions/min_terminated_length": 512.0, + "completions/max_terminated_length": 760.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "reward": 7.0, + "reward_std": 0.0, + "frac_reward_zero_std": 1.0, + "sampling/sampling_logp_difference/mean": 0.029371261596679688, + "sampling/sampling_logp_difference/max": 0.42199182510375977, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.43593448400497437, + "sampling/importance_sampling_ratio/max": 1.5048863887786865, + "kl": 0.0760874121915549, + "entropy": 1.2918337136507034, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.80712998472154, + "epoch": 0.552, + "step": 276 + }, + { + "loss": 0.09041914343833923, + "grad_norm": 0.18090865015983582, + "learning_rate": 3e-05, + "num_tokens": 2851795.0, + "completions/mean_length": 598.4375, + "completions/min_length": 477.0, + "completions/max_length": 757.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 598.4375, + "completions/min_terminated_length": 477.0, + "completions/max_terminated_length": 757.0, + "rewards/quality_reward/mean": 7.125, + "rewards/quality_reward/std": 0.5, + "reward": 7.125, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028774311766028404, + "sampling/sampling_logp_difference/max": 0.5233750343322754, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5874509811401367, + "sampling/importance_sampling_ratio/max": 2.8326914310455322, + "kl": 0.07271571340970695, + "entropy": 1.265094794332981, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.066202180925757, + "epoch": 0.554, + "step": 277 + }, + { + "loss": -0.230061873793602, + "grad_norm": 0.23268753290176392, + "learning_rate": 3e-05, + "num_tokens": 2863051.0, + "completions/mean_length": 612.5, + "completions/min_length": 521.0, + "completions/max_length": 733.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 612.5, + "completions/min_terminated_length": 521.0, + "completions/max_terminated_length": 733.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.57373046875, + "reward": 7.0625, + "reward_std": 0.57373046875, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028104104101657867, + "sampling/sampling_logp_difference/max": 0.7588817477226257, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4506221413612366, + "sampling/importance_sampling_ratio/max": 2.076450824737549, + "kl": 0.06670796708203852, + "entropy": 1.1233563423156738, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.9273390378803, + "epoch": 0.556, + "step": 278 + }, + { + "loss": -0.07226230949163437, + "grad_norm": 0.0667547881603241, + "learning_rate": 3e-05, + "num_tokens": 2874765.0, + "completions/mean_length": 628.125, + "completions/min_length": 575.0, + "completions/max_length": 748.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 628.125, + "completions/min_terminated_length": 575.0, + "completions/max_terminated_length": 748.0, + "rewards/quality_reward/mean": 7.3125, + "rewards/quality_reward/std": 0.4787135720252991, + "reward": 7.3125, + "reward_std": 0.4787135720252991, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029238566756248474, + "sampling/sampling_logp_difference/max": 0.3805513381958008, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.49886637926101685, + "sampling/importance_sampling_ratio/max": 1.8878028392791748, + "kl": 0.06879452662542462, + "entropy": 1.2298871502280235, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.18924847058952, + "epoch": 0.558, + "step": 279 + }, + { + "loss": -0.038590408861637115, + "grad_norm": 0.14418913424015045, + "learning_rate": 3e-05, + "num_tokens": 2886149.0, + "completions/mean_length": 599.5, + "completions/min_length": 503.0, + "completions/max_length": 716.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 599.5, + "completions/min_terminated_length": 503.0, + "completions/max_terminated_length": 716.0, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.40311288833618164, + "reward": 7.1875, + "reward_std": 0.40311288833618164, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027626870200037956, + "sampling/sampling_logp_difference/max": 0.3773159980773926, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.4855646789073944, + "sampling/importance_sampling_ratio/max": 1.162408709526062, + "kl": 0.06538815144449472, + "entropy": 1.2089616432785988, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 44.65077885752544, + "epoch": 0.56, + "step": 280 + }, + { + "loss": 0.07250604778528214, + "grad_norm": 0.38874274492263794, + "learning_rate": 3e-05, + "num_tokens": 2897730.0, + "completions/mean_length": 609.8125, + "completions/min_length": 496.0, + "completions/max_length": 703.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 609.8125, + "completions/min_terminated_length": 496.0, + "completions/max_terminated_length": 703.0, + "rewards/quality_reward/mean": 6.625, + "rewards/quality_reward/std": 0.5, + "reward": 6.625, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027132270857691765, + "sampling/sampling_logp_difference/max": 0.4089934825897217, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8224037885665894, + "sampling/importance_sampling_ratio/max": 2.67866849899292, + "kl": 0.052375795086845756, + "entropy": 1.1101448722183704, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.093744847457856, + "epoch": 0.562, + "step": 281 + }, + { + "loss": 0.18828153610229492, + "grad_norm": 0.5568169355392456, + "learning_rate": 3e-05, + "num_tokens": 2908907.0, + "completions/mean_length": 597.5625, + "completions/min_length": 539.0, + "completions/max_length": 675.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 597.5625, + "completions/min_terminated_length": 539.0, + "completions/max_terminated_length": 675.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.5439056158065796, + "reward": 6.8125, + "reward_std": 0.5439056158065796, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03135806694626808, + "sampling/sampling_logp_difference/max": 0.41974592208862305, + "sampling/importance_sampling_ratio/min": 0.12026017159223557, + "sampling/importance_sampling_ratio/mean": 1.130048155784607, + "sampling/importance_sampling_ratio/max": 2.7599775791168213, + "kl": 0.06042969529516995, + "entropy": 1.4377392679452896, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.58286938164383, + "epoch": 0.564, + "step": 282 + }, + { + "loss": 0.16503384709358215, + "grad_norm": 0.10773341357707977, + "learning_rate": 3e-05, + "num_tokens": 2920914.0, + "completions/mean_length": 652.9375, + "completions/min_length": 567.0, + "completions/max_length": 843.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 652.9375, + "completions/min_terminated_length": 567.0, + "completions/max_terminated_length": 843.0, + "rewards/quality_reward/mean": 6.3125, + "rewards/quality_reward/std": 1.25, + "reward": 6.3125, + "reward_std": 1.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02949238382279873, + "sampling/sampling_logp_difference/max": 0.4577202796936035, + "sampling/importance_sampling_ratio/min": 0.02368989959359169, + "sampling/importance_sampling_ratio/mean": 0.3585829436779022, + "sampling/importance_sampling_ratio/max": 1.6338802576065063, + "kl": 0.06649435753934085, + "entropy": 1.3557419702410698, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.548663158435374, + "epoch": 0.566, + "step": 283 + }, + { + "loss": 0.14570997655391693, + "grad_norm": 0.22535188496112823, + "learning_rate": 3e-05, + "num_tokens": 2932113.0, + "completions/mean_length": 595.9375, + "completions/min_length": 503.0, + "completions/max_length": 887.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 595.9375, + "completions/min_terminated_length": 503.0, + "completions/max_terminated_length": 887.0, + "rewards/quality_reward/mean": 6.5, + "rewards/quality_reward/std": 1.7511900663375854, + "reward": 6.5, + "reward_std": 1.7511900663375854, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02905319817364216, + "sampling/sampling_logp_difference/max": 0.34877145290374756, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5842634439468384, + "sampling/importance_sampling_ratio/max": 1.3565518856048584, + "kl": 0.06671181181445718, + "entropy": 1.1663579158484936, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 22.762956948485225, + "epoch": 0.568, + "step": 284 + }, + { + "loss": -0.13110309839248657, + "grad_norm": 0.24843654036521912, + "learning_rate": 3e-05, + "num_tokens": 2942261.0, + "completions/mean_length": 523.75, + "completions/min_length": 318.0, + "completions/max_length": 791.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 523.75, + "completions/min_terminated_length": 318.0, + "completions/max_terminated_length": 791.0, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.8563488721847534, + "reward": 7.25, + "reward_std": 0.8563488721847534, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027958959341049194, + "sampling/sampling_logp_difference/max": 0.7687346339225769, + "sampling/importance_sampling_ratio/min": 0.10829401761293411, + "sampling/importance_sampling_ratio/mean": 0.6225794553756714, + "sampling/importance_sampling_ratio/max": 2.508721351623535, + "kl": 0.056764260167256, + "entropy": 1.2438273057341576, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.391164038795978, + "epoch": 0.57, + "step": 285 + }, + { + "loss": 0.0612037368118763, + "grad_norm": 0.04263085126876831, + "learning_rate": 3e-05, + "num_tokens": 2953400.0, + "completions/mean_length": 575.6875, + "completions/min_length": 498.0, + "completions/max_length": 635.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 575.6875, + "completions/min_terminated_length": 498.0, + "completions/max_terminated_length": 635.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "reward": 7.0625, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029160577803850174, + "sampling/sampling_logp_difference/max": 0.563321590423584, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.38760584592819214, + "sampling/importance_sampling_ratio/max": 1.4990487098693848, + "kl": 0.06749766110442579, + "entropy": 1.278195135295391, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 27.7587122018449, + "epoch": 0.572, + "step": 286 + }, + { + "loss": -0.042751215398311615, + "grad_norm": 0.15738150477409363, + "learning_rate": 3e-05, + "num_tokens": 2964441.0, + "completions/mean_length": 596.0625, + "completions/min_length": 499.0, + "completions/max_length": 654.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 596.0625, + "completions/min_terminated_length": 499.0, + "completions/max_terminated_length": 654.0, + "rewards/quality_reward/mean": 7.1875, + "rewards/quality_reward/std": 0.75, + "reward": 7.1875, + "reward_std": 0.75, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028714079409837723, + "sampling/sampling_logp_difference/max": 0.36002135276794434, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.594788670539856, + "sampling/importance_sampling_ratio/max": 1.975547432899475, + "kl": 0.06657169410027564, + "entropy": 1.1991046443581581, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 17.97522668587044, + "epoch": 0.574, + "step": 287 + }, + { + "loss": -0.055044494569301605, + "grad_norm": 0.13965441286563873, + "learning_rate": 3e-05, + "num_tokens": 2975274.0, + "completions/mean_length": 575.0625, + "completions/min_length": 506.0, + "completions/max_length": 673.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 575.0625, + "completions/min_terminated_length": 506.0, + "completions/max_terminated_length": 673.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.5, + "reward": 6.875, + "reward_std": 0.5, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027103029191493988, + "sampling/sampling_logp_difference/max": 0.34395354986190796, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.768635630607605, + "sampling/importance_sampling_ratio/max": 2.8281614780426025, + "kl": 0.05857925652526319, + "entropy": 1.2212486639618874, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.630491987802088, + "epoch": 0.576, + "step": 288 + }, + { + "loss": 0.2526615262031555, + "grad_norm": 0.2643212676048279, + "learning_rate": 3e-05, + "num_tokens": 2987238.0, + "completions/mean_length": 637.75, + "completions/min_length": 538.0, + "completions/max_length": 1008.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 637.75, + "completions/min_terminated_length": 538.0, + "completions/max_terminated_length": 1008.0, + "rewards/quality_reward/mean": 6.375, + "rewards/quality_reward/std": 1.8574175834655762, + "reward": 6.375, + "reward_std": 1.8574175834655762, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028626006096601486, + "sampling/sampling_logp_difference/max": 0.411831259727478, + "sampling/importance_sampling_ratio/min": 0.06620145589113235, + "sampling/importance_sampling_ratio/mean": 0.4440639913082123, + "sampling/importance_sampling_ratio/max": 1.345821738243103, + "kl": 0.049948622239753604, + "entropy": 1.1949424967169762, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 40.26761668827385, + "epoch": 0.578, + "step": 289 + }, + { + "loss": -0.28555187582969666, + "grad_norm": 0.2776772677898407, + "learning_rate": 3e-05, + "num_tokens": 2998528.0, + "completions/mean_length": 596.125, + "completions/min_length": 502.0, + "completions/max_length": 649.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 596.125, + "completions/min_terminated_length": 502.0, + "completions/max_terminated_length": 649.0, + "rewards/quality_reward/mean": 7.25, + "rewards/quality_reward/std": 0.44721361994743347, + "reward": 7.25, + "reward_std": 0.44721361994743347, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02966075949370861, + "sampling/sampling_logp_difference/max": 0.4653182029724121, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.9335561990737915, + "sampling/importance_sampling_ratio/max": 2.459705114364624, + "kl": 0.06474088784307241, + "entropy": 1.1830484792590141, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.85909090936184, + "epoch": 0.58, + "step": 290 + }, + { + "loss": 0.31567299365997314, + "grad_norm": 0.23154416680335999, + "learning_rate": 3e-05, + "num_tokens": 3009683.0, + "completions/mean_length": 593.6875, + "completions/min_length": 532.0, + "completions/max_length": 685.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 593.6875, + "completions/min_terminated_length": 532.0, + "completions/max_terminated_length": 685.0, + "rewards/quality_reward/mean": 6.5625, + "rewards/quality_reward/std": 1.412739634513855, + "reward": 6.5625, + "reward_std": 1.412739634513855, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027346499264240265, + "sampling/sampling_logp_difference/max": 0.6209149360656738, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6376235485076904, + "sampling/importance_sampling_ratio/max": 2.0061569213867188, + "kl": 0.05466599250212312, + "entropy": 1.0875738225877285, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 21.547887252643704, + "epoch": 0.582, + "step": 291 + }, + { + "loss": 0.036074671894311905, + "grad_norm": 0.2807851731777191, + "learning_rate": 3e-05, + "num_tokens": 3020003.0, + "completions/mean_length": 552.5, + "completions/min_length": 458.0, + "completions/max_length": 597.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 552.5, + "completions/min_terminated_length": 458.0, + "completions/max_terminated_length": 597.0, + "rewards/quality_reward/mean": 6.8125, + "rewards/quality_reward/std": 0.8341662883758545, + "reward": 6.8125, + "reward_std": 0.8341662883758545, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02795255184173584, + "sampling/sampling_logp_difference/max": 0.3504910469055176, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5984793901443481, + "sampling/importance_sampling_ratio/max": 2.0109286308288574, + "kl": 0.05562997469678521, + "entropy": 1.238592490553856, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.77090792078525, + "epoch": 0.584, + "step": 292 + }, + { + "loss": -0.1038510799407959, + "grad_norm": 0.15615873038768768, + "learning_rate": 3e-05, + "num_tokens": 3031240.0, + "completions/mean_length": 592.3125, + "completions/min_length": 518.0, + "completions/max_length": 658.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 592.3125, + "completions/min_terminated_length": 518.0, + "completions/max_terminated_length": 658.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.4425306022167206, + "reward": 7.0625, + "reward_std": 0.4425306022167206, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.03072209097445011, + "sampling/sampling_logp_difference/max": 0.7037668228149414, + "sampling/importance_sampling_ratio/min": 0.007596802897751331, + "sampling/importance_sampling_ratio/mean": 0.544894814491272, + "sampling/importance_sampling_ratio/max": 1.661234736442566, + "kl": 0.054585910867899656, + "entropy": 1.180466279387474, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 19.1619019433856, + "epoch": 0.586, + "step": 293 + }, + { + "loss": -0.12249961495399475, + "grad_norm": 0.16230997443199158, + "learning_rate": 3e-05, + "num_tokens": 3042436.0, + "completions/mean_length": 571.75, + "completions/min_length": 497.0, + "completions/max_length": 645.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 571.75, + "completions/min_terminated_length": 497.0, + "completions/max_terminated_length": 645.0, + "rewards/quality_reward/mean": 6.4375, + "rewards/quality_reward/std": 0.8920949101448059, + "reward": 6.4375, + "reward_std": 0.8920949101448059, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028705568984150887, + "sampling/sampling_logp_difference/max": 0.5716366767883301, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.621848464012146, + "sampling/importance_sampling_ratio/max": 2.383788585662842, + "kl": 0.05031474749557674, + "entropy": 1.3075302839279175, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 20.532019450329244, + "epoch": 0.588, + "step": 294 + }, + { + "loss": 0.14228732883930206, + "grad_norm": 0.10194012522697449, + "learning_rate": 3e-05, + "num_tokens": 3053575.0, + "completions/mean_length": 581.1875, + "completions/min_length": 492.0, + "completions/max_length": 716.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 581.1875, + "completions/min_terminated_length": 492.0, + "completions/max_terminated_length": 716.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "reward": 7.0625, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.028240300714969635, + "sampling/sampling_logp_difference/max": 0.408158540725708, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.5425832271575928, + "sampling/importance_sampling_ratio/max": 2.1675894260406494, + "kl": 0.0485304044559598, + "entropy": 1.3449261263012886, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 26.749822621699423, + "epoch": 0.59, + "step": 295 + }, + { + "loss": -0.013912655413150787, + "grad_norm": 0.32519620656967163, + "learning_rate": 3e-05, + "num_tokens": 3064324.0, + "completions/mean_length": 549.8125, + "completions/min_length": 501.0, + "completions/max_length": 613.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 549.8125, + "completions/min_terminated_length": 501.0, + "completions/max_terminated_length": 613.0, + "rewards/quality_reward/mean": 6.875, + "rewards/quality_reward/std": 0.3415650427341461, + "reward": 6.875, + "reward_std": 0.3415650427341461, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.029748085886240005, + "sampling/sampling_logp_difference/max": 0.37021374702453613, + "sampling/importance_sampling_ratio/min": 0.05226827412843704, + "sampling/importance_sampling_ratio/mean": 0.8356897830963135, + "sampling/importance_sampling_ratio/max": 2.0540385246276855, + "kl": 0.05967968609184027, + "entropy": 1.345760464668274, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 16.617265206295997, + "epoch": 0.592, + "step": 296 + }, + { + "loss": -0.4677557647228241, + "grad_norm": 0.6430356502532959, + "learning_rate": 3e-05, + "num_tokens": 3075172.0, + "completions/mean_length": 579.0, + "completions/min_length": 489.0, + "completions/max_length": 662.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 579.0, + "completions/min_terminated_length": 489.0, + "completions/max_terminated_length": 662.0, + "rewards/quality_reward/mean": 7.0625, + "rewards/quality_reward/std": 0.25, + "reward": 7.0625, + "reward_std": 0.25, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.02859537862241268, + "sampling/sampling_logp_difference/max": 0.4263639450073242, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.8718785047531128, + "sampling/importance_sampling_ratio/max": 2.9863228797912598, + "kl": 0.04937166138552129, + "entropy": 1.1249969974160194, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 25.6860204776749, + "epoch": 0.594, + "step": 297 + }, + { + "loss": 0.03550681099295616, + "grad_norm": 0.28020939230918884, + "learning_rate": 3e-05, + "num_tokens": 3085846.0, + "completions/mean_length": 555.625, + "completions/min_length": 450.0, + "completions/max_length": 710.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 555.625, + "completions/min_terminated_length": 450.0, + "completions/max_terminated_length": 710.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.026732780039310455, + "sampling/sampling_logp_difference/max": 0.35025501251220703, + "sampling/importance_sampling_ratio/min": 0.08425833284854889, + "sampling/importance_sampling_ratio/mean": 0.7580450177192688, + "sampling/importance_sampling_ratio/max": 2.5230090618133545, + "kl": 0.035641232039779425, + "entropy": 1.1057527735829353, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 18.157300523016602, + "epoch": 0.596, + "step": 298 + }, + { + "loss": -0.09046114981174469, + "grad_norm": 0.17403694987297058, + "learning_rate": 3e-05, + "num_tokens": 3096033.0, + "completions/mean_length": 530.1875, + "completions/min_length": 487.0, + "completions/max_length": 598.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 530.1875, + "completions/min_terminated_length": 487.0, + "completions/max_terminated_length": 598.0, + "rewards/quality_reward/mean": 6.75, + "rewards/quality_reward/std": 0.8563488721847534, + "reward": 6.75, + "reward_std": 0.8563488721847534, + "frac_reward_zero_std": 0.0, + "sampling/sampling_logp_difference/mean": 0.027955062687397003, + "sampling/sampling_logp_difference/max": 0.314223051071167, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6789309978485107, + "sampling/importance_sampling_ratio/max": 2.300510883331299, + "kl": 0.04222825774922967, + "entropy": 1.1326002702116966, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 15.391770029906183, + "epoch": 0.598, + "step": 299 + }, + { + "loss": 0.0007643185090273619, + "grad_norm": 0.007904416881501675, + "learning_rate": 3e-05, + "num_tokens": 3106243.0, + "completions/mean_length": 535.125, + "completions/min_length": 459.0, + "completions/max_length": 613.0, + "completions/clipped_ratio": 0.0, + "completions/mean_terminated_length": 535.125, + "completions/min_terminated_length": 459.0, + "completions/max_terminated_length": 613.0, + "rewards/quality_reward/mean": 7.0, + "rewards/quality_reward/std": 0.0, + "reward": 7.0, + "reward_std": 0.0, + "frac_reward_zero_std": 1.0, + "sampling/sampling_logp_difference/mean": 0.027106385678052902, + "sampling/sampling_logp_difference/max": 0.4390592575073242, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/importance_sampling_ratio/mean": 0.6089779138565063, + "sampling/importance_sampling_ratio/max": 1.4233957529067993, + "kl": 0.03820930456276983, + "entropy": 1.1391064934432507, + "clip_ratio/low_mean": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/region_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/high_max": 0.0, + "step_time": 23.217237341683358, + "epoch": 0.6, + "step": 300 + }, + { + "train_runtime": 7341.7828, + "train_samples_per_second": 0.654, + "train_steps_per_second": 0.041, + "total_flos": 0.0, + "train_loss": 0.01336301638240305, + "epoch": 0.6, + "step": 300 + } +] \ No newline at end of file