Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,60 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "dtype": "float32",
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 40960,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "do_sample": true,
+  "eos_token_id": [
+    151645
+  ],
+  "max_new_tokens": 256,
+  "no_repeat_ngram_size": 3,
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.2,
+  "temperature": 0.8,
+  "top_p": 0.9,
+  "transformers_version": "4.57.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12653519e7d39024353f4a93f19edf37f54caf6ffc07c68b6be6b33cb93f0d98
+size 2384234968

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16f5d31ce3f94b227a9d4aa959feefa425365447269aff6899cca46566d6ead4
+size 4768669395

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35059eaf262b99e25d2c89d034a29d2e08b6edbee1cb70e6dc4f5d5c11035ca2
+size 15877

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fe6a4b07713b30a25984a8dfee04d369213e2b4f829b83a3b5317cad7b9545a
+size 15877

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d76f48ea09dc133c9ecbc6e1c2ed10f9450cfd601b85b1ae1b0c11c8427a7db
+size 15877

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:625137f2eb94fef4201a4c4649e55b1db1f7f74a2aeb909aaaaf7dc793a74ce5
+size 15941

rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:428bdf950c87573bc2ea172844984571a5b17561d48f684620ca1e992f2d2c7c
+size 15877

rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efdde4beb41e3d4ddbb9aba88a5ac0b95b701500c1e28655ce4c10888b4ae952
+size 15877

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bc307f08d8a3dc6e024728ee9eed82e329c09ca8e6fe88c614e51e39f1bca91
+size 1465

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67cc0080ffd7555f723f423c27cfef314e1ad9d335c8b79f465c5faba1ed478b
+size 11422821

tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,2274 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.8,
+  "eval_steps": 500,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6791666865348815,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 247.0,
+      "completions/mean_length": 236.5416717529297,
+      "completions/mean_terminated_length": 200.61068420410157,
+      "completions/min_length": 127.2,
+      "completions/min_terminated_length": 127.2,
+      "entropy": 1.070425021648407,
+      "epoch": 0.01,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.5985119342803955,
+      "kl": 0.0035211448557674885,
+      "learning_rate": 4.980000000000001e-06,
+      "loss": 0.0011,
+      "num_tokens": 73826.0,
+      "reward": 0.2776068687438965,
+      "reward_std": 0.5232907950878143,
+      "rewards/politeness_reward_func/mean": 0.27760685607790947,
+      "rewards/politeness_reward_func/std": 0.5940477013587951,
+      "step": 5,
+      "step_time": 11.145096338540316
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6208333492279052,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 246.8,
+      "completions/mean_length": 232.80417175292968,
+      "completions/mean_terminated_length": 204.57913513183593,
+      "completions/min_length": 148.6,
+      "completions/min_terminated_length": 148.6,
+      "entropy": 1.070945155620575,
+      "epoch": 0.02,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.3468329906463623,
+      "kl": 0.02206921111792326,
+      "learning_rate": 4.955e-06,
+      "loss": 0.0224,
+      "num_tokens": 147283.0,
+      "reward": 0.6233123302459717,
+      "reward_std": 0.4648557245731354,
+      "rewards/politeness_reward_func/mean": 0.6233122944831848,
+      "rewards/politeness_reward_func/std": 0.553675526380539,
+      "step": 10,
+      "step_time": 10.835208699852227
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5000000119209289,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.6,
+      "completions/mean_length": 221.8000030517578,
+      "completions/mean_terminated_length": 188.9316619873047,
+      "completions/min_length": 110.8,
+      "completions/min_terminated_length": 110.8,
+      "entropy": 1.1163462162017823,
+      "epoch": 0.03,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.3379204273223877,
+      "kl": 0.04084419272840023,
+      "learning_rate": 4.93e-06,
+      "loss": -0.0052,
+      "num_tokens": 218131.0,
+      "reward": 0.8122925400733948,
+      "reward_std": 0.41203114986419676,
+      "rewards/politeness_reward_func/mean": 0.8122925400733948,
+      "rewards/politeness_reward_func/std": 0.4989388585090637,
+      "step": 15,
+      "step_time": 10.806195054203272
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6791666865348815,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 251.6,
+      "completions/mean_length": 238.19584045410156,
+      "completions/mean_terminated_length": 201.80428771972657,
+      "completions/min_length": 130.2,
+      "completions/min_terminated_length": 130.2,
+      "entropy": 1.1251786351203918,
+      "epoch": 0.04,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.3837928771972656,
+      "kl": 0.0385904410853982,
+      "learning_rate": 4.9050000000000005e-06,
+      "loss": -0.0031,
+      "num_tokens": 292306.0,
+      "reward": 0.6734941720962524,
+      "reward_std": 0.448628181219101,
+      "rewards/politeness_reward_func/mean": 0.6734941720962524,
+      "rewards/politeness_reward_func/std": 0.509108817577362,
+      "step": 20,
+      "step_time": 10.808321141451597
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.7333333551883697,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 246.6,
+      "completions/mean_length": 239.7041748046875,
+      "completions/mean_terminated_length": 201.94395141601564,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
+      "entropy": 1.0987902224063872,
+      "epoch": 0.05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.3908112049102783,
+      "kl": 0.04714170750230551,
+      "learning_rate": 4.880000000000001e-06,
+      "loss": 0.013,
+      "num_tokens": 366731.0,
+      "reward": 0.7238378047943115,
+      "reward_std": 0.5012153267860413,
+      "rewards/politeness_reward_func/mean": 0.7238377809524537,
+      "rewards/politeness_reward_func/std": 0.5635832965373992,
+      "step": 25,
+      "step_time": 10.778460966795683
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.7041666984558106,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 251.4,
+      "completions/mean_length": 238.33334045410157,
+      "completions/mean_terminated_length": 209.13802490234374,
+      "completions/min_length": 135.6,
+      "completions/min_terminated_length": 135.6,
+      "entropy": 1.0942281484603882,
+      "epoch": 0.06,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.272942304611206,
+      "kl": 0.04730561040341854,
+      "learning_rate": 4.855e-06,
+      "loss": -0.0003,
+      "num_tokens": 441211.0,
+      "reward": 0.7387123763561249,
+      "reward_std": 0.3938853979110718,
+      "rewards/politeness_reward_func/mean": 0.7387123882770539,
+      "rewards/politeness_reward_func/std": 0.5152743279933929,
+      "step": 30,
+      "step_time": 10.791689745336772
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6000000178813935,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.8,
+      "completions/mean_length": 232.95417175292968,
+      "completions/mean_terminated_length": 200.11821594238282,
+      "completions/min_length": 126.0,
+      "completions/min_terminated_length": 126.0,
+      "entropy": 1.1196449041366576,
+      "epoch": 0.07,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.515143394470215,
+      "kl": 0.055315449833869934,
+      "learning_rate": 4.83e-06,
+      "loss": 0.0018,
+      "num_tokens": 514144.0,
+      "reward": 0.9808868050575257,
+      "reward_std": 0.3419617712497711,
+      "rewards/politeness_reward_func/mean": 0.9808867692947387,
+      "rewards/politeness_reward_func/std": 0.4772939085960388,
+      "step": 35,
+      "step_time": 10.767037071287632
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.4333333432674408,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 253.2,
+      "completions/mean_length": 216.89583740234374,
+      "completions/mean_terminated_length": 186.88653869628905,
+      "completions/min_length": 90.4,
+      "completions/min_terminated_length": 90.4,
+      "entropy": 1.062067198753357,
+      "epoch": 0.08,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.2335519790649414,
+      "kl": 0.06222313120961189,
+      "learning_rate": 4.805000000000001e-06,
+      "loss": 0.0425,
+      "num_tokens": 583095.0,
+      "reward": 0.873325777053833,
+      "reward_std": 0.4448536515235901,
+      "rewards/politeness_reward_func/mean": 0.873325777053833,
+      "rewards/politeness_reward_func/std": 0.49484293460845946,
+      "step": 40,
+      "step_time": 10.696332201361656
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5666666746139526,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 248.6,
+      "completions/mean_length": 220.82500610351562,
+      "completions/mean_terminated_length": 175.39512634277344,
+      "completions/min_length": 75.6,
+      "completions/min_terminated_length": 75.6,
+      "entropy": 1.0480997025966645,
+      "epoch": 0.09,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.452233076095581,
+      "kl": 0.07400040999054909,
+      "learning_rate": 4.78e-06,
+      "loss": 0.0107,
+      "num_tokens": 653373.0,
+      "reward": 0.9472709178924561,
+      "reward_std": 0.40665341913700104,
+      "rewards/politeness_reward_func/mean": 0.9472708940505982,
+      "rewards/politeness_reward_func/std": 0.4510787308216095,
+      "step": 45,
+      "step_time": 10.80831560343504
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5416666865348816,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.8,
+      "completions/mean_length": 208.1916717529297,
+      "completions/mean_terminated_length": 153.92529907226563,
+      "completions/min_length": 61.8,
+      "completions/min_terminated_length": 61.8,
+      "entropy": 1.1236772537231445,
+      "epoch": 0.1,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.160369634628296,
+      "kl": 0.08055943846702576,
+      "learning_rate": 4.755e-06,
+      "loss": 0.0611,
+      "num_tokens": 720619.0,
+      "reward": 0.8770254850387573,
+      "reward_std": 0.5482801616191864,
+      "rewards/politeness_reward_func/mean": 0.8770254492759705,
+      "rewards/politeness_reward_func/std": 0.5923300921916962,
+      "step": 50,
+      "step_time": 10.79710350483656
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.4083333373069763,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 245.4,
+      "completions/mean_length": 180.80833740234374,
+      "completions/mean_terminated_length": 134.77063598632813,
+      "completions/min_length": 46.0,
+      "completions/min_terminated_length": 46.0,
+      "entropy": 1.2555986642837524,
+      "epoch": 0.11,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.3433427810668945,
+      "kl": 0.09562707468867301,
+      "learning_rate": 4.7300000000000005e-06,
+      "loss": 0.0773,
+      "num_tokens": 781261.0,
+      "reward": 1.092347764968872,
+      "reward_std": 0.4649295687675476,
+      "rewards/politeness_reward_func/mean": 1.0923476934432983,
+      "rewards/politeness_reward_func/std": 0.5502241551876068,
+      "step": 55,
+      "step_time": 10.756779878586531
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2875000063329935,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 238.8,
+      "completions/mean_length": 154.1916702270508,
+      "completions/mean_terminated_length": 119.38919677734376,
+      "completions/min_length": 42.8,
+      "completions/min_terminated_length": 42.8,
+      "entropy": 1.3319262266159058,
+      "epoch": 0.12,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.3366644382476807,
+      "kl": 0.14686549603939056,
+      "learning_rate": 4.705e-06,
+      "loss": 0.0456,
+      "num_tokens": 835835.0,
+      "reward": 1.4423248767852783,
+      "reward_std": 0.36013287007808686,
+      "rewards/politeness_reward_func/mean": 1.4423248767852783,
+      "rewards/politeness_reward_func/std": 0.40978127121925356,
+      "step": 60,
+      "step_time": 10.825969719141721
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.41250001192092894,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 247.4,
+      "completions/mean_length": 183.07083740234376,
+      "completions/mean_terminated_length": 133.48724670410155,
+      "completions/min_length": 46.8,
+      "completions/min_terminated_length": 46.8,
+      "entropy": 1.3182121157646178,
+      "epoch": 0.13,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 25.555892944335938,
+      "kl": 0.17301606237888337,
+      "learning_rate": 4.680000000000001e-06,
+      "loss": 0.0985,
+      "num_tokens": 896732.0,
+      "reward": 1.3051372289657592,
+      "reward_std": 0.43976340293884275,
+      "rewards/politeness_reward_func/mean": 1.3051372170448303,
+      "rewards/politeness_reward_func/std": 0.5006832242012024,
+      "step": 65,
+      "step_time": 10.645895641297102
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2708333432674408,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 246.4,
+      "completions/mean_length": 157.8791717529297,
+      "completions/mean_terminated_length": 123.51410827636718,
+      "completions/min_length": 41.4,
+      "completions/min_terminated_length": 41.4,
+      "entropy": 1.2897038578987121,
+      "epoch": 0.14,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.8760225772857666,
+      "kl": 0.19847530573606492,
+      "learning_rate": 4.655e-06,
+      "loss": 0.1906,
+      "num_tokens": 951999.0,
+      "reward": 1.420637583732605,
+      "reward_std": 0.430054372549057,
+      "rewards/politeness_reward_func/mean": 1.420637583732605,
+      "rewards/politeness_reward_func/std": 0.48181390166282656,
+      "step": 70,
+      "step_time": 10.596996573358775
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2083333432674408,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 240.4,
+      "completions/mean_length": 146.60000610351562,
+      "completions/mean_terminated_length": 118.93135223388671,
+      "completions/min_length": 43.2,
+      "completions/min_terminated_length": 43.2,
+      "entropy": 1.4299420952796935,
+      "epoch": 0.15,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.174029350280762,
+      "kl": 0.20222627967596055,
+      "learning_rate": 4.6300000000000006e-06,
+      "loss": 0.1187,
+      "num_tokens": 1004287.0,
+      "reward": 1.5081899881362915,
+      "reward_std": 0.33264915347099305,
+      "rewards/politeness_reward_func/mean": 1.508189868927002,
+      "rewards/politeness_reward_func/std": 0.36398229002952576,
+      "step": 75,
+      "step_time": 10.52198056280613
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.12916667088866235,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 242.6,
+      "completions/mean_length": 142.55417175292968,
+      "completions/mean_terminated_length": 126.68900451660156,
+      "completions/min_length": 48.2,
+      "completions/min_terminated_length": 48.2,
+      "entropy": 1.3346869945526123,
+      "epoch": 0.16,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.63394021987915,
+      "kl": 0.19363965839147568,
+      "learning_rate": 4.605000000000001e-06,
+      "loss": 0.0898,
+      "num_tokens": 1056308.0,
+      "reward": 1.459042239189148,
+      "reward_std": 0.2736098259687424,
+      "rewards/politeness_reward_func/mean": 1.45904221534729,
+      "rewards/politeness_reward_func/std": 0.3277650147676468,
+      "step": 80,
+      "step_time": 10.517034325748682
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.20416666716337203,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 241.2,
+      "completions/mean_length": 137.8166717529297,
+      "completions/mean_terminated_length": 107.75043640136718,
+      "completions/min_length": 39.2,
+      "completions/min_terminated_length": 39.2,
+      "entropy": 1.0585298597812653,
+      "epoch": 0.17,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.915429592132568,
+      "kl": 0.2862268269062042,
+      "learning_rate": 4.58e-06,
+      "loss": 0.2369,
+      "num_tokens": 1106584.0,
+      "reward": 1.339914321899414,
+      "reward_std": 0.42014909982681276,
+      "rewards/politeness_reward_func/mean": 1.3399142742156982,
+      "rewards/politeness_reward_func/std": 0.46948710083961487,
+      "step": 85,
+      "step_time": 10.558716806769372
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.26250001005828383,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 232.6,
+      "completions/mean_length": 154.93750457763673,
+      "completions/mean_terminated_length": 121.31997375488281,
+      "completions/min_length": 47.0,
+      "completions/min_terminated_length": 47.0,
+      "entropy": 1.208097517490387,
+      "epoch": 0.18,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.7058539390563965,
+      "kl": 0.17084270417690278,
+      "learning_rate": 4.5550000000000004e-06,
+      "loss": 0.0854,
+      "num_tokens": 1161209.0,
+      "reward": 1.4180654048919679,
+      "reward_std": 0.24261603355407715,
+      "rewards/politeness_reward_func/mean": 1.4180653095245361,
+      "rewards/politeness_reward_func/std": 0.3126596033573151,
+      "step": 90,
+      "step_time": 10.573557010293007
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15000000707805156,
+      "completions/max_length": 245.4,
+      "completions/max_terminated_length": 235.8,
+      "completions/mean_length": 145.20000457763672,
+      "completions/mean_terminated_length": 128.7294189453125,
+      "completions/min_length": 50.8,
+      "completions/min_terminated_length": 50.8,
+      "entropy": 1.2446547746658325,
+      "epoch": 0.19,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.650216579437256,
+      "kl": 0.19710368812084197,
+      "learning_rate": 4.530000000000001e-06,
+      "loss": 0.1361,
+      "num_tokens": 1213177.0,
+      "reward": 1.3565361022949218,
+      "reward_std": 0.3620565414428711,
+      "rewards/politeness_reward_func/mean": 1.356536078453064,
+      "rewards/politeness_reward_func/std": 0.4186973810195923,
+      "step": 95,
+      "step_time": 10.10843729674816
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15416666865348816,
+      "completions/max_length": 253.2,
+      "completions/max_terminated_length": 239.6,
+      "completions/mean_length": 127.69167022705078,
+      "completions/mean_terminated_length": 104.76427154541015,
+      "completions/min_length": 33.6,
+      "completions/min_terminated_length": 33.6,
+      "entropy": 1.21082746386528,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.778512477874756,
+      "kl": 0.23306063413619996,
+      "learning_rate": 4.505e-06,
+      "loss": 0.1579,
+      "num_tokens": 1261199.0,
+      "reward": 1.4624977350234984,
+      "reward_std": 0.27724905014038087,
+      "rewards/politeness_reward_func/mean": 1.4624976634979248,
+      "rewards/politeness_reward_func/std": 0.353417181968689,
+      "step": 100,
+      "step_time": 10.490663215517998
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2916666746139526,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 239.8,
+      "completions/mean_length": 164.0000030517578,
+      "completions/mean_terminated_length": 131.73128814697264,
+      "completions/min_length": 52.0,
+      "completions/min_terminated_length": 52.0,
+      "entropy": 1.227693748474121,
+      "epoch": 0.21,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.480128765106201,
+      "kl": 0.21182419136166572,
+      "learning_rate": 4.48e-06,
+      "loss": 0.1385,
+      "num_tokens": 1317887.0,
+      "reward": 1.4506232976913451,
+      "reward_std": 0.38838809728622437,
+      "rewards/politeness_reward_func/mean": 1.4506232500076295,
+      "rewards/politeness_reward_func/std": 0.415321010351181,
+      "step": 105,
+      "step_time": 10.568055501580238
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2875000134110451,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 246.6,
+      "completions/mean_length": 164.9791717529297,
+      "completions/mean_terminated_length": 129.34453430175782,
+      "completions/min_length": 50.8,
+      "completions/min_terminated_length": 50.8,
+      "entropy": 1.2591872215270996,
+      "epoch": 0.22,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.477797985076904,
+      "kl": 0.21972680687904358,
+      "learning_rate": 4.4550000000000005e-06,
+      "loss": 0.118,
+      "num_tokens": 1374570.0,
+      "reward": 1.3832386493682862,
+      "reward_std": 0.36756529808044436,
+      "rewards/politeness_reward_func/mean": 1.3832386016845704,
+      "rewards/politeness_reward_func/std": 0.45387519598007203,
+      "step": 110,
+      "step_time": 10.580036842823029
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.4041666805744171,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 239.6,
+      "completions/mean_length": 189.48334045410155,
+      "completions/mean_terminated_length": 144.77314147949218,
+      "completions/min_length": 55.6,
+      "completions/min_terminated_length": 55.6,
+      "entropy": 1.4673572778701782,
+      "epoch": 0.23,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.5870041847229,
+      "kl": 0.18002651631832123,
+      "learning_rate": 4.430000000000001e-06,
+      "loss": 0.0902,
+      "num_tokens": 1437278.0,
+      "reward": 1.463021945953369,
+      "reward_std": 0.36559332013130186,
+      "rewards/politeness_reward_func/mean": 1.4630219221115113,
+      "rewards/politeness_reward_func/std": 0.4403323769569397,
+      "step": 115,
+      "step_time": 10.606720576435327
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.29166667759418485,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.4,
+      "completions/mean_length": 168.28750610351562,
+      "completions/mean_terminated_length": 134.1982864379883,
+      "completions/min_length": 45.0,
+      "completions/min_terminated_length": 45.0,
+      "entropy": 1.439770805835724,
+      "epoch": 0.24,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.188452243804932,
+      "kl": 0.23004925400018691,
+      "learning_rate": 4.405e-06,
+      "loss": 0.1038,
+      "num_tokens": 1495123.0,
+      "reward": 1.6009358167648315,
+      "reward_std": 0.3070066928863525,
+      "rewards/politeness_reward_func/mean": 1.6009357929229737,
+      "rewards/politeness_reward_func/std": 0.34360362887382506,
+      "step": 120,
+      "step_time": 10.64953521117568
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1833333395421505,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 243.2,
+      "completions/mean_length": 140.8666748046875,
+      "completions/mean_terminated_length": 116.34450073242188,
+      "completions/min_length": 37.8,
+      "completions/min_terminated_length": 37.8,
+      "entropy": 1.2707997798919677,
+      "epoch": 0.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.1139140129089355,
+      "kl": 0.2768221229314804,
+      "learning_rate": 4.38e-06,
+      "loss": 0.146,
+      "num_tokens": 1546755.0,
+      "reward": 1.528564214706421,
+      "reward_std": 0.29817952811717985,
+      "rewards/politeness_reward_func/mean": 1.528564167022705,
+      "rewards/politeness_reward_func/std": 0.353902006149292,
+      "step": 125,
+      "step_time": 10.536190098524093
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.12500000298023223,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 228.6,
+      "completions/mean_length": 126.03333587646485,
+      "completions/mean_terminated_length": 108.7752197265625,
+      "completions/min_length": 37.6,
+      "completions/min_terminated_length": 37.6,
+      "entropy": 1.239863795042038,
+      "epoch": 0.26,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 20.54998207092285,
+      "kl": 0.4258181616663933,
+      "learning_rate": 4.355000000000001e-06,
+      "loss": 0.205,
+      "num_tokens": 1594475.0,
+      "reward": 1.5162237882614136,
+      "reward_std": 0.37349834442138674,
+      "rewards/politeness_reward_func/mean": 1.5162237882614136,
+      "rewards/politeness_reward_func/std": 0.4025087893009186,
+      "step": 130,
+      "step_time": 10.561264203488827
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1375000063329935,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 235.4,
+      "completions/mean_length": 121.34167175292968,
+      "completions/mean_terminated_length": 100.33185272216797,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.44633287191391,
+      "epoch": 0.27,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.6437859535217285,
+      "kl": 0.2876953512430191,
+      "learning_rate": 4.33e-06,
+      "loss": 0.1474,
+      "num_tokens": 1640909.0,
+      "reward": 1.6570862054824829,
+      "reward_std": 0.35202938318252563,
+      "rewards/politeness_reward_func/mean": 1.657086157798767,
+      "rewards/politeness_reward_func/std": 0.4086485207080841,
+      "step": 135,
+      "step_time": 10.50367113724351
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25000000596046446,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 232.4,
+      "completions/mean_length": 165.46250610351564,
+      "completions/mean_terminated_length": 134.87125549316406,
+      "completions/min_length": 50.2,
+      "completions/min_terminated_length": 50.2,
+      "entropy": 1.5645981192588807,
+      "epoch": 0.28,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.930655479431152,
+      "kl": 0.20734833329916,
+      "learning_rate": 4.305e-06,
+      "loss": 0.1052,
+      "num_tokens": 1698396.0,
+      "reward": 1.5796484470367431,
+      "reward_std": 0.39563443660736086,
+      "rewards/politeness_reward_func/mean": 1.5796483516693116,
+      "rewards/politeness_reward_func/std": 0.43083653450012205,
+      "step": 140,
+      "step_time": 10.580496142059564
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3666666731238365,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 245.4,
+      "completions/mean_length": 182.1541717529297,
+      "completions/mean_terminated_length": 148.0070785522461,
+      "completions/min_length": 67.8,
+      "completions/min_terminated_length": 67.8,
+      "entropy": 1.4628460884094239,
+      "epoch": 0.29,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.1963322162628174,
+      "kl": 0.20285871326923371,
+      "learning_rate": 4.2800000000000005e-06,
+      "loss": 0.0054,
+      "num_tokens": 1759217.0,
+      "reward": 1.5254708528518677,
+      "reward_std": 0.3578204929828644,
+      "rewards/politeness_reward_func/mean": 1.5254708051681518,
+      "rewards/politeness_reward_func/std": 0.39698067903518675,
+      "step": 145,
+      "step_time": 10.641796179115772
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.325000011920929,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 243.8,
+      "completions/mean_length": 185.4041717529297,
+      "completions/mean_terminated_length": 151.72132263183593,
+      "completions/min_length": 54.2,
+      "completions/min_terminated_length": 54.2,
+      "entropy": 1.550300669670105,
+      "epoch": 0.3,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.5268337726593018,
+      "kl": 0.26886349618434907,
+      "learning_rate": 4.255e-06,
+      "loss": 0.0341,
+      "num_tokens": 1821250.0,
+      "reward": 1.5432605028152466,
+      "reward_std": 0.35273998975753784,
+      "rewards/politeness_reward_func/mean": 1.5432604551315308,
+      "rewards/politeness_reward_func/std": 0.39439594745635986,
+      "step": 150,
+      "step_time": 10.68586125895381
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.20416667610406875,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 241.8,
+      "completions/mean_length": 162.42083740234375,
+      "completions/mean_terminated_length": 139.60369567871095,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 1.665429162979126,
+      "epoch": 0.31,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.6844170093536377,
+      "kl": 0.22049252390861512,
+      "learning_rate": 4.23e-06,
+      "loss": 0.0256,
+      "num_tokens": 1877383.0,
+      "reward": 1.6982928276062013,
+      "reward_std": 0.30497619807720183,
+      "rewards/politeness_reward_func/mean": 1.6982927560806274,
+      "rewards/politeness_reward_func/std": 0.3323679566383362,
+      "step": 155,
+      "step_time": 10.646523833274841
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.12083333805203438,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 238.4,
+      "completions/mean_length": 141.9625045776367,
+      "completions/mean_terminated_length": 126.35040893554688,
+      "completions/min_length": 53.0,
+      "completions/min_terminated_length": 53.0,
+      "entropy": 1.493414855003357,
+      "epoch": 0.32,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.583273410797119,
+      "kl": 0.25222020596265793,
+      "learning_rate": 4.205e-06,
+      "loss": 0.0511,
+      "num_tokens": 1928318.0,
+      "reward": 1.6409826755523682,
+      "reward_std": 0.2959387719631195,
+      "rewards/politeness_reward_func/mean": 1.6409826040267945,
+      "rewards/politeness_reward_func/std": 0.3353587478399277,
+      "step": 160,
+      "step_time": 10.597700411826372
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.22916667088866233,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 248.2,
+      "completions/mean_length": 159.16250762939453,
+      "completions/mean_terminated_length": 130.86475067138673,
+      "completions/min_length": 52.4,
+      "completions/min_terminated_length": 52.4,
+      "entropy": 1.5048229694366455,
+      "epoch": 0.33,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 8.112273216247559,
+      "kl": 0.26160909086465833,
+      "learning_rate": 4.18e-06,
+      "loss": 0.1583,
+      "num_tokens": 1983861.0,
+      "reward": 1.5394394874572754,
+      "reward_std": 0.4292898178100586,
+      "rewards/politeness_reward_func/mean": 1.5394394636154174,
+      "rewards/politeness_reward_func/std": 0.4557326793670654,
+      "step": 165,
+      "step_time": 10.650840406119823
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3583333373069763,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.8,
+      "completions/mean_length": 181.42084045410155,
+      "completions/mean_terminated_length": 142.14824829101562,
+      "completions/min_length": 47.6,
+      "completions/min_terminated_length": 47.6,
+      "entropy": 1.6774910688400269,
+      "epoch": 0.34,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.154572486877441,
+      "kl": 0.21350446194410325,
+      "learning_rate": 4.155e-06,
+      "loss": 0.0845,
+      "num_tokens": 2044922.0,
+      "reward": 1.617796039581299,
+      "reward_std": 0.42567009329795835,
+      "rewards/politeness_reward_func/mean": 1.617796039581299,
+      "rewards/politeness_reward_func/std": 0.5039380311965942,
+      "step": 170,
+      "step_time": 10.696013970673084
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.21666667759418487,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 235.8,
+      "completions/mean_length": 150.75000610351563,
+      "completions/mean_terminated_length": 122.61416320800781,
+      "completions/min_length": 41.4,
+      "completions/min_terminated_length": 41.4,
+      "entropy": 1.7118729948997498,
+      "epoch": 0.35,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.096772193908691,
+      "kl": 0.24443377554416656,
+      "learning_rate": 4.13e-06,
+      "loss": 0.062,
+      "num_tokens": 2098654.0,
+      "reward": 1.7112194776535035,
+      "reward_std": 0.41009202003479006,
+      "rewards/politeness_reward_func/mean": 1.7112194299697876,
+      "rewards/politeness_reward_func/std": 0.46850563287734986,
+      "step": 175,
+      "step_time": 10.681096732616425
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.24583333544433117,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.4,
+      "completions/mean_length": 161.39583892822264,
+      "completions/mean_terminated_length": 134.3897247314453,
+      "completions/min_length": 56.0,
+      "completions/min_terminated_length": 56.0,
+      "entropy": 1.6281691670417786,
+      "epoch": 0.36,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 39.266761779785156,
+      "kl": 0.3168014049530029,
+      "learning_rate": 4.1050000000000005e-06,
+      "loss": 0.0628,
+      "num_tokens": 2154557.0,
+      "reward": 1.592983651161194,
+      "reward_std": 0.4036149501800537,
+      "rewards/politeness_reward_func/mean": 1.5929836750030517,
+      "rewards/politeness_reward_func/std": 0.4594584465026855,
+      "step": 180,
+      "step_time": 10.576303614675998
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2041666716337204,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 248.0,
+      "completions/mean_length": 154.63750305175782,
+      "completions/mean_terminated_length": 128.9908935546875,
+      "completions/min_length": 47.2,
+      "completions/min_terminated_length": 47.2,
+      "entropy": 1.5905902862548829,
+      "epoch": 0.37,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.8977794647216797,
+      "kl": 0.2617978900671005,
+      "learning_rate": 4.08e-06,
+      "loss": 0.0605,
+      "num_tokens": 2208870.0,
+      "reward": 1.608209228515625,
+      "reward_std": 0.40417273342609406,
+      "rewards/politeness_reward_func/mean": 1.608209204673767,
+      "rewards/politeness_reward_func/std": 0.44712826013565066,
+      "step": 185,
+      "step_time": 10.657971718162297
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25416666865348814,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 243.6,
+      "completions/mean_length": 167.1416748046875,
+      "completions/mean_terminated_length": 136.75882873535156,
+      "completions/min_length": 46.6,
+      "completions/min_terminated_length": 46.6,
+      "entropy": 1.76783127784729,
+      "epoch": 0.38,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.072940826416016,
+      "kl": 0.29045032411813737,
+      "learning_rate": 4.055000000000001e-06,
+      "loss": 0.0313,
+      "num_tokens": 2266296.0,
+      "reward": 1.7399524927139283,
+      "reward_std": 0.4326803207397461,
+      "rewards/politeness_reward_func/mean": 1.7399524688720702,
+      "rewards/politeness_reward_func/std": 0.44698241353034973,
+      "step": 190,
+      "step_time": 10.595029101520776
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.387500011920929,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.8,
+      "completions/mean_length": 190.1125030517578,
+      "completions/mean_terminated_length": 150.54798278808593,
+      "completions/min_length": 57.8,
+      "completions/min_terminated_length": 57.8,
+      "entropy": 1.7596600532531739,
+      "epoch": 0.39,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.128026008605957,
+      "kl": 0.25340984016656876,
+      "learning_rate": 4.03e-06,
+      "loss": 0.0384,
+      "num_tokens": 2329619.0,
+      "reward": 1.6809997797012328,
+      "reward_std": 0.4062549531459808,
+      "rewards/politeness_reward_func/mean": 1.6809997320175172,
+      "rewards/politeness_reward_func/std": 0.44869658946990965,
+      "step": 195,
+      "step_time": 10.63182978257537
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.320833346247673,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.8,
+      "completions/mean_length": 182.68334045410157,
+      "completions/mean_terminated_length": 149.52076721191406,
+      "completions/min_length": 60.2,
+      "completions/min_terminated_length": 60.2,
+      "entropy": 1.7882241249084472,
+      "epoch": 0.4,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.352699279785156,
+      "kl": 0.2817923933267593,
+      "learning_rate": 4.005000000000001e-06,
+      "loss": -0.0338,
+      "num_tokens": 2390743.0,
+      "reward": 1.7947330713272094,
+      "reward_std": 0.3602103054523468,
+      "rewards/politeness_reward_func/mean": 1.7947329998016357,
+      "rewards/politeness_reward_func/std": 0.37249717116355896,
+      "step": 200,
+      "step_time": 10.771137349307537
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.279166679084301,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 251.0,
+      "completions/mean_length": 175.2791717529297,
+      "completions/mean_terminated_length": 146.66289672851562,
+      "completions/min_length": 54.2,
+      "completions/min_terminated_length": 54.2,
+      "entropy": 1.805497646331787,
+      "epoch": 0.41,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.236753463745117,
+      "kl": 0.4257870987057686,
+      "learning_rate": 3.980000000000001e-06,
+      "loss": 0.0117,
+      "num_tokens": 2449866.0,
+      "reward": 1.7829206705093383,
+      "reward_std": 0.4116291403770447,
+      "rewards/politeness_reward_func/mean": 1.7829206705093383,
+      "rewards/politeness_reward_func/std": 0.43859742283821107,
+      "step": 205,
+      "step_time": 10.78743917644024
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.254166679084301,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 248.2,
+      "completions/mean_length": 171.39584045410157,
+      "completions/mean_terminated_length": 143.78868103027344,
+      "completions/min_length": 52.0,
+      "completions/min_terminated_length": 52.0,
+      "entropy": 1.6819992780685424,
+      "epoch": 0.42,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.016842842102051,
+      "kl": 0.2606677010655403,
+      "learning_rate": 3.955e-06,
+      "loss": 0.0507,
+      "num_tokens": 2507913.0,
+      "reward": 1.6511780500411988,
+      "reward_std": 0.4131308376789093,
+      "rewards/politeness_reward_func/mean": 1.651178002357483,
+      "rewards/politeness_reward_func/std": 0.43051646947860717,
+      "step": 210,
+      "step_time": 10.86044239550829
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2250000059604645,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.2,
+      "completions/mean_length": 166.20833740234374,
+      "completions/mean_terminated_length": 139.74808349609376,
+      "completions/min_length": 58.8,
+      "completions/min_terminated_length": 58.8,
+      "entropy": 1.5043580889701844,
+      "epoch": 0.43,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.5619537830352783,
+      "kl": 0.24042100906372071,
+      "learning_rate": 3.9300000000000005e-06,
+      "loss": -0.0266,
+      "num_tokens": 2565323.0,
+      "reward": 1.6339158296585083,
+      "reward_std": 0.33398547768592834,
+      "rewards/politeness_reward_func/mean": 1.6339158535003662,
+      "rewards/politeness_reward_func/std": 0.37013601660728457,
+      "step": 215,
+      "step_time": 10.860479059070348
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2291666731238365,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 238.6,
+      "completions/mean_length": 162.5666717529297,
+      "completions/mean_terminated_length": 137.25745391845703,
+      "completions/min_length": 52.4,
+      "completions/min_terminated_length": 52.4,
+      "entropy": 1.707606041431427,
+      "epoch": 0.44,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.356007099151611,
+      "kl": 0.275185127556324,
+      "learning_rate": 3.905000000000001e-06,
+      "loss": 0.0385,
+      "num_tokens": 2621347.0,
+      "reward": 1.7018348932266236,
+      "reward_std": 0.3872075915336609,
+      "rewards/politeness_reward_func/mean": 1.7018348217010497,
+      "rewards/politeness_reward_func/std": 0.43171402215957644,
+      "step": 220,
+      "step_time": 10.862962251901626
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.25000000447034837,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 247.6,
+      "completions/mean_length": 170.00000610351563,
+      "completions/mean_terminated_length": 142.47680053710937,
+      "completions/min_length": 51.4,
+      "completions/min_terminated_length": 51.4,
+      "entropy": 1.7443210363388062,
+      "epoch": 0.45,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.761515140533447,
+      "kl": 0.2994952619075775,
+      "learning_rate": 3.88e-06,
+      "loss": 0.0254,
+      "num_tokens": 2679443.0,
+      "reward": 1.6937603950500488,
+      "reward_std": 0.42128287851810453,
+      "rewards/politeness_reward_func/mean": 1.6937604188919066,
+      "rewards/politeness_reward_func/std": 0.4645624727010727,
+      "step": 225,
+      "step_time": 10.798750822246074
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2750000089406967,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 238.4,
+      "completions/mean_length": 171.07500305175782,
+      "completions/mean_terminated_length": 139.3609603881836,
+      "completions/min_length": 51.4,
+      "completions/min_terminated_length": 51.4,
+      "entropy": 1.7655636191368103,
+      "epoch": 0.46,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.219749450683594,
+      "kl": 0.30960221886634826,
+      "learning_rate": 3.855e-06,
+      "loss": 0.0374,
+      "num_tokens": 2738565.0,
+      "reward": 1.74673752784729,
+      "reward_std": 0.49648854732513426,
+      "rewards/politeness_reward_func/mean": 1.7467374801635742,
+      "rewards/politeness_reward_func/std": 0.5134409368038177,
+      "step": 230,
+      "step_time": 10.840561749786138
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.24166667759418486,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.4,
+      "completions/mean_length": 170.58750610351564,
+      "completions/mean_terminated_length": 144.16094665527345,
+      "completions/min_length": 49.6,
+      "completions/min_terminated_length": 49.6,
+      "entropy": 1.5835796117782592,
+      "epoch": 0.47,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 8.280583381652832,
+      "kl": 0.28185769021511076,
+      "learning_rate": 3.830000000000001e-06,
+      "loss": 0.0433,
+      "num_tokens": 2797282.0,
+      "reward": 1.6517542362213136,
+      "reward_std": 0.4238882720470428,
+      "rewards/politeness_reward_func/mean": 1.6517542362213136,
+      "rewards/politeness_reward_func/std": 0.46576651334762575,
+      "step": 235,
+      "step_time": 10.934168311953545
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1833333358168602,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 248.4,
+      "completions/mean_length": 149.59166870117187,
+      "completions/mean_terminated_length": 125.1296600341797,
+      "completions/min_length": 40.4,
+      "completions/min_terminated_length": 40.4,
+      "entropy": 1.5539511442184448,
+      "epoch": 0.48,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.9952290058135986,
+      "kl": 0.2962296575307846,
+      "learning_rate": 3.8050000000000004e-06,
+      "loss": 0.0772,
+      "num_tokens": 2850656.0,
+      "reward": 1.5875454187393188,
+      "reward_std": 0.33082354068756104,
+      "rewards/politeness_reward_func/mean": 1.5875454187393188,
+      "rewards/politeness_reward_func/std": 0.4173043370246887,
+      "step": 240,
+      "step_time": 10.826450176537037
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.42500001192092896,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 252.4,
+      "completions/mean_length": 203.55417175292968,
+      "completions/mean_terminated_length": 167.60662841796875,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
+      "entropy": 1.732195258140564,
+      "epoch": 0.49,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.070302963256836,
+      "kl": 0.2232723578810692,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 0.0646,
+      "num_tokens": 2916917.0,
+      "reward": 1.5489000082015991,
+      "reward_std": 0.44896683692932127,
+      "rewards/politeness_reward_func/mean": 1.5488999366760254,
+      "rewards/politeness_reward_func/std": 0.4965748846530914,
+      "step": 245,
+      "step_time": 10.93318238928914
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3375000059604645,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.6,
+      "completions/mean_length": 175.0666717529297,
+      "completions/mean_terminated_length": 135.0446517944336,
+      "completions/min_length": 47.4,
+      "completions/min_terminated_length": 47.4,
+      "entropy": 1.7236797571182252,
+      "epoch": 0.5,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 15.248210906982422,
+      "kl": 0.31031118631362914,
+      "learning_rate": 3.7550000000000005e-06,
+      "loss": 0.0402,
+      "num_tokens": 2976245.0,
+      "reward": 1.6219825506210328,
+      "reward_std": 0.4284421443939209,
+      "rewards/politeness_reward_func/mean": 1.6219825744628906,
+      "rewards/politeness_reward_func/std": 0.4772031903266907,
+      "step": 250,
+      "step_time": 10.9129977889359
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.420833358168602,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 251.8,
+      "completions/mean_length": 198.55000915527344,
+      "completions/mean_terminated_length": 162.55760955810547,
+      "completions/min_length": 71.8,
+      "completions/min_terminated_length": 71.8,
+      "entropy": 1.9296841025352478,
+      "epoch": 0.51,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.592284679412842,
+      "kl": 0.25315294712781905,
+      "learning_rate": 3.7300000000000003e-06,
+      "loss": 0.0132,
+      "num_tokens": 3041337.0,
+      "reward": 1.6332777500152589,
+      "reward_std": 0.4870866537094116,
+      "rewards/politeness_reward_func/mean": 1.6332777261734008,
+      "rewards/politeness_reward_func/std": 0.5504487872123718,
+      "step": 255,
+      "step_time": 10.853592294454575
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.32083334028720856,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.0,
+      "completions/mean_length": 184.1916717529297,
+      "completions/mean_terminated_length": 150.5311309814453,
+      "completions/min_length": 62.8,
+      "completions/min_terminated_length": 62.8,
+      "entropy": 1.8794168829917908,
+      "epoch": 0.52,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.218177795410156,
+      "kl": 0.24962413907051087,
+      "learning_rate": 3.705e-06,
+      "loss": 0.058,
+      "num_tokens": 3102695.0,
+      "reward": 1.7052141666412353,
+      "reward_std": 0.4909538745880127,
+      "rewards/politeness_reward_func/mean": 1.7052141666412353,
+      "rewards/politeness_reward_func/std": 0.5124517560005188,
+      "step": 260,
+      "step_time": 10.939885137230158
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.39166667461395266,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 240.8,
+      "completions/mean_length": 182.05833740234374,
+      "completions/mean_terminated_length": 135.85120544433593,
+      "completions/min_length": 47.8,
+      "completions/min_terminated_length": 47.8,
+      "entropy": 1.78580139875412,
+      "epoch": 0.53,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.726942777633667,
+      "kl": 0.26940242052078245,
+      "learning_rate": 3.6800000000000003e-06,
+      "loss": 0.0129,
+      "num_tokens": 3163621.0,
+      "reward": 1.6984041452407836,
+      "reward_std": 0.4623105704784393,
+      "rewards/politeness_reward_func/mean": 1.69840407371521,
+      "rewards/politeness_reward_func/std": 0.5121756374835968,
+      "step": 265,
+      "step_time": 10.816243136674165
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.37500000596046446,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 242.6,
+      "completions/mean_length": 196.69584045410156,
+      "completions/mean_terminated_length": 160.64813537597655,
+      "completions/min_length": 66.4,
+      "completions/min_terminated_length": 66.4,
+      "entropy": 1.7563302516937256,
+      "epoch": 0.54,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.206515312194824,
+      "kl": 0.3476646825671196,
+      "learning_rate": 3.655e-06,
+      "loss": 0.0463,
+      "num_tokens": 3227836.0,
+      "reward": 1.7140352249145507,
+      "reward_std": 0.41706904768943787,
+      "rewards/politeness_reward_func/mean": 1.714035153388977,
+      "rewards/politeness_reward_func/std": 0.4722850799560547,
+      "step": 270,
+      "step_time": 10.934283661842347
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.43750002086162565,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 246.0,
+      "completions/mean_length": 200.75416870117186,
+      "completions/mean_terminated_length": 159.0064697265625,
+      "completions/min_length": 59.6,
+      "completions/min_terminated_length": 59.6,
+      "entropy": 1.853849744796753,
+      "epoch": 0.55,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.3859314918518066,
+      "kl": 0.2895158067345619,
+      "learning_rate": 3.6300000000000004e-06,
+      "loss": -0.0048,
+      "num_tokens": 3292993.0,
+      "reward": 1.8271270751953126,
+      "reward_std": 0.42741702795028685,
+      "rewards/politeness_reward_func/mean": 1.8271270513534545,
+      "rewards/politeness_reward_func/std": 0.47546608448028566,
+      "step": 275,
+      "step_time": 10.752403935790062
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.35833334624767305,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 241.8,
+      "completions/mean_length": 189.95000305175782,
+      "completions/mean_terminated_length": 153.42260131835937,
+      "completions/min_length": 57.8,
+      "completions/min_terminated_length": 57.8,
+      "entropy": 1.8662793278694152,
+      "epoch": 0.56,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.938138484954834,
+      "kl": 0.29017033874988557,
+      "learning_rate": 3.6050000000000002e-06,
+      "loss": -0.0162,
+      "num_tokens": 3355893.0,
+      "reward": 1.7445276975631714,
+      "reward_std": 0.4672766387462616,
+      "rewards/politeness_reward_func/mean": 1.744527554512024,
+      "rewards/politeness_reward_func/std": 0.4982309639453888,
+      "step": 280,
+      "step_time": 10.624229270964861
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.31250000596046446,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.4,
+      "completions/mean_length": 190.03333740234376,
+      "completions/mean_terminated_length": 162.05103149414063,
+      "completions/min_length": 65.8,
+      "completions/min_terminated_length": 65.8,
+      "entropy": 1.74795663356781,
+      "epoch": 0.57,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.650832176208496,
+      "kl": 0.21563028842210769,
+      "learning_rate": 3.58e-06,
+      "loss": 0.0183,
+      "num_tokens": 3418413.0,
+      "reward": 1.6855457305908204,
+      "reward_std": 0.3864980161190033,
+      "rewards/politeness_reward_func/mean": 1.6855456590652467,
+      "rewards/politeness_reward_func/std": 0.4192918837070465,
+      "step": 285,
+      "step_time": 10.76889206841588
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.28750001192092894,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 240.4,
+      "completions/mean_length": 174.78333740234376,
+      "completions/mean_terminated_length": 144.00184173583983,
+      "completions/min_length": 54.4,
+      "completions/min_terminated_length": 54.4,
+      "entropy": 1.5682233929634095,
+      "epoch": 0.58,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.823276042938232,
+      "kl": 0.25128297358751295,
+      "learning_rate": 3.5550000000000003e-06,
+      "loss": 0.0429,
+      "num_tokens": 3477785.0,
+      "reward": 1.6693106412887573,
+      "reward_std": 0.3742055296897888,
+      "rewards/politeness_reward_func/mean": 1.6693106412887573,
+      "rewards/politeness_reward_func/std": 0.4070769190788269,
+      "step": 290,
+      "step_time": 10.749481346458197
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.27083333730697634,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 240.8,
+      "completions/mean_length": 174.8166717529297,
+      "completions/mean_terminated_length": 145.73609619140626,
+      "completions/min_length": 56.6,
+      "completions/min_terminated_length": 56.6,
+      "entropy": 1.7244809031486512,
+      "epoch": 0.59,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.0669989585876465,
+      "kl": 0.28962416797876356,
+      "learning_rate": 3.53e-06,
+      "loss": 0.0175,
+      "num_tokens": 3537101.0,
+      "reward": 1.6176325321197509,
+      "reward_std": 0.4998854100704193,
+      "rewards/politeness_reward_func/mean": 1.6176324844360352,
+      "rewards/politeness_reward_func/std": 0.5255676746368408,
+      "step": 295,
+      "step_time": 10.83822975307703
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.29166668355464936,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.2,
+      "completions/mean_length": 176.39166870117188,
+      "completions/mean_terminated_length": 145.56358642578124,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 1.7615143656730652,
+      "epoch": 0.6,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 9.959527015686035,
+      "kl": 0.30573274195194244,
+      "learning_rate": 3.505e-06,
+      "loss": 0.0663,
+      "num_tokens": 3597051.0,
+      "reward": 1.7090127229690553,
+      "reward_std": 0.4356145679950714,
+      "rewards/politeness_reward_func/mean": 1.7090126276016235,
+      "rewards/politeness_reward_func/std": 0.458771163225174,
+      "step": 300,
+      "step_time": 10.85788278505206
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.325000011920929,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.2,
+      "completions/mean_length": 181.22083740234376,
+      "completions/mean_terminated_length": 146.1617431640625,
+      "completions/min_length": 55.0,
+      "completions/min_terminated_length": 55.0,
+      "entropy": 1.9170334219932557,
+      "epoch": 0.61,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.597347259521484,
+      "kl": 0.2534967973828316,
+      "learning_rate": 3.48e-06,
+      "loss": -0.019,
+      "num_tokens": 3657680.0,
+      "reward": 1.8059730291366578,
+      "reward_std": 0.4023502767086029,
+      "rewards/politeness_reward_func/mean": 1.8059730291366578,
+      "rewards/politeness_reward_func/std": 0.42335187196731566,
+      "step": 305,
+      "step_time": 10.813059192150831
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3458333432674408,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 247.6,
+      "completions/mean_length": 192.33333435058594,
+      "completions/mean_terminated_length": 158.2584686279297,
+      "completions/min_length": 61.0,
+      "completions/min_terminated_length": 61.0,
+      "entropy": 1.7168800473213195,
+      "epoch": 0.62,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.3291268348693848,
+      "kl": 0.30181923806667327,
+      "learning_rate": 3.455e-06,
+      "loss": 0.0438,
+      "num_tokens": 3721168.0,
+      "reward": 1.6650686740875245,
+      "reward_std": 0.4736558377742767,
+      "rewards/politeness_reward_func/mean": 1.6650686979293823,
+      "rewards/politeness_reward_func/std": 0.5267638087272644,
+      "step": 310,
+      "step_time": 10.904363192617893
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2708333432674408,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 248.4,
+      "completions/mean_length": 178.20833740234374,
+      "completions/mean_terminated_length": 150.5656768798828,
+      "completions/min_length": 52.8,
+      "completions/min_terminated_length": 52.8,
+      "entropy": 1.6762184143066405,
+      "epoch": 0.63,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.4125494956970215,
+      "kl": 0.3411761596798897,
+      "learning_rate": 3.4300000000000006e-06,
+      "loss": 0.0739,
+      "num_tokens": 3781170.0,
+      "reward": 1.698883295059204,
+      "reward_std": 0.5051921904087067,
+      "rewards/politeness_reward_func/mean": 1.6988832235336304,
+      "rewards/politeness_reward_func/std": 0.5214821577072144,
+      "step": 315,
+      "step_time": 10.870812387019395
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3333333432674408,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 243.0,
+      "completions/mean_length": 174.2375061035156,
+      "completions/mean_terminated_length": 133.60752563476564,
+      "completions/min_length": 38.8,
+      "completions/min_terminated_length": 38.8,
+      "entropy": 1.608705174922943,
+      "epoch": 0.64,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.398674011230469,
+      "kl": 0.3067095875740051,
+      "learning_rate": 3.4050000000000004e-06,
+      "loss": 0.0628,
+      "num_tokens": 3840075.0,
+      "reward": 1.6573886156082154,
+      "reward_std": 0.4343807339668274,
+      "rewards/politeness_reward_func/mean": 1.6573885917663573,
+      "rewards/politeness_reward_func/std": 0.4526840627193451,
+      "step": 320,
+      "step_time": 10.640466004610062
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2666666716337204,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 248.8,
+      "completions/mean_length": 177.06666870117186,
+      "completions/mean_terminated_length": 148.47830200195312,
+      "completions/min_length": 59.6,
+      "completions/min_terminated_length": 59.6,
+      "entropy": 1.832581627368927,
+      "epoch": 0.65,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.723278522491455,
+      "kl": 0.2649755135178566,
+      "learning_rate": 3.3800000000000007e-06,
+      "loss": -0.026,
+      "num_tokens": 3899579.0,
+      "reward": 1.81795072555542,
+      "reward_std": 0.39153199791908266,
+      "rewards/politeness_reward_func/mean": 1.8179506063461304,
+      "rewards/politeness_reward_func/std": 0.41539669036865234,
+      "step": 325,
+      "step_time": 10.751856955885888
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.24583333730697632,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 238.2,
+      "completions/mean_length": 164.57083740234376,
+      "completions/mean_terminated_length": 135.28192138671875,
+      "completions/min_length": 47.6,
+      "completions/min_terminated_length": 47.6,
+      "entropy": 1.6712656617164612,
+      "epoch": 0.66,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.251152992248535,
+      "kl": 0.2780707836151123,
+      "learning_rate": 3.3550000000000005e-06,
+      "loss": 0.0515,
+      "num_tokens": 3956132.0,
+      "reward": 1.6795947074890136,
+      "reward_std": 0.4304037630558014,
+      "rewards/politeness_reward_func/mean": 1.6795947313308717,
+      "rewards/politeness_reward_func/std": 0.456265515089035,
+      "step": 330,
+      "step_time": 10.65747187435627
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.27083334028720857,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.8,
+      "completions/mean_length": 170.53750610351562,
+      "completions/mean_terminated_length": 139.22777709960937,
+      "completions/min_length": 48.4,
+      "completions/min_terminated_length": 48.4,
+      "entropy": 1.6737295269966126,
+      "epoch": 0.67,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.981579065322876,
+      "kl": 0.27397735267877577,
+      "learning_rate": 3.3300000000000003e-06,
+      "loss": 0.0586,
+      "num_tokens": 4014149.0,
+      "reward": 1.639498805999756,
+      "reward_std": 0.3987973630428314,
+      "rewards/politeness_reward_func/mean": 1.639498782157898,
+      "rewards/politeness_reward_func/std": 0.4488051861524582,
+      "step": 335,
+      "step_time": 10.64305683746934
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.24583334028720855,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 247.6,
+      "completions/mean_length": 170.57083435058593,
+      "completions/mean_terminated_length": 143.0279296875,
+      "completions/min_length": 53.6,
+      "completions/min_terminated_length": 53.6,
+      "entropy": 1.8374268651008605,
+      "epoch": 0.68,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.304769515991211,
+      "kl": 0.2890111759305,
+      "learning_rate": 3.3050000000000005e-06,
+      "loss": 0.0539,
+      "num_tokens": 4072318.0,
+      "reward": 1.827410674095154,
+      "reward_std": 0.34623334407806394,
+      "rewards/politeness_reward_func/mean": 1.8274105072021485,
+      "rewards/politeness_reward_func/std": 0.3635041773319244,
+      "step": 340,
+      "step_time": 10.67860155031085
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3791666865348816,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 246.8,
+      "completions/mean_length": 182.3541717529297,
+      "completions/mean_terminated_length": 138.33062896728515,
+      "completions/min_length": 54.4,
+      "completions/min_terminated_length": 54.4,
+      "entropy": 1.8917979717254638,
+      "epoch": 0.69,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.783524751663208,
+      "kl": 0.3112198129296303,
+      "learning_rate": 3.2800000000000004e-06,
+      "loss": -0.0079,
+      "num_tokens": 4133395.0,
+      "reward": 1.8497365951538085,
+      "reward_std": 0.41627883315086367,
+      "rewards/politeness_reward_func/mean": 1.8497365236282348,
+      "rewards/politeness_reward_func/std": 0.4410283327102661,
+      "step": 345,
+      "step_time": 10.664074825495481
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.43750001192092897,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.8,
+      "completions/mean_length": 192.87083740234374,
+      "completions/mean_terminated_length": 144.26836700439452,
+      "completions/min_length": 62.4,
+      "completions/min_terminated_length": 62.4,
+      "entropy": 1.9642783045768737,
+      "epoch": 0.7,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.8300986289978027,
+      "kl": 0.3175328865647316,
+      "learning_rate": 3.255e-06,
+      "loss": 0.033,
+      "num_tokens": 4197028.0,
+      "reward": 1.8747668027877809,
+      "reward_std": 0.485896635055542,
+      "rewards/politeness_reward_func/mean": 1.8747668027877809,
+      "rewards/politeness_reward_func/std": 0.5172119975090027,
+      "step": 350,
+      "step_time": 10.599913079291582
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.30416667759418486,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 245.4,
+      "completions/mean_length": 173.94584045410156,
+      "completions/mean_terminated_length": 138.12840881347657,
+      "completions/min_length": 50.4,
+      "completions/min_terminated_length": 50.4,
+      "entropy": 1.6719454765319823,
+      "epoch": 0.71,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.62943172454834,
+      "kl": 0.354923115670681,
+      "learning_rate": 3.2300000000000004e-06,
+      "loss": 0.04,
+      "num_tokens": 4256759.0,
+      "reward": 1.746033239364624,
+      "reward_std": 0.46536014080047605,
+      "rewards/politeness_reward_func/mean": 1.7460331916809082,
+      "rewards/politeness_reward_func/std": 0.513632845878601,
+      "step": 355,
+      "step_time": 10.574684323370457
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2166666716337204,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 246.4,
+      "completions/mean_length": 164.38750610351562,
+      "completions/mean_terminated_length": 139.19703826904296,
+      "completions/min_length": 58.2,
+      "completions/min_terminated_length": 58.2,
+      "entropy": 1.7568098187446595,
+      "epoch": 0.72,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.490025997161865,
+      "kl": 0.2866540029644966,
+      "learning_rate": 3.2050000000000002e-06,
+      "loss": -0.0106,
+      "num_tokens": 4313620.0,
+      "reward": 1.7767923593521118,
+      "reward_std": 0.4481996238231659,
+      "rewards/politeness_reward_func/mean": 1.776792311668396,
+      "rewards/politeness_reward_func/std": 0.4569097697734833,
+      "step": 360,
+      "step_time": 10.563734823465348
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3833333432674408,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 249.8,
+      "completions/mean_length": 192.51250305175782,
+      "completions/mean_terminated_length": 152.48777465820314,
+      "completions/min_length": 54.8,
+      "completions/min_terminated_length": 54.8,
+      "entropy": 1.7603564739227295,
+      "epoch": 0.73,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.731227159500122,
+      "kl": 0.2559724062681198,
+      "learning_rate": 3.1800000000000005e-06,
+      "loss": 0.0058,
+      "num_tokens": 4377423.0,
+      "reward": 1.7886356353759765,
+      "reward_std": 0.43659440279006956,
+      "rewards/politeness_reward_func/mean": 1.7886356592178345,
+      "rewards/politeness_reward_func/std": 0.4865987479686737,
+      "step": 365,
+      "step_time": 10.596465566009282
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3208333432674408,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 253.0,
+      "completions/mean_length": 183.37500610351563,
+      "completions/mean_terminated_length": 150.60201873779297,
+      "completions/min_length": 56.2,
+      "completions/min_terminated_length": 56.2,
+      "entropy": 1.599543821811676,
+      "epoch": 0.74,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.495799541473389,
+      "kl": 0.26432290226221083,
+      "learning_rate": 3.1550000000000003e-06,
+      "loss": 0.0365,
+      "num_tokens": 4438841.0,
+      "reward": 1.663888192176819,
+      "reward_std": 0.4724281966686249,
+      "rewards/politeness_reward_func/mean": 1.663888168334961,
+      "rewards/politeness_reward_func/std": 0.49500845074653627,
+      "step": 370,
+      "step_time": 10.550004740059375
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2833333402872086,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 245.8,
+      "completions/mean_length": 176.43333740234374,
+      "completions/mean_terminated_length": 145.9316650390625,
+      "completions/min_length": 46.8,
+      "completions/min_terminated_length": 46.8,
+      "entropy": 1.649828588962555,
+      "epoch": 0.75,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.405642509460449,
+      "kl": 0.3023358851671219,
+      "learning_rate": 3.13e-06,
+      "loss": 0.036,
+      "num_tokens": 4498209.0,
+      "reward": 1.7260100841522217,
+      "reward_std": 0.4888251006603241,
+      "rewards/politeness_reward_func/mean": 1.726010012626648,
+      "rewards/politeness_reward_func/std": 0.5257126033306122,
+      "step": 375,
+      "step_time": 10.784504148364068
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2750000059604645,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 251.0,
+      "completions/mean_length": 177.4791717529297,
+      "completions/mean_terminated_length": 149.0453887939453,
+      "completions/min_length": 52.6,
+      "completions/min_terminated_length": 52.6,
+      "entropy": 1.669038712978363,
+      "epoch": 0.76,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.453659534454346,
+      "kl": 0.26074831187725067,
+      "learning_rate": 3.1050000000000003e-06,
+      "loss": 0.0133,
+      "num_tokens": 4558116.0,
+      "reward": 1.7338483333587646,
+      "reward_std": 0.40128968358039857,
+      "rewards/politeness_reward_func/mean": 1.7338482856750488,
+      "rewards/politeness_reward_func/std": 0.4201090157032013,
+      "step": 380,
+      "step_time": 10.85576168820262
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3166666775941849,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 248.4,
+      "completions/mean_length": 175.9791687011719,
+      "completions/mean_terminated_length": 141.09287872314454,
+      "completions/min_length": 60.8,
+      "completions/min_terminated_length": 60.8,
+      "entropy": 1.70971120595932,
+      "epoch": 0.77,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.21511173248291,
+      "kl": 0.33568228632211683,
+      "learning_rate": 3.08e-06,
+      "loss": 0.0539,
+      "num_tokens": 4617679.0,
+      "reward": 1.8021876573562623,
+      "reward_std": 0.39429293274879457,
+      "rewards/politeness_reward_func/mean": 1.8021875858306884,
+      "rewards/politeness_reward_func/std": 0.4176251709461212,
+      "step": 385,
+      "step_time": 10.889974600821734
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.24583333879709243,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.4,
+      "completions/mean_length": 162.72084045410156,
+      "completions/mean_terminated_length": 133.57429962158204,
+      "completions/min_length": 56.8,
+      "completions/min_terminated_length": 56.8,
+      "entropy": 1.5501957058906555,
+      "epoch": 0.78,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.835730075836182,
+      "kl": 0.30908732712268827,
+      "learning_rate": 3.0550000000000004e-06,
+      "loss": 0.0817,
+      "num_tokens": 4674108.0,
+      "reward": 1.7301536321640014,
+      "reward_std": 0.44032427072525027,
+      "rewards/politeness_reward_func/mean": 1.7301535844802856,
+      "rewards/politeness_reward_func/std": 0.4708732008934021,
+      "step": 390,
+      "step_time": 10.865944185107946
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1791666716337204,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 247.8,
+      "completions/mean_length": 157.25000915527343,
+      "completions/mean_terminated_length": 135.5115005493164,
+      "completions/min_length": 51.2,
+      "completions/min_terminated_length": 51.2,
+      "entropy": 1.6409943580627442,
+      "epoch": 0.79,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.950301647186279,
+      "kl": 0.36195332258939744,
+      "learning_rate": 3.0300000000000002e-06,
+      "loss": 0.0805,
+      "num_tokens": 4729176.0,
+      "reward": 1.737137007713318,
+      "reward_std": 0.4313428819179535,
+      "rewards/politeness_reward_func/mean": 1.737137007713318,
+      "rewards/politeness_reward_func/std": 0.4497229218482971,
+      "step": 395,
+      "step_time": 10.928832749277353
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3291666775941849,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 250.4,
+      "completions/mean_length": 180.15833740234376,
+      "completions/mean_terminated_length": 143.84142608642577,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 1.7865488886833192,
+      "epoch": 0.8,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.5700058937072754,
+      "kl": 0.2882836848497391,
+      "learning_rate": 3.005e-06,
+      "loss": 0.0432,
+      "num_tokens": 4790094.0,
+      "reward": 1.7929600477218628,
+      "reward_std": 0.4272151470184326,
+      "rewards/politeness_reward_func/mean": 1.792960000038147,
+      "rewards/politeness_reward_func/std": 0.4529218733310699,
+      "step": 400,
+      "step_time": 10.825692503154277
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 4790094,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82b31eee2e9e5185f99ca7159dbcba50f81778df0098ecfea28a3e0cfd83b36e
+size 7569