Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.51.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d20c947d9f5edb92a3b0807650bbff6b22f9e7c027a68f094f350a64ddfce25d
+size 4877660776

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:218b7675ae599db65e8341097b2146c75bf2a33a7ceb1d22ff30723eb826b738
+size 4932751008

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e3daf1e5a6d666d8c36007a7e250bbb032d79f2c5e7f7aae1bfdb27fa0ff810
+size 4330865200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73434dc50f06b13f0d76c536d594673152d08c849e2333d5fedae6daef97b7e8
+size 1089994880

model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 15231233024
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,3034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.192,
+  "eval_steps": 500,
+  "global_step": 240,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2761.8828125,
+      "epoch": 0.0008,
+      "grad_norm": 0.5506983381296895,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "loss": 0.0464,
+      "reward": 0.23066407296573743,
+      "reward_std": 0.45144409872591496,
+      "rewards/end_of_conversation_reward_func": 0.046093751094304025,
+      "rewards/end_rm_reward_func": 0.2392578125,
+      "rewards/length_reward_func": -0.0546875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 1
+    },
+    {
+      "clip_ratio": 0.004609759445884265,
+      "epoch": 0.0016,
+      "grad_norm": 0.5329509809633897,
+      "kl": 0.0012302398681640625,
+      "learning_rate": 5e-07,
+      "loss": 0.0463,
+      "step": 2
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3313.046875,
+      "epoch": 0.0024,
+      "grad_norm": 0.44037361030440025,
+      "kl": 0.0013537406921386719,
+      "learning_rate": 5e-07,
+      "loss": 0.0676,
+      "reward": 0.0886718796682544,
+      "reward_std": 0.45538751501590014,
+      "rewards/end_of_conversation_reward_func": 0.047656250768341124,
+      "rewards/end_rm_reward_func": 0.146484375,
+      "rewards/length_reward_func": -0.10546875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 3
+    },
+    {
+      "clip_ratio": 0.004155338363489136,
+      "epoch": 0.0032,
+      "grad_norm": 0.4238348103819464,
+      "kl": 0.00133514404296875,
+      "learning_rate": 5e-07,
+      "loss": 0.0677,
+      "step": 4
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3180.3046875,
+      "epoch": 0.004,
+      "grad_norm": 0.487685724080159,
+      "kl": 0.00140380859375,
+      "learning_rate": 5e-07,
+      "loss": 0.0617,
+      "reward": 0.12480469699949026,
+      "reward_std": 0.393667503958568,
+      "rewards/end_of_conversation_reward_func": 0.043750000768341124,
+      "rewards/end_rm_reward_func": 0.1279296875,
+      "rewards/length_reward_func": -0.046875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 5
+    },
+    {
+      "clip_ratio": 0.00448301769210957,
+      "epoch": 0.0048,
+      "grad_norm": 0.49115718476339393,
+      "kl": 0.001739501953125,
+      "learning_rate": 5e-07,
+      "loss": 0.0615,
+      "step": 6
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3680.03125,
+      "epoch": 0.0056,
+      "grad_norm": 0.36669507381078525,
+      "kl": 0.0017371177673339844,
+      "learning_rate": 5e-07,
+      "loss": 0.1115,
+      "reward": 0.12597656878642738,
+      "reward_std": 0.42063513631001115,
+      "rewards/end_of_conversation_reward_func": 0.035156251047737896,
+      "rewards/end_rm_reward_func": 0.1845703125,
+      "rewards/length_reward_func": -0.09375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 7
+    },
+    {
+      "clip_ratio": 0.004220222064759582,
+      "epoch": 0.0064,
+      "grad_norm": 0.36704470764422,
+      "kl": 0.001245260238647461,
+      "learning_rate": 5e-07,
+      "loss": 0.1116,
+      "step": 8
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3056.125,
+      "epoch": 0.0072,
+      "grad_norm": 0.3963636827663633,
+      "kl": 0.0019788742065429688,
+      "learning_rate": 5e-07,
+      "loss": 0.0823,
+      "reward": 0.259960952505935,
+      "reward_std": 0.4595574662089348,
+      "rewards/end_of_conversation_reward_func": 0.05000000080326572,
+      "rewards/end_rm_reward_func": 0.2685546875,
+      "rewards/length_reward_func": -0.05859375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 9
+    },
+    {
+      "clip_ratio": 0.004534602427156642,
+      "epoch": 0.008,
+      "grad_norm": 0.38789534112248125,
+      "kl": 0.001991748809814453,
+      "learning_rate": 5e-07,
+      "loss": 0.0822,
+      "step": 10
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3453.7265625,
+      "epoch": 0.0088,
+      "grad_norm": 0.38064460431215713,
+      "kl": 0.002022266387939453,
+      "learning_rate": 5e-07,
+      "loss": 0.0951,
+      "reward": 0.15000000572763383,
+      "reward_std": 0.4482696056365967,
+      "rewards/end_of_conversation_reward_func": 0.044531251245643944,
+      "rewards/end_rm_reward_func": 0.19140625,
+      "rewards/length_reward_func": -0.0859375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 11
+    },
+    {
+      "clip_ratio": 0.004099749305169098,
+      "epoch": 0.0096,
+      "grad_norm": 0.3751699254242308,
+      "kl": 0.0019767284393310547,
+      "learning_rate": 5e-07,
+      "loss": 0.0951,
+      "step": 12
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3352.5234375,
+      "epoch": 0.0104,
+      "grad_norm": 0.3498351189608962,
+      "kl": 0.0030684471130371094,
+      "learning_rate": 5e-07,
+      "loss": 0.1119,
+      "reward": 0.13613282376900315,
+      "reward_std": 0.4642677316442132,
+      "rewards/end_of_conversation_reward_func": 0.041406251140870154,
+      "rewards/end_rm_reward_func": 0.1650390625,
+      "rewards/length_reward_func": -0.0703125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 13
+    },
+    {
+      "clip_ratio": 0.0042787125712493435,
+      "epoch": 0.0112,
+      "grad_norm": 0.34790390559755574,
+      "kl": 0.0029201507568359375,
+      "learning_rate": 5e-07,
+      "loss": 0.112,
+      "step": 14
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2548.421875,
+      "epoch": 0.012,
+      "grad_norm": 0.5282889686332808,
+      "kl": 0.003574371337890625,
+      "learning_rate": 5e-07,
+      "loss": 0.0372,
+      "reward": 0.3990234467200935,
+      "reward_std": 0.3981876859907061,
+      "rewards/end_of_conversation_reward_func": 0.0523437510128133,
+      "rewards/end_rm_reward_func": 0.3779296875,
+      "rewards/length_reward_func": -0.03125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 15
+    },
+    {
+      "clip_ratio": 0.0044180305558256805,
+      "epoch": 0.0128,
+      "grad_norm": 0.5245952101164439,
+      "kl": 0.004303455352783203,
+      "learning_rate": 5e-07,
+      "loss": 0.0371,
+      "step": 16
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3264.7578125,
+      "epoch": 0.0136,
+      "grad_norm": 0.4930092476738244,
+      "kl": 0.0028238296508789062,
+      "learning_rate": 5e-07,
+      "loss": 0.0877,
+      "reward": 0.058398447930812836,
+      "reward_std": 0.43455274496227503,
+      "rewards/end_of_conversation_reward_func": 0.043750000884756446,
+      "rewards/end_rm_reward_func": 0.1083984375,
+      "rewards/length_reward_func": -0.09375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 17
+    },
+    {
+      "clip_ratio": 0.0050234832015121356,
+      "epoch": 0.0144,
+      "grad_norm": 0.4879552129621058,
+      "kl": 0.0030431747436523438,
+      "learning_rate": 5e-07,
+      "loss": 0.0877,
+      "step": 18
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3004.109375,
+      "epoch": 0.0152,
+      "grad_norm": 0.44669678095839005,
+      "kl": 0.0027985572814941406,
+      "learning_rate": 5e-07,
+      "loss": 0.0234,
+      "reward": 0.30312501173466444,
+      "reward_std": 0.38858885364606977,
+      "rewards/end_of_conversation_reward_func": 0.04140625096624717,
+      "rewards/end_rm_reward_func": 0.30078125,
+      "rewards/length_reward_func": -0.0390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 19
+    },
+    {
+      "clip_ratio": 0.003934504988137633,
+      "epoch": 0.016,
+      "grad_norm": 0.44274866510741123,
+      "kl": 0.0028510093688964844,
+      "learning_rate": 5e-07,
+      "loss": 0.0232,
+      "step": 20
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3229.953125,
+      "epoch": 0.0168,
+      "grad_norm": 0.48924113242313605,
+      "kl": 0.0034546852111816406,
+      "learning_rate": 5e-07,
+      "loss": 0.0991,
+      "reward": 0.26152344699949026,
+      "reward_std": 0.45786723494529724,
+      "rewards/end_of_conversation_reward_func": 0.05546875047730282,
+      "rewards/end_rm_reward_func": 0.2724609375,
+      "rewards/length_reward_func": -0.06640625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 21
+    },
+    {
+      "clip_ratio": 0.004537505708867684,
+      "epoch": 0.0176,
+      "grad_norm": 0.4709410237868635,
+      "kl": 0.0032873153686523438,
+      "learning_rate": 5e-07,
+      "loss": 0.0992,
+      "step": 22
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3771.703125,
+      "epoch": 0.0184,
+      "grad_norm": 0.4520127965646581,
+      "kl": 0.003025531768798828,
+      "learning_rate": 5e-07,
+      "loss": 0.0923,
+      "reward": 0.1816406373400241,
+      "reward_std": 0.3960149073973298,
+      "rewards/end_of_conversation_reward_func": 0.04296875064028427,
+      "rewards/end_rm_reward_func": 0.236328125,
+      "rewards/length_reward_func": -0.09765625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 23
+    },
+    {
+      "clip_ratio": 0.004418392883962952,
+      "epoch": 0.0192,
+      "grad_norm": 0.44165197354566593,
+      "kl": 0.003177642822265625,
+      "learning_rate": 5e-07,
+      "loss": 0.0922,
+      "step": 24
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2827.5234375,
+      "epoch": 0.02,
+      "grad_norm": 0.4898734505712748,
+      "kl": 0.003976345062255859,
+      "learning_rate": 5e-07,
+      "loss": 0.0859,
+      "reward": 0.29980469786096364,
+      "reward_std": 0.4965838538482785,
+      "rewards/end_of_conversation_reward_func": 0.05078125186264515,
+      "rewards/end_rm_reward_func": 0.2919921875,
+      "rewards/length_reward_func": -0.04296875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 25
+    },
+    {
+      "clip_ratio": 0.004215578097500838,
+      "epoch": 0.0208,
+      "grad_norm": 0.4604932294168012,
+      "kl": 0.0045299530029296875,
+      "learning_rate": 5e-07,
+      "loss": 0.0857,
+      "step": 26
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2700.25,
+      "epoch": 0.0216,
+      "grad_norm": 0.5569547586903592,
+      "kl": 0.006651878356933594,
+      "learning_rate": 5e-07,
+      "loss": 0.0628,
+      "reward": 0.21445313654839993,
+      "reward_std": 0.4763609850779176,
+      "rewards/end_of_conversation_reward_func": 0.048437501303851604,
+      "rewards/end_rm_reward_func": 0.224609375,
+      "rewards/length_reward_func": -0.05859375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 27
+    },
+    {
+      "clip_ratio": 0.00526866911968682,
+      "epoch": 0.0224,
+      "grad_norm": 0.5363885121679981,
+      "kl": 0.008578300476074219,
+      "learning_rate": 5e-07,
+      "loss": 0.0626,
+      "step": 28
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3302.2109375,
+      "epoch": 0.0232,
+      "grad_norm": 0.461624948266616,
+      "kl": 0.18498802185058594,
+      "learning_rate": 5e-07,
+      "loss": 0.1148,
+      "reward": 0.18046875682193786,
+      "reward_std": 0.3992706071585417,
+      "rewards/end_of_conversation_reward_func": 0.043750000826548785,
+      "rewards/end_rm_reward_func": 0.21875,
+      "rewards/length_reward_func": -0.08203125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 29
+    },
+    {
+      "clip_ratio": 0.004589079937431961,
+      "epoch": 0.024,
+      "grad_norm": 0.47299298250006305,
+      "kl": 0.14548110961914062,
+      "learning_rate": 5e-07,
+      "loss": 0.1149,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3118.6015625,
+      "epoch": 0.0248,
+      "grad_norm": 0.42713160024118774,
+      "kl": 0.008479118347167969,
+      "learning_rate": 5e-07,
+      "loss": 0.1257,
+      "reward": 0.34414063580334187,
+      "reward_std": 0.5011807959526777,
+      "rewards/end_of_conversation_reward_func": 0.05312500102445483,
+      "rewards/end_rm_reward_func": 0.361328125,
+      "rewards/length_reward_func": -0.0703125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 31
+    },
+    {
+      "clip_ratio": 0.004479929368244484,
+      "epoch": 0.0256,
+      "grad_norm": 0.44373075948513674,
+      "kl": 0.010387420654296875,
+      "learning_rate": 5e-07,
+      "loss": 0.1258,
+      "step": 32
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3374.3125,
+      "epoch": 0.0264,
+      "grad_norm": 0.4853755980068411,
+      "kl": 0.010019302368164062,
+      "learning_rate": 5e-07,
+      "loss": 0.0936,
+      "reward": 0.18574219779111445,
+      "reward_std": 0.39575440785847604,
+      "rewards/end_of_conversation_reward_func": 0.05000000144354999,
+      "rewards/end_rm_reward_func": 0.2138671875,
+      "rewards/length_reward_func": -0.078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 33
+    },
+    {
+      "clip_ratio": 0.0049804064619820565,
+      "epoch": 0.0272,
+      "grad_norm": 0.48990044698626367,
+      "kl": 0.010984420776367188,
+      "learning_rate": 5e-07,
+      "loss": 0.0935,
+      "step": 34
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3179.3671875,
+      "epoch": 0.028,
+      "grad_norm": 0.5301584639571147,
+      "kl": 0.016666412353515625,
+      "learning_rate": 5e-07,
+      "loss": 0.0573,
+      "reward": 0.20410157088190317,
+      "reward_std": 0.5211825617589056,
+      "rewards/end_of_conversation_reward_func": 0.04687500069849193,
+      "rewards/end_rm_reward_func": 0.2197265625,
+      "rewards/length_reward_func": -0.0625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 35
+    },
+    {
+      "clip_ratio": 0.004679829871747643,
+      "epoch": 0.0288,
+      "grad_norm": 0.5145033225785893,
+      "kl": 0.021465301513671875,
+      "learning_rate": 5e-07,
+      "loss": 0.0571,
+      "step": 36
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3041.6875,
+      "epoch": 0.0296,
+      "grad_norm": 0.4531386948890339,
+      "kl": 0.014142990112304688,
+      "learning_rate": 5e-07,
+      "loss": 0.061,
+      "reward": 0.17304688156582415,
+      "reward_std": 0.42873420752584934,
+      "rewards/end_of_conversation_reward_func": 0.04609375062864274,
+      "rewards/end_rm_reward_func": 0.181640625,
+      "rewards/length_reward_func": -0.0546875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 37
+    },
+    {
+      "clip_ratio": 0.004625552362995222,
+      "epoch": 0.0304,
+      "grad_norm": 0.4389212247837138,
+      "kl": 0.01320648193359375,
+      "learning_rate": 5e-07,
+      "loss": 0.0608,
+      "step": 38
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2900.984375,
+      "epoch": 0.0312,
+      "grad_norm": 0.513108187473543,
+      "kl": 0.0476226806640625,
+      "learning_rate": 5e-07,
+      "loss": 0.0364,
+      "reward": 0.26054688123986125,
+      "reward_std": 0.4113442550878972,
+      "rewards/end_of_conversation_reward_func": 0.0515625016996637,
+      "rewards/end_rm_reward_func": 0.244140625,
+      "rewards/length_reward_func": -0.03515625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 39
+    },
+    {
+      "clip_ratio": 0.005045634257839993,
+      "epoch": 0.032,
+      "grad_norm": 0.4960768449341314,
+      "kl": 0.023647308349609375,
+      "learning_rate": 5e-07,
+      "loss": 0.0363,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3094.5390625,
+      "epoch": 0.0328,
+      "grad_norm": 0.4111608127743996,
+      "kl": 0.023374557495117188,
+      "learning_rate": 5e-07,
+      "loss": 0.1087,
+      "reward": 0.18554687942378223,
+      "reward_std": 0.48099791165441275,
+      "rewards/end_of_conversation_reward_func": 0.03906250058207661,
+      "rewards/end_rm_reward_func": 0.205078125,
+      "rewards/length_reward_func": -0.05859375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 41
+    },
+    {
+      "clip_ratio": 0.004160198179306462,
+      "epoch": 0.0336,
+      "grad_norm": 0.4050446568608628,
+      "kl": 0.020130157470703125,
+      "learning_rate": 5e-07,
+      "loss": 0.1086,
+      "step": 42
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2732.453125,
+      "epoch": 0.0344,
+      "grad_norm": 0.5932767606982138,
+      "kl": 0.07580184936523438,
+      "learning_rate": 5e-07,
+      "loss": 0.0115,
+      "reward": 0.10800781659781933,
+      "reward_std": 0.4096634048037231,
+      "rewards/end_of_conversation_reward_func": 0.052343750663567334,
+      "rewards/end_rm_reward_func": 0.1103515625,
+      "rewards/length_reward_func": -0.0546875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 43
+    },
+    {
+      "clip_ratio": 0.005395003347075544,
+      "epoch": 0.0352,
+      "grad_norm": 0.562119953077685,
+      "kl": 0.1378173828125,
+      "learning_rate": 5e-07,
+      "loss": 0.0112,
+      "step": 44
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3149.03125,
+      "epoch": 0.036,
+      "grad_norm": 0.43738882704141613,
+      "kl": 0.07685470581054688,
+      "learning_rate": 5e-07,
+      "loss": 0.0961,
+      "reward": 0.18593750661239028,
+      "reward_std": 0.459837670205161,
+      "rewards/end_of_conversation_reward_func": 0.04140625079162419,
+      "rewards/end_rm_reward_func": 0.2109375,
+      "rewards/length_reward_func": -0.06640625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 45
+    },
+    {
+      "clip_ratio": 0.004886353664915077,
+      "epoch": 0.0368,
+      "grad_norm": 0.4246516268286063,
+      "kl": 0.07813262939453125,
+      "learning_rate": 5e-07,
+      "loss": 0.0961,
+      "step": 46
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2791.0625,
+      "epoch": 0.0376,
+      "grad_norm": 0.4757940646494544,
+      "kl": 0.041767120361328125,
+      "learning_rate": 5e-07,
+      "loss": 0.0631,
+      "reward": 0.2837890684604645,
+      "reward_std": 0.4126680698245764,
+      "rewards/end_of_conversation_reward_func": 0.056250001303851604,
+      "rewards/end_rm_reward_func": 0.2626953125,
+      "rewards/length_reward_func": -0.03515625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 47
+    },
+    {
+      "clip_ratio": 0.005135874525876716,
+      "epoch": 0.0384,
+      "grad_norm": 0.47547921032378754,
+      "kl": 0.09936141967773438,
+      "learning_rate": 5e-07,
+      "loss": 0.063,
+      "step": 48
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2754.609375,
+      "epoch": 0.0392,
+      "grad_norm": 0.49754042199838183,
+      "kl": 0.12404251098632812,
+      "learning_rate": 5e-07,
+      "loss": 0.0849,
+      "reward": 0.32207031920552254,
+      "reward_std": 0.46512654330581427,
+      "rewards/end_of_conversation_reward_func": 0.051562500884756446,
+      "rewards/end_rm_reward_func": 0.3017578125,
+      "rewards/length_reward_func": -0.03125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 49
+    },
+    {
+      "clip_ratio": 0.004882045017438941,
+      "epoch": 0.04,
+      "grad_norm": 0.4817430120792817,
+      "kl": 0.11900711059570312,
+      "learning_rate": 5e-07,
+      "loss": 0.0847,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2970.8984375,
+      "epoch": 0.0408,
+      "grad_norm": 0.5071054759904158,
+      "kl": 0.057147979736328125,
+      "learning_rate": 5e-07,
+      "loss": 0.1121,
+      "reward": 0.2646484528668225,
+      "reward_std": 0.37708853278309107,
+      "rewards/end_of_conversation_reward_func": 0.05468750081490725,
+      "rewards/end_rm_reward_func": 0.2607421875,
+      "rewards/length_reward_func": -0.05078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 51
+    },
+    {
+      "clip_ratio": 0.004119776640436612,
+      "epoch": 0.0416,
+      "grad_norm": 0.4894389235389238,
+      "kl": 0.05866241455078125,
+      "learning_rate": 5e-07,
+      "loss": 0.112,
+      "step": 52
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2454.84375,
+      "epoch": 0.0424,
+      "grad_norm": 0.6397704779379643,
+      "kl": 0.10573577880859375,
+      "learning_rate": 5e-07,
+      "loss": 0.0424,
+      "reward": 0.29101564083248377,
+      "reward_std": 0.3862592806108296,
+      "rewards/end_of_conversation_reward_func": 0.058593750989530236,
+      "rewards/end_rm_reward_func": 0.271484375,
+      "rewards/length_reward_func": -0.0390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 53
+    },
+    {
+      "clip_ratio": 0.005806522516650148,
+      "epoch": 0.0432,
+      "grad_norm": 0.5935861300569137,
+      "kl": 0.22344207763671875,
+      "learning_rate": 5e-07,
+      "loss": 0.0421,
+      "step": 54
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2823.3515625,
+      "epoch": 0.044,
+      "grad_norm": 0.5249791154692802,
+      "kl": 0.09177398681640625,
+      "learning_rate": 5e-07,
+      "loss": 0.0814,
+      "reward": 0.26269532449077815,
+      "reward_std": 0.40016138833016157,
+      "rewards/end_of_conversation_reward_func": 0.05078125122236088,
+      "rewards/end_rm_reward_func": 0.2626953125,
+      "rewards/length_reward_func": -0.05078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 55
+    },
+    {
+      "clip_ratio": 0.00533542905759532,
+      "epoch": 0.0448,
+      "grad_norm": 0.49819488661627054,
+      "kl": 0.0757293701171875,
+      "learning_rate": 5e-07,
+      "loss": 0.0811,
+      "step": 56
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2891.5859375,
+      "epoch": 0.0456,
+      "grad_norm": 0.5318285198540439,
+      "kl": 0.3561859130859375,
+      "learning_rate": 5e-07,
+      "loss": 0.0162,
+      "reward": 0.3619140787050128,
+      "reward_std": 0.45814486034214497,
+      "rewards/end_of_conversation_reward_func": 0.056250000721774995,
+      "rewards/end_rm_reward_func": 0.3486328125,
+      "rewards/length_reward_func": -0.04296875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 57
+    },
+    {
+      "clip_ratio": 0.005233535193838179,
+      "epoch": 0.0464,
+      "grad_norm": 0.5029588635123279,
+      "kl": 0.7606353759765625,
+      "learning_rate": 5e-07,
+      "loss": 0.0161,
+      "step": 58
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3584.8203125,
+      "epoch": 0.0472,
+      "grad_norm": 0.4934060163626245,
+      "kl": 0.31105804443359375,
+      "learning_rate": 5e-07,
+      "loss": 0.0458,
+      "reward": 0.20800781692378223,
+      "reward_std": 0.45996399596333504,
+      "rewards/end_of_conversation_reward_func": 0.03906250069849193,
+      "rewards/end_rm_reward_func": 0.2314453125,
+      "rewards/length_reward_func": -0.0625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 59
+    },
+    {
+      "clip_ratio": 0.004666702181566507,
+      "epoch": 0.048,
+      "grad_norm": 0.45620632413031426,
+      "kl": 0.1880645751953125,
+      "learning_rate": 5e-07,
+      "loss": 0.0457,
+      "step": 60
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2510.5703125,
+      "epoch": 0.0488,
+      "grad_norm": 0.62998979474741,
+      "kl": 0.158233642578125,
+      "learning_rate": 5e-07,
+      "loss": 0.0642,
+      "reward": 0.2816406446509063,
+      "reward_std": 0.35436337254941463,
+      "rewards/end_of_conversation_reward_func": 0.06093750131549314,
+      "rewards/end_rm_reward_func": 0.251953125,
+      "rewards/length_reward_func": -0.03125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 61
+    },
+    {
+      "clip_ratio": 0.006271788995945826,
+      "epoch": 0.0496,
+      "grad_norm": 0.6009428790886027,
+      "kl": 0.271392822265625,
+      "learning_rate": 5e-07,
+      "loss": 0.0639,
+      "step": 62
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2657.3671875,
+      "epoch": 0.0504,
+      "grad_norm": 0.6144033583238725,
+      "kl": 0.100860595703125,
+      "learning_rate": 5e-07,
+      "loss": 0.1188,
+      "reward": 0.266796879703179,
+      "reward_std": 0.39745712792500854,
+      "rewards/end_of_conversation_reward_func": 0.04609375091968104,
+      "rewards/end_rm_reward_func": 0.255859375,
+      "rewards/length_reward_func": -0.03515625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 63
+    },
+    {
+      "clip_ratio": 0.005829609275679104,
+      "epoch": 0.0512,
+      "grad_norm": 0.5803952639816394,
+      "kl": 0.1117401123046875,
+      "learning_rate": 5e-07,
+      "loss": 0.1186,
+      "step": 64
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2555.703125,
+      "epoch": 0.052,
+      "grad_norm": 0.607729281892395,
+      "kl": 1.57684326171875,
+      "learning_rate": 5e-07,
+      "loss": 0.0828,
+      "reward": 0.28125000261934474,
+      "reward_std": 0.4093571212142706,
+      "rewards/end_of_conversation_reward_func": 0.054687500349245965,
+      "rewards/end_rm_reward_func": 0.26171875,
+      "rewards/length_reward_func": -0.03515625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 65
+    },
+    {
+      "clip_ratio": 0.005639895083731972,
+      "epoch": 0.0528,
+      "grad_norm": 0.5870178045196847,
+      "kl": 3.4336700439453125,
+      "learning_rate": 5e-07,
+      "loss": 0.0823,
+      "step": 66
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3058.078125,
+      "epoch": 0.0536,
+      "grad_norm": 0.6392048858380123,
+      "kl": 0.2032470703125,
+      "learning_rate": 5e-07,
+      "loss": 0.0872,
+      "reward": 0.2697265713941306,
+      "reward_std": 0.46659763529896736,
+      "rewards/end_of_conversation_reward_func": 0.05000000074505806,
+      "rewards/end_rm_reward_func": 0.2705078125,
+      "rewards/length_reward_func": -0.05078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 67
+    },
+    {
+      "clip_ratio": 0.005416913933004253,
+      "epoch": 0.0544,
+      "grad_norm": 0.5540548636439222,
+      "kl": 0.2025146484375,
+      "learning_rate": 5e-07,
+      "loss": 0.0868,
+      "step": 68
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3101.0234375,
+      "epoch": 0.0552,
+      "grad_norm": 0.5462471595428263,
+      "kl": 0.8668975830078125,
+      "learning_rate": 5e-07,
+      "loss": 0.1123,
+      "reward": 0.13945313333533704,
+      "reward_std": 0.4813323845155537,
+      "rewards/end_of_conversation_reward_func": 0.0515625006519258,
+      "rewards/end_rm_reward_func": 0.134765625,
+      "rewards/length_reward_func": -0.046875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 69
+    },
+    {
+      "clip_ratio": 0.005168676114408299,
+      "epoch": 0.056,
+      "grad_norm": 0.528066600936896,
+      "kl": 0.857147216796875,
+      "learning_rate": 5e-07,
+      "loss": 0.1122,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2047.375,
+      "epoch": 0.0568,
+      "grad_norm": 0.691284897693145,
+      "kl": 15.969619750976562,
+      "learning_rate": 5e-07,
+      "loss": 0.0289,
+      "reward": 0.487109393812716,
+      "reward_std": 0.4644232392311096,
+      "rewards/end_of_conversation_reward_func": 0.07500000158324838,
+      "rewards/end_rm_reward_func": 0.427734375,
+      "rewards/length_reward_func": -0.015625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 71
+    },
+    {
+      "clip_ratio": 0.005814553995151073,
+      "epoch": 0.0576,
+      "grad_norm": 0.6404202865162691,
+      "kl": 30.811477661132812,
+      "learning_rate": 5e-07,
+      "loss": 0.0288,
+      "step": 72
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2905.2890625,
+      "epoch": 0.0584,
+      "grad_norm": 0.6445835293449337,
+      "kl": 1.801666259765625,
+      "learning_rate": 5e-07,
+      "loss": 0.0275,
+      "reward": 0.15878907404839993,
+      "reward_std": 0.3413613960146904,
+      "rewards/end_of_conversation_reward_func": 0.05625000089639798,
+      "rewards/end_rm_reward_func": 0.1337890625,
+      "rewards/length_reward_func": -0.03125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 73
+    },
+    {
+      "clip_ratio": 0.004967361441231333,
+      "epoch": 0.0592,
+      "grad_norm": 0.5958211572085673,
+      "kl": 1.816192626953125,
+      "learning_rate": 5e-07,
+      "loss": 0.027,
+      "step": 74
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3157.5546875,
+      "epoch": 0.06,
+      "grad_norm": 0.5787729026187358,
+      "kl": 1.14288330078125,
+      "learning_rate": 5e-07,
+      "loss": 0.0495,
+      "reward": 0.2250000073108822,
+      "reward_std": 0.38128331024199724,
+      "rewards/end_of_conversation_reward_func": 0.045312500617001206,
+      "rewards/end_rm_reward_func": 0.234375,
+      "rewards/length_reward_func": -0.0546875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 75
+    },
+    {
+      "clip_ratio": 0.0048050573241198435,
+      "epoch": 0.0608,
+      "grad_norm": 0.5568993189176473,
+      "kl": 0.77630615234375,
+      "learning_rate": 5e-07,
+      "loss": 0.0494,
+      "step": 76
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2372.6640625,
+      "epoch": 0.0616,
+      "grad_norm": 0.6173163766112336,
+      "kl": 0.433197021484375,
+      "learning_rate": 5e-07,
+      "loss": 0.085,
+      "reward": 0.21562500204890966,
+      "reward_std": 0.35906139435246587,
+      "rewards/end_of_conversation_reward_func": 0.055468750826548785,
+      "rewards/end_rm_reward_func": 0.1796875,
+      "rewards/length_reward_func": -0.01953125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 77
+    },
+    {
+      "clip_ratio": 0.005464794070576318,
+      "epoch": 0.0624,
+      "grad_norm": 0.5907021152198737,
+      "kl": 1.02618408203125,
+      "learning_rate": 5e-07,
+      "loss": 0.0846,
+      "step": 78
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2704.46875,
+      "epoch": 0.0632,
+      "grad_norm": 0.7611829534310949,
+      "kl": 3.341705322265625,
+      "learning_rate": 5e-07,
+      "loss": 0.0493,
+      "reward": 0.23417970072478056,
+      "reward_std": 0.36300004506483674,
+      "rewards/end_of_conversation_reward_func": 0.06328125088475645,
+      "rewards/end_rm_reward_func": 0.2255859375,
+      "rewards/length_reward_func": -0.0546875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 79
+    },
+    {
+      "clip_ratio": 0.005534523181268014,
+      "epoch": 0.064,
+      "grad_norm": 0.6276167653414357,
+      "kl": 1.77569580078125,
+      "learning_rate": 5e-07,
+      "loss": 0.0488,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2530.2265625,
+      "epoch": 0.0648,
+      "grad_norm": 0.8681960156746225,
+      "kl": 1.415252685546875,
+      "learning_rate": 5e-07,
+      "loss": 0.0355,
+      "reward": 0.31796875852160156,
+      "reward_std": 0.4489464135840535,
+      "rewards/end_of_conversation_reward_func": 0.060156250721774995,
+      "rewards/end_rm_reward_func": 0.28515625,
+      "rewards/length_reward_func": -0.02734375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 81
+    },
+    {
+      "clip_ratio": 0.006155758979730308,
+      "epoch": 0.0656,
+      "grad_norm": 0.619758928092397,
+      "kl": 3.17950439453125,
+      "learning_rate": 5e-07,
+      "loss": 0.035,
+      "step": 82
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2703.9140625,
+      "epoch": 0.0664,
+      "grad_norm": 0.6148471519549782,
+      "kl": 3.2836151123046875,
+      "learning_rate": 5e-07,
+      "loss": 0.1116,
+      "reward": 0.2703125097323209,
+      "reward_std": 0.4398716832511127,
+      "rewards/end_of_conversation_reward_func": 0.05937500071013346,
+      "rewards/end_rm_reward_func": 0.24609375,
+      "rewards/length_reward_func": -0.03515625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 83
+    },
+    {
+      "clip_ratio": 0.005438056687125936,
+      "epoch": 0.0672,
+      "grad_norm": 0.6073114120206211,
+      "kl": 2.31671142578125,
+      "learning_rate": 5e-07,
+      "loss": 0.1117,
+      "step": 84
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2557.8515625,
+      "epoch": 0.068,
+      "grad_norm": 0.6280176622559991,
+      "kl": 2.861480712890625,
+      "learning_rate": 5e-07,
+      "loss": 0.072,
+      "reward": 0.22968750726431608,
+      "reward_std": 0.3977765201125294,
+      "rewards/end_of_conversation_reward_func": 0.05781250086147338,
+      "rewards/end_rm_reward_func": 0.19921875,
+      "rewards/length_reward_func": -0.02734375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 85
+    },
+    {
+      "clip_ratio": 0.00582106395449955,
+      "epoch": 0.0688,
+      "grad_norm": 0.5717462929165856,
+      "kl": 4.08203125,
+      "learning_rate": 5e-07,
+      "loss": 0.0716,
+      "step": 86
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2243.3359375,
+      "epoch": 0.0696,
+      "grad_norm": 183.6986339811298,
+      "kl": 88.17166137695312,
+      "learning_rate": 5e-07,
+      "loss": 0.1291,
+      "reward": 0.3546875163447112,
+      "reward_std": 0.4497223272919655,
+      "rewards/end_of_conversation_reward_func": 0.06562500132713467,
+      "rewards/end_rm_reward_func": 0.3203125,
+      "rewards/length_reward_func": -0.03125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 87
+    },
+    {
+      "clip_ratio": 0.00647925166413188,
+      "epoch": 0.0704,
+      "grad_norm": 20.29826374460904,
+      "kl": 29.604248046875,
+      "learning_rate": 5e-07,
+      "loss": 0.0455,
+      "step": 88
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2714.2265625,
+      "epoch": 0.0712,
+      "grad_norm": 0.5834283983979679,
+      "kl": 4.820281982421875,
+      "learning_rate": 5e-07,
+      "loss": 0.0376,
+      "reward": 0.3076171970460564,
+      "reward_std": 0.40063817566260695,
+      "rewards/end_of_conversation_reward_func": 0.06250000093132257,
+      "rewards/end_rm_reward_func": 0.2802734375,
+      "rewards/length_reward_func": -0.03515625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 89
+    },
+    {
+      "clip_ratio": 0.005377560810302384,
+      "epoch": 0.072,
+      "grad_norm": 0.5713867955563853,
+      "kl": 4.233428955078125,
+      "learning_rate": 5e-07,
+      "loss": 0.0374,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2687.90625,
+      "epoch": 0.0728,
+      "grad_norm": 0.6405648178568916,
+      "kl": 11.98095703125,
+      "learning_rate": 5e-07,
+      "loss": 0.0957,
+      "reward": 0.18671875854488462,
+      "reward_std": 0.38381645642220974,
+      "rewards/end_of_conversation_reward_func": 0.053906251094304025,
+      "rewards/end_rm_reward_func": 0.1640625,
+      "rewards/length_reward_func": -0.03125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 91
+    },
+    {
+      "clip_ratio": 0.005748806172050536,
+      "epoch": 0.0736,
+      "grad_norm": 0.6406292246920093,
+      "kl": 20.21795654296875,
+      "learning_rate": 5e-07,
+      "loss": 0.0954,
+      "step": 92
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2914.953125,
+      "epoch": 0.0744,
+      "grad_norm": 0.5367785008596705,
+      "kl": 4.4202880859375,
+      "learning_rate": 5e-07,
+      "loss": 0.0375,
+      "reward": 0.22890625800937414,
+      "reward_std": 0.4514218312688172,
+      "rewards/end_of_conversation_reward_func": 0.053125001140870154,
+      "rewards/end_rm_reward_func": 0.19921875,
+      "rewards/length_reward_func": -0.0234375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 93
+    },
+    {
+      "clip_ratio": 0.0050481803773436695,
+      "epoch": 0.0752,
+      "grad_norm": 0.5190161687082562,
+      "kl": 4.3549346923828125,
+      "learning_rate": 5e-07,
+      "loss": 0.0372,
+      "step": 94
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 3141.1796875,
+      "epoch": 0.076,
+      "grad_norm": 0.6792418921682883,
+      "kl": 51.4798583984375,
+      "learning_rate": 5e-07,
+      "loss": 0.1449,
+      "reward": 0.2333984500146471,
+      "reward_std": 0.4384781029075384,
+      "rewards/end_of_conversation_reward_func": 0.054687501047737896,
+      "rewards/end_rm_reward_func": 0.2607421875,
+      "rewards/length_reward_func": -0.08203125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 95
+    },
+    {
+      "clip_ratio": 0.004932425799779594,
+      "epoch": 0.0768,
+      "grad_norm": 0.5823614093199818,
+      "kl": 39.2484130859375,
+      "learning_rate": 5e-07,
+      "loss": 0.1446,
+      "step": 96
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2914.5,
+      "epoch": 0.0776,
+      "grad_norm": 0.5836376252053956,
+      "kl": 3.556396484375,
+      "learning_rate": 5e-07,
+      "loss": 0.113,
+      "reward": 0.12675781635334715,
+      "reward_std": 0.3375853835605085,
+      "rewards/end_of_conversation_reward_func": 0.05546875100117177,
+      "rewards/end_rm_reward_func": 0.1181640625,
+      "rewards/length_reward_func": -0.046875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 97
+    },
+    {
+      "clip_ratio": 0.005294336486258544,
+      "epoch": 0.0784,
+      "grad_norm": 0.600979136556631,
+      "kl": 2.90380859375,
+      "learning_rate": 5e-07,
+      "loss": 0.1127,
+      "step": 98
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2303.875,
+      "epoch": 0.0792,
+      "grad_norm": 1.206634850995924,
+      "kl": 2.009918212890625,
+      "learning_rate": 5e-07,
+      "loss": 0.0546,
+      "reward": 0.30605470086447895,
+      "reward_std": 0.41795148770324886,
+      "rewards/end_of_conversation_reward_func": 0.06875000114087015,
+      "rewards/end_rm_reward_func": 0.2724609375,
+      "rewards/length_reward_func": -0.03515625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 99
+    },
+    {
+      "clip_ratio": 0.006350497540552169,
+      "epoch": 0.08,
+      "grad_norm": 0.8269327675220272,
+      "kl": 1.482818603515625,
+      "learning_rate": 5e-07,
+      "loss": 0.054,
+      "step": 100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1956.078125,
+      "epoch": 0.0808,
+      "grad_norm": 0.9861989653570116,
+      "kl": 2.6380615234375,
+      "learning_rate": 5e-07,
+      "loss": 0.0302,
+      "reward": 0.35351563640870154,
+      "reward_std": 0.46613426599651575,
+      "rewards/end_of_conversation_reward_func": 0.058593750931322575,
+      "rewards/end_rm_reward_func": 0.310546875,
+      "rewards/length_reward_func": -0.015625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 101
+    },
+    {
+      "clip_ratio": 0.007070519757689908,
+      "epoch": 0.0816,
+      "grad_norm": 0.7756715106427584,
+      "kl": 1.98809814453125,
+      "learning_rate": 5e-07,
+      "loss": 0.0295,
+      "step": 102
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2405.5546875,
+      "epoch": 0.0824,
+      "grad_norm": 0.8282537056915061,
+      "kl": 22.3995361328125,
+      "learning_rate": 5e-07,
+      "loss": 0.0809,
+      "reward": 0.2816406422643922,
+      "reward_std": 0.4782843650318682,
+      "rewards/end_of_conversation_reward_func": 0.06093750102445483,
+      "rewards/end_rm_reward_func": 0.251953125,
+      "rewards/length_reward_func": -0.03125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 103
+    },
+    {
+      "clip_ratio": 0.006699568155454472,
+      "epoch": 0.0832,
+      "grad_norm": 0.7403840647308474,
+      "kl": 18.2386474609375,
+      "learning_rate": 5e-07,
+      "loss": 0.0804,
+      "step": 104
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2866.75,
+      "epoch": 0.084,
+      "grad_norm": 0.6852221586130566,
+      "kl": 157.30657958984375,
+      "learning_rate": 5e-07,
+      "loss": 0.1108,
+      "reward": 0.13496094022411853,
+      "reward_std": 0.354372289031744,
+      "rewards/end_of_conversation_reward_func": 0.06171875010477379,
+      "rewards/end_rm_reward_func": 0.1123046875,
+      "rewards/length_reward_func": -0.0390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 105
+    },
+    {
+      "clip_ratio": 0.004895409845630638,
+      "epoch": 0.0848,
+      "grad_norm": 0.6017523870744153,
+      "kl": 252.56585693359375,
+      "learning_rate": 5e-07,
+      "loss": 0.1103,
+      "step": 106
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2348.046875,
+      "epoch": 0.0856,
+      "grad_norm": 0.914456395912047,
+      "kl": 21.73077392578125,
+      "learning_rate": 5e-07,
+      "loss": 0.0961,
+      "reward": 0.3595703274477273,
+      "reward_std": 0.4024982349947095,
+      "rewards/end_of_conversation_reward_func": 0.06562500080326572,
+      "rewards/end_rm_reward_func": 0.3095703125,
+      "rewards/length_reward_func": -0.015625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 107
+    },
+    {
+      "clip_ratio": 0.0052587240061257035,
+      "epoch": 0.0864,
+      "grad_norm": 0.7438499135211943,
+      "kl": 30.40380859375,
+      "learning_rate": 5e-07,
+      "loss": 0.0951,
+      "step": 108
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2089.40625,
+      "epoch": 0.0872,
+      "grad_norm": 0.7458266319039674,
+      "kl": 90.74044799804688,
+      "learning_rate": 5e-07,
+      "loss": 0.0166,
+      "reward": 0.43183596106246114,
+      "reward_std": 0.32141723250970244,
+      "rewards/end_of_conversation_reward_func": 0.06953125190921128,
+      "rewards/end_rm_reward_func": 0.3818359375,
+      "rewards/length_reward_func": -0.01953125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 109
+    },
+    {
+      "clip_ratio": 0.00607385064358823,
+      "epoch": 0.088,
+      "grad_norm": 0.6925768537695978,
+      "kl": 94.23519897460938,
+      "learning_rate": 5e-07,
+      "loss": 0.0162,
+      "step": 110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2093.875,
+      "epoch": 0.0888,
+      "grad_norm": 0.6947966762196172,
+      "kl": 107.85791015625,
+      "learning_rate": 5e-07,
+      "loss": 0.053,
+      "reward": 0.36738282535225153,
+      "reward_std": 0.41466561891138554,
+      "rewards/end_of_conversation_reward_func": 0.05781250121071935,
+      "rewards/end_rm_reward_func": 0.3212890625,
+      "rewards/length_reward_func": -0.01171875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 111
+    },
+    {
+      "clip_ratio": 0.006751221866579726,
+      "epoch": 0.0896,
+      "grad_norm": 0.6430098763794685,
+      "kl": 92.2412109375,
+      "learning_rate": 5e-07,
+      "loss": 0.0524,
+      "step": 112
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2857.453125,
+      "epoch": 0.0904,
+      "grad_norm": 5.39283424098446,
+      "kl": 3.7830810546875,
+      "learning_rate": 5e-07,
+      "loss": 0.0634,
+      "reward": 0.2548828301951289,
+      "reward_std": 0.3852043147198856,
+      "rewards/end_of_conversation_reward_func": 0.06640625046566129,
+      "rewards/end_rm_reward_func": 0.2431640625,
+      "rewards/length_reward_func": -0.0546875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 113
+    },
+    {
+      "clip_ratio": 0.005811518072732724,
+      "epoch": 0.0912,
+      "grad_norm": 2.422126311857217,
+      "kl": 3.3790283203125,
+      "learning_rate": 5e-07,
+      "loss": 0.0607,
+      "step": 114
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2221.609375,
+      "epoch": 0.092,
+      "grad_norm": 1.6598304873547154,
+      "kl": 20.3868408203125,
+      "learning_rate": 5e-07,
+      "loss": 0.084,
+      "reward": 0.39785158331505954,
+      "reward_std": 0.4529835293069482,
+      "rewards/end_of_conversation_reward_func": 0.07265625102445483,
+      "rewards/end_rm_reward_func": 0.3330078125,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 115
+    },
+    {
+      "clip_ratio": 0.00665678849327378,
+      "epoch": 0.0928,
+      "grad_norm": 1.0235295266605744,
+      "kl": 17.92303466796875,
+      "learning_rate": 5e-07,
+      "loss": 0.083,
+      "step": 116
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2825.5546875,
+      "epoch": 0.0936,
+      "grad_norm": 0.6848549595945271,
+      "kl": 1.4246826171875,
+      "learning_rate": 5e-07,
+      "loss": 0.0459,
+      "reward": 0.3335937559604645,
+      "reward_std": 0.3852117173373699,
+      "rewards/end_of_conversation_reward_func": 0.064062500721775,
+      "rewards/end_rm_reward_func": 0.3046875,
+      "rewards/length_reward_func": -0.03515625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 117
+    },
+    {
+      "clip_ratio": 0.006251567567232996,
+      "epoch": 0.0944,
+      "grad_norm": 0.6767843027822119,
+      "kl": 1.2928466796875,
+      "learning_rate": 5e-07,
+      "loss": 0.0453,
+      "step": 118
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2318.1640625,
+      "epoch": 0.0952,
+      "grad_norm": 10.653439671553652,
+      "kl": 23.6943359375,
+      "learning_rate": 5e-07,
+      "loss": 0.0204,
+      "reward": 0.3726562592200935,
+      "reward_std": 0.4175419304519892,
+      "rewards/end_of_conversation_reward_func": 0.06406250066356733,
+      "rewards/end_rm_reward_func": 0.33203125,
+      "rewards/length_reward_func": -0.0234375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 119
+    },
+    {
+      "clip_ratio": 0.005671160441124812,
+      "epoch": 0.096,
+      "grad_norm": 2.3552929076073736,
+      "kl": 25.97442626953125,
+      "learning_rate": 5e-07,
+      "loss": 0.0138,
+      "step": 120
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1880.5078125,
+      "epoch": 0.0968,
+      "grad_norm": 0.7833349498274812,
+      "kl": 6.0826416015625,
+      "learning_rate": 5e-07,
+      "loss": 0.0555,
+      "reward": 0.4580078381113708,
+      "reward_std": 0.47520614485256374,
+      "rewards/end_of_conversation_reward_func": 0.07031250139698386,
+      "rewards/end_rm_reward_func": 0.4111328125,
+      "rewards/length_reward_func": -0.0234375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 121
+    },
+    {
+      "clip_ratio": 0.005595599315711297,
+      "epoch": 0.0976,
+      "grad_norm": 0.7882700293427872,
+      "kl": 8.645355224609375,
+      "learning_rate": 5e-07,
+      "loss": 0.0553,
+      "step": 122
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2476.6640625,
+      "epoch": 0.0984,
+      "grad_norm": 0.912901642164902,
+      "kl": 1.38836669921875,
+      "learning_rate": 5e-07,
+      "loss": 0.0478,
+      "reward": 0.3851562616182491,
+      "reward_std": 0.4384188293479383,
+      "rewards/end_of_conversation_reward_func": 0.06093750067520887,
+      "rewards/end_rm_reward_func": 0.328125,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 123
+    },
+    {
+      "clip_ratio": 0.005667043791618198,
+      "epoch": 0.0992,
+      "grad_norm": 0.6713987475135793,
+      "kl": 1.342529296875,
+      "learning_rate": 5e-07,
+      "loss": 0.047,
+      "step": 124
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1897.9921875,
+      "epoch": 0.1,
+      "grad_norm": 1.8396052910399312,
+      "kl": 52.3482666015625,
+      "learning_rate": 5e-07,
+      "loss": 0.0427,
+      "reward": 0.36113282782025635,
+      "reward_std": 0.4283226621337235,
+      "rewards/end_of_conversation_reward_func": 0.06328125204890966,
+      "rewards/end_rm_reward_func": 0.2978515625,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 125
+    },
+    {
+      "clip_ratio": 0.008028718992136419,
+      "epoch": 0.1008,
+      "grad_norm": 0.9744864161133155,
+      "kl": 69.21954345703125,
+      "learning_rate": 5e-07,
+      "loss": 0.0419,
+      "step": 126
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2345.0234375,
+      "epoch": 0.1016,
+      "grad_norm": 0.9724894359575696,
+      "kl": 32.419189453125,
+      "learning_rate": 5e-07,
+      "loss": -0.0304,
+      "reward": 0.3443359527736902,
+      "reward_std": 0.4752398133277893,
+      "rewards/end_of_conversation_reward_func": 0.06796875188592821,
+      "rewards/end_rm_reward_func": 0.2841796875,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 127
+    },
+    {
+      "clip_ratio": 0.006735242568538524,
+      "epoch": 0.1024,
+      "grad_norm": 0.7654776812407493,
+      "kl": 16.9310302734375,
+      "learning_rate": 5e-07,
+      "loss": -0.031,
+      "step": 128
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2669.90625,
+      "epoch": 0.1032,
+      "grad_norm": 6.438509726974655,
+      "kl": 2.64239501953125,
+      "learning_rate": 5e-07,
+      "loss": 0.0327,
+      "reward": 0.21308594616129994,
+      "reward_std": 0.44294028589501977,
+      "rewards/end_of_conversation_reward_func": 0.0578125006868504,
+      "rewards/end_rm_reward_func": 0.1787109375,
+      "rewards/length_reward_func": -0.0234375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 129
+    },
+    {
+      "clip_ratio": 0.005368985250242986,
+      "epoch": 0.104,
+      "grad_norm": 0.8747379437194369,
+      "kl": 1.28466796875,
+      "learning_rate": 5e-07,
+      "loss": 0.0294,
+      "step": 130
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2500.625,
+      "epoch": 0.1048,
+      "grad_norm": 0.6477351309697729,
+      "kl": 2.7420654296875,
+      "learning_rate": 5e-07,
+      "loss": -0.0209,
+      "reward": 0.31210938445292413,
+      "reward_std": 0.37651610001921654,
+      "rewards/end_of_conversation_reward_func": 0.060156251420266926,
+      "rewards/end_rm_reward_func": 0.279296875,
+      "rewards/length_reward_func": -0.02734375,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 131
+    },
+    {
+      "clip_ratio": 0.005497576406924054,
+      "epoch": 0.1056,
+      "grad_norm": 0.6479794324694829,
+      "kl": 1.95208740234375,
+      "learning_rate": 5e-07,
+      "loss": -0.0211,
+      "step": 132
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2211.4453125,
+      "epoch": 0.1064,
+      "grad_norm": 1.0082144435462406,
+      "kl": 1.69384765625,
+      "learning_rate": 5e-07,
+      "loss": 0.0535,
+      "reward": 0.3933593933470547,
+      "reward_std": 0.4518305202946067,
+      "rewards/end_of_conversation_reward_func": 0.06328125129221007,
+      "rewards/end_rm_reward_func": 0.361328125,
+      "rewards/length_reward_func": -0.03125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 133
+    },
+    {
+      "clip_ratio": 0.0054282144556054845,
+      "epoch": 0.1072,
+      "grad_norm": 0.7238908888250458,
+      "kl": 1.4329833984375,
+      "learning_rate": 5e-07,
+      "loss": 0.0526,
+      "step": 134
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2542.8046875,
+      "epoch": 0.108,
+      "grad_norm": 0.639586024156753,
+      "kl": 125.3736572265625,
+      "learning_rate": 5e-07,
+      "loss": 0.0567,
+      "reward": 0.27167970268055797,
+      "reward_std": 0.4119821102358401,
+      "rewards/end_of_conversation_reward_func": 0.06562500086147338,
+      "rewards/end_rm_reward_func": 0.2255859375,
+      "rewards/length_reward_func": -0.01953125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 135
+    },
+    {
+      "clip_ratio": 0.005144305920111947,
+      "epoch": 0.1088,
+      "grad_norm": 0.6324039246445893,
+      "kl": 171.12823486328125,
+      "learning_rate": 5e-07,
+      "loss": 0.0565,
+      "step": 136
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2629.3046875,
+      "epoch": 0.1096,
+      "grad_norm": 0.6701878690042156,
+      "kl": 3.42401123046875,
+      "learning_rate": 5e-07,
+      "loss": 0.0713,
+      "reward": 0.30917970347218215,
+      "reward_std": 0.48418763000518084,
+      "rewards/end_of_conversation_reward_func": 0.06015625176951289,
+      "rewards/end_rm_reward_func": 0.2685546875,
+      "rewards/length_reward_func": -0.01953125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 137
+    },
+    {
+      "clip_ratio": 0.004802481475053355,
+      "epoch": 0.1104,
+      "grad_norm": 0.6296621696156859,
+      "kl": 4.0013427734375,
+      "learning_rate": 5e-07,
+      "loss": 0.0709,
+      "step": 138
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2347.1875,
+      "epoch": 0.1112,
+      "grad_norm": 0.6949091471055229,
+      "kl": 29.50958251953125,
+      "learning_rate": 5e-07,
+      "loss": 0.0567,
+      "reward": 0.23105469765141606,
+      "reward_std": 0.35395989241078496,
+      "rewards/end_of_conversation_reward_func": 0.056250001885928214,
+      "rewards/end_rm_reward_func": 0.1904296875,
+      "rewards/length_reward_func": -0.015625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 139
+    },
+    {
+      "clip_ratio": 0.005910978390602395,
+      "epoch": 0.112,
+      "grad_norm": 0.649618448371434,
+      "kl": 13.6500244140625,
+      "learning_rate": 5e-07,
+      "loss": 0.0561,
+      "step": 140
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2259.3046875,
+      "epoch": 0.1128,
+      "grad_norm": 0.7642501957021136,
+      "kl": 1.39849853515625,
+      "learning_rate": 5e-07,
+      "loss": 0.0365,
+      "reward": 0.44687501853331923,
+      "reward_std": 0.44018132728524506,
+      "rewards/end_of_conversation_reward_func": 0.06796875060535967,
+      "rewards/end_rm_reward_func": 0.390625,
+      "rewards/length_reward_func": -0.01171875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 141
+    },
+    {
+      "clip_ratio": 0.006374053191393614,
+      "epoch": 0.1136,
+      "grad_norm": 0.6891960123007369,
+      "kl": 1.7943115234375,
+      "learning_rate": 5e-07,
+      "loss": 0.0359,
+      "step": 142
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2309.171875,
+      "epoch": 0.1144,
+      "grad_norm": 0.8952677516080388,
+      "kl": 1.2454833984375,
+      "learning_rate": 5e-07,
+      "loss": 0.0781,
+      "reward": 0.4085937717463821,
+      "reward_std": 0.43159425538033247,
+      "rewards/end_of_conversation_reward_func": 0.06875000125728548,
+      "rewards/end_rm_reward_func": 0.34375,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 143
+    },
+    {
+      "clip_ratio": 0.005346491030650213,
+      "epoch": 0.1152,
+      "grad_norm": 0.8232427405557934,
+      "kl": 2.2227783203125,
+      "learning_rate": 5e-07,
+      "loss": 0.0771,
+      "step": 144
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2151.1484375,
+      "epoch": 0.116,
+      "grad_norm": 1.151846893342153,
+      "kl": 2.817626953125,
+      "learning_rate": 5e-07,
+      "loss": 0.0217,
+      "reward": 0.2933593824855052,
+      "reward_std": 0.3206550294999033,
+      "rewards/end_of_conversation_reward_func": 0.07265625079162419,
+      "rewards/end_rm_reward_func": 0.228515625,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 145
+    },
+    {
+      "clip_ratio": 0.006764089310308918,
+      "epoch": 0.1168,
+      "grad_norm": 0.9623961306886136,
+      "kl": 3.7879638671875,
+      "learning_rate": 5e-07,
+      "loss": 0.0207,
+      "step": 146
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2613.7265625,
+      "epoch": 0.1176,
+      "grad_norm": 0.924531089984747,
+      "kl": 0.8990478515625,
+      "learning_rate": 5e-07,
+      "loss": 0.0676,
+      "reward": 0.39667969988659024,
+      "reward_std": 0.4251254890114069,
+      "rewards/end_of_conversation_reward_func": 0.06562500074505806,
+      "rewards/end_rm_reward_func": 0.3466796875,
+      "rewards/length_reward_func": -0.015625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 147
+    },
+    {
+      "clip_ratio": 0.005301513941958547,
+      "epoch": 0.1184,
+      "grad_norm": 0.8069427285929358,
+      "kl": 1.39166259765625,
+      "learning_rate": 5e-07,
+      "loss": 0.0669,
+      "step": 148
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2272.7734375,
+      "epoch": 0.1192,
+      "grad_norm": 0.9723370963767047,
+      "kl": 0.997314453125,
+      "learning_rate": 5e-07,
+      "loss": 0.0187,
+      "reward": 0.4199218891444616,
+      "reward_std": 0.4015230783261359,
+      "rewards/end_of_conversation_reward_func": 0.06640625093132257,
+      "rewards/end_rm_reward_func": 0.365234375,
+      "rewards/length_reward_func": -0.01171875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 149
+    },
+    {
+      "clip_ratio": 0.005973719438770786,
+      "epoch": 0.12,
+      "grad_norm": 0.8532859182657302,
+      "kl": 0.9261474609375,
+      "learning_rate": 5e-07,
+      "loss": 0.0178,
+      "step": 150
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2505.875,
+      "epoch": 0.1208,
+      "grad_norm": 0.8842796434094805,
+      "kl": 9.147705078125,
+      "learning_rate": 5e-07,
+      "loss": 0.0431,
+      "reward": 0.3224609545432031,
+      "reward_std": 0.512933537364006,
+      "rewards/end_of_conversation_reward_func": 0.06171875155996531,
+      "rewards/end_rm_reward_func": 0.2724609375,
+      "rewards/length_reward_func": -0.01171875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 151
+    },
+    {
+      "clip_ratio": 0.006512533509521745,
+      "epoch": 0.1216,
+      "grad_norm": 0.8332956238777746,
+      "kl": 6.146240234375,
+      "learning_rate": 5e-07,
+      "loss": 0.0423,
+      "step": 152
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2544.953125,
+      "epoch": 0.1224,
+      "grad_norm": 0.9874027538635356,
+      "kl": 14.062255859375,
+      "learning_rate": 5e-07,
+      "loss": -0.0164,
+      "reward": 0.17890625371364877,
+      "reward_std": 0.379497186280787,
+      "rewards/end_of_conversation_reward_func": 0.053906250395812094,
+      "rewards/end_rm_reward_func": 0.13671875,
+      "rewards/length_reward_func": -0.01171875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 153
+    },
+    {
+      "clip_ratio": 0.004760816096677445,
+      "epoch": 0.1232,
+      "grad_norm": 0.8194951383155811,
+      "kl": 25.712646484375,
+      "learning_rate": 5e-07,
+      "loss": -0.017,
+      "step": 154
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2325.8828125,
+      "epoch": 0.124,
+      "grad_norm": 0.8787405473781513,
+      "kl": 2.60302734375,
+      "learning_rate": 5e-07,
+      "loss": 0.0489,
+      "reward": 0.43007815058808774,
+      "reward_std": 0.37591410893946886,
+      "rewards/end_of_conversation_reward_func": 0.07656250160653144,
+      "rewards/end_rm_reward_func": 0.373046875,
+      "rewards/length_reward_func": -0.01953125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 155
+    },
+    {
+      "clip_ratio": 0.005131626487127505,
+      "epoch": 0.1248,
+      "grad_norm": 0.860882474096142,
+      "kl": 2.1358642578125,
+      "learning_rate": 5e-07,
+      "loss": 0.0482,
+      "step": 156
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1997.8984375,
+      "epoch": 0.1256,
+      "grad_norm": 4.174478136933485,
+      "kl": 126.7437744140625,
+      "learning_rate": 5e-07,
+      "loss": 0.0147,
+      "reward": 0.3750000196741894,
+      "reward_std": 0.40887491311877966,
+      "rewards/end_of_conversation_reward_func": 0.07421875069849193,
+      "rewards/end_rm_reward_func": 0.3046875,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 157
+    },
+    {
+      "clip_ratio": 0.006120361678767949,
+      "epoch": 0.1264,
+      "grad_norm": 1.4963821994530664,
+      "kl": 273.1322021484375,
+      "learning_rate": 5e-07,
+      "loss": 0.0139,
+      "step": 158
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2511.359375,
+      "epoch": 0.1272,
+      "grad_norm": 5.367528712708823,
+      "kl": 5.2606201171875,
+      "learning_rate": 5e-07,
+      "loss": 0.0698,
+      "reward": 0.3287109578377567,
+      "reward_std": 0.3992913840338588,
+      "rewards/end_of_conversation_reward_func": 0.06406250142026693,
+      "rewards/end_rm_reward_func": 0.2724609375,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 159
+    },
+    {
+      "clip_ratio": 0.005198820639634505,
+      "epoch": 0.128,
+      "grad_norm": 2.8531678389715998,
+      "kl": 4.1246337890625,
+      "learning_rate": 5e-07,
+      "loss": 0.0663,
+      "step": 160
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2238.2734375,
+      "epoch": 0.1288,
+      "grad_norm": 1.2910611646405146,
+      "kl": 2.57861328125,
+      "learning_rate": 5e-07,
+      "loss": 0.0318,
+      "reward": 0.24882814101874828,
+      "reward_std": 0.3937357231043279,
+      "rewards/end_of_conversation_reward_func": 0.0710937506519258,
+      "rewards/end_rm_reward_func": 0.185546875,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 161
+    },
+    {
+      "clip_ratio": 0.006061407417291775,
+      "epoch": 0.1296,
+      "grad_norm": 1.1788151032961487,
+      "kl": 2.96826171875,
+      "learning_rate": 5e-07,
+      "loss": 0.0305,
+      "step": 162
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1621.8671875,
+      "epoch": 0.1304,
+      "grad_norm": 1.082482183152501,
+      "kl": 22.7098388671875,
+      "learning_rate": 5e-07,
+      "loss": 0.0309,
+      "reward": 0.4742187724914402,
+      "reward_std": 0.42131141386926174,
+      "rewards/end_of_conversation_reward_func": 0.07968750153668225,
+      "rewards/end_rm_reward_func": 0.39453125,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 163
+    },
+    {
+      "clip_ratio": 0.007301741745322943,
+      "epoch": 0.1312,
+      "grad_norm": 0.999311109978907,
+      "kl": 28.77685546875,
+      "learning_rate": 5e-07,
+      "loss": 0.0297,
+      "step": 164
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2164.078125,
+      "epoch": 0.132,
+      "grad_norm": 1.2711815845501575,
+      "kl": 11.022705078125,
+      "learning_rate": 5e-07,
+      "loss": 0.007,
+      "reward": 0.266796886222437,
+      "reward_std": 0.38234878331422806,
+      "rewards/end_of_conversation_reward_func": 0.06953125045401976,
+      "rewards/end_rm_reward_func": 0.208984375,
+      "rewards/length_reward_func": -0.01171875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 165
+    },
+    {
+      "clip_ratio": 0.005417483043856919,
+      "epoch": 0.1328,
+      "grad_norm": 1.0349824323527632,
+      "kl": 56.939208984375,
+      "learning_rate": 5e-07,
+      "loss": 0.0058,
+      "step": 166
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1862.1171875,
+      "epoch": 0.1336,
+      "grad_norm": 1.9132332469984104,
+      "kl": 15.81640625,
+      "learning_rate": 5e-07,
+      "loss": 0.008,
+      "reward": 0.20527344779111445,
+      "reward_std": 0.31551461201161146,
+      "rewards/end_of_conversation_reward_func": 0.07734375126892701,
+      "rewards/end_rm_reward_func": 0.1318359375,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 167
+    },
+    {
+      "clip_ratio": 0.006755854556104168,
+      "epoch": 0.1344,
+      "grad_norm": 1.1876373433288951,
+      "kl": 14.31689453125,
+      "learning_rate": 5e-07,
+      "loss": 0.0065,
+      "step": 168
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1881.125,
+      "epoch": 0.1352,
+      "grad_norm": 1.1641180467318737,
+      "kl": 314.045654296875,
+      "learning_rate": 5e-07,
+      "loss": 0.0178,
+      "reward": 0.3947265843162313,
+      "reward_std": 0.41996311163529754,
+      "rewards/end_of_conversation_reward_func": 0.07343749981373549,
+      "rewards/end_rm_reward_func": 0.3291015625,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 169
+    },
+    {
+      "clip_ratio": 0.0057207249192288145,
+      "epoch": 0.136,
+      "grad_norm": 1.0261348607435299,
+      "kl": 365.5631103515625,
+      "learning_rate": 5e-07,
+      "loss": 0.0164,
+      "step": 170
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2508.2734375,
+      "epoch": 0.1368,
+      "grad_norm": 1.8601217689473675,
+      "kl": 4.70849609375,
+      "learning_rate": 5e-07,
+      "loss": 0.0189,
+      "reward": 0.37148438789881766,
+      "reward_std": 0.39709692588075995,
+      "rewards/end_of_conversation_reward_func": 0.06874999997671694,
+      "rewards/end_rm_reward_func": 0.310546875,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 171
+    },
+    {
+      "clip_ratio": 0.004716894996818155,
+      "epoch": 0.1376,
+      "grad_norm": 1.1874233704490673,
+      "kl": 17.053955078125,
+      "learning_rate": 5e-07,
+      "loss": 0.017,
+      "step": 172
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2176.171875,
+      "epoch": 0.1384,
+      "grad_norm": 2.060042420988078,
+      "kl": 181.342529296875,
+      "learning_rate": 5e-07,
+      "loss": 0.0493,
+      "reward": 0.30000001145526767,
+      "reward_std": 0.35801823483780026,
+      "rewards/end_of_conversation_reward_func": 0.06953125121071935,
+      "rewards/end_rm_reward_func": 0.23046875,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 173
+    },
+    {
+      "clip_ratio": 0.012096417922293767,
+      "epoch": 0.1392,
+      "grad_norm": 2.4188316605382605,
+      "kl": 385.28857421875,
+      "learning_rate": 5e-07,
+      "loss": 0.0476,
+      "step": 174
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1939.6875,
+      "epoch": 0.14,
+      "grad_norm": 2.2213507537695856,
+      "kl": 412.21533203125,
+      "learning_rate": 5e-07,
+      "loss": 0.0376,
+      "reward": 0.5171875233063474,
+      "reward_std": 0.4006890896707773,
+      "rewards/end_of_conversation_reward_func": 0.0796875013038516,
+      "rewards/end_rm_reward_func": 0.44140625,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 175
+    },
+    {
+      "clip_ratio": 0.008669883391121402,
+      "epoch": 0.1408,
+      "grad_norm": 2.0154021188215534,
+      "kl": 349.34228515625,
+      "learning_rate": 5e-07,
+      "loss": 0.035,
+      "step": 176
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2474.7265625,
+      "epoch": 0.1416,
+      "grad_norm": 2.9429100925925993,
+      "kl": 36.90185546875,
+      "learning_rate": 5e-07,
+      "loss": 0.0301,
+      "reward": 0.47812502505257726,
+      "reward_std": 0.4000142142176628,
+      "rewards/end_of_conversation_reward_func": 0.07578125200234354,
+      "rewards/end_rm_reward_func": 0.4140625,
+      "rewards/length_reward_func": -0.01171875,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 177
+    },
+    {
+      "clip_ratio": 0.007487929397029802,
+      "epoch": 0.1424,
+      "grad_norm": 1.7072132896520176,
+      "kl": 37.833984375,
+      "learning_rate": 5e-07,
+      "loss": 0.0267,
+      "step": 178
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2216.28125,
+      "epoch": 0.1432,
+      "grad_norm": 3.4715981881060096,
+      "kl": 153.06494140625,
+      "learning_rate": 5e-07,
+      "loss": 0.0629,
+      "reward": 0.430859393440187,
+      "reward_std": 0.4135680345352739,
+      "rewards/end_of_conversation_reward_func": 0.06562500190921128,
+      "rewards/end_rm_reward_func": 0.369140625,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 179
+    },
+    {
+      "clip_ratio": 0.007125947595341131,
+      "epoch": 0.144,
+      "grad_norm": 1.7642237159324998,
+      "kl": 372.9677734375,
+      "learning_rate": 5e-07,
+      "loss": 0.0582,
+      "step": 180
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2396.046875,
+      "epoch": 0.1448,
+      "grad_norm": 2.626090932602815,
+      "kl": 134.4459228515625,
+      "learning_rate": 5e-07,
+      "loss": 0.0593,
+      "reward": 0.3869140747701749,
+      "reward_std": 0.3422806654125452,
+      "rewards/end_of_conversation_reward_func": 0.06953125167638063,
+      "rewards/end_rm_reward_func": 0.3212890625,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 181
+    },
+    {
+      "clip_ratio": 0.01481604718719609,
+      "epoch": 0.1456,
+      "grad_norm": 4.174800170660322,
+      "kl": 131.9783935546875,
+      "learning_rate": 5e-07,
+      "loss": 0.0572,
+      "step": 182
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2284.40625,
+      "epoch": 0.1464,
+      "grad_norm": 4.005039457486521,
+      "kl": 1621.6591796875,
+      "learning_rate": 5e-07,
+      "loss": 0.0388,
+      "reward": 0.3603515757713467,
+      "reward_std": 0.4121582773514092,
+      "rewards/end_of_conversation_reward_func": 0.07421875139698386,
+      "rewards/end_rm_reward_func": 0.2861328125,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 183
+    },
+    {
+      "clip_ratio": 0.009662889962783083,
+      "epoch": 0.1472,
+      "grad_norm": 2.116076197062108,
+      "kl": 1933.222412109375,
+      "learning_rate": 5e-07,
+      "loss": 0.0342,
+      "step": 184
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2164.0546875,
+      "epoch": 0.148,
+      "grad_norm": 3.79272536544489,
+      "kl": 8153.509033203125,
+      "learning_rate": 5e-07,
+      "loss": 0.0173,
+      "reward": 0.24707032029982656,
+      "reward_std": 0.33177149668335915,
+      "rewards/end_of_conversation_reward_func": 0.07031250128056854,
+      "rewards/end_rm_reward_func": 0.1767578125,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 185
+    },
+    {
+      "clip_ratio": 0.007355456036748365,
+      "epoch": 0.1488,
+      "grad_norm": 2.492559543240363,
+      "kl": 19430.03955078125,
+      "learning_rate": 5e-07,
+      "loss": 0.0143,
+      "step": 186
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2088.5234375,
+      "epoch": 0.1496,
+      "grad_norm": 3.215340142433475,
+      "kl": 1088.0703125,
+      "learning_rate": 5e-07,
+      "loss": 0.0347,
+      "reward": 0.32714844518341124,
+      "reward_std": 0.3099647618364543,
+      "rewards/end_of_conversation_reward_func": 0.07812500139698386,
+      "rewards/end_rm_reward_func": 0.2490234375,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 187
+    },
+    {
+      "clip_ratio": 0.007153055747039616,
+      "epoch": 0.1504,
+      "grad_norm": 2.979896488060616,
+      "kl": 884.30126953125,
+      "learning_rate": 5e-07,
+      "loss": 0.0304,
+      "step": 188
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2295.28125,
+      "epoch": 0.1512,
+      "grad_norm": 4.27184297703827,
+      "kl": 24.62060546875,
+      "learning_rate": 5e-07,
+      "loss": -0.0026,
+      "reward": 0.48769533238373697,
+      "reward_std": 0.4020336586982012,
+      "rewards/end_of_conversation_reward_func": 0.07265625131549314,
+      "rewards/end_rm_reward_func": 0.4150390625,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 189
+    },
+    {
+      "clip_ratio": 0.005939892347669229,
+      "epoch": 0.152,
+      "grad_norm": 3.3442192954819,
+      "kl": 17.80908203125,
+      "learning_rate": 5e-07,
+      "loss": -0.0069,
+      "step": 190
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2123.578125,
+      "epoch": 0.1528,
+      "grad_norm": 6.018069933911673,
+      "kl": 6821.18798828125,
+      "learning_rate": 5e-07,
+      "loss": 0.0186,
+      "reward": 0.44394533056765795,
+      "reward_std": 0.4120704475790262,
+      "rewards/end_of_conversation_reward_func": 0.07968750083819032,
+      "rewards/end_rm_reward_func": 0.3642578125,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 191
+    },
+    {
+      "clip_ratio": 0.019422179379034787,
+      "epoch": 0.1536,
+      "grad_norm": 6.0000850540905555,
+      "kl": 10920.389404296875,
+      "learning_rate": 5e-07,
+      "loss": 0.0159,
+      "step": 192
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2315.28125,
+      "epoch": 0.1544,
+      "grad_norm": 5.51028975235637,
+      "kl": 113.30224609375,
+      "learning_rate": 5e-07,
+      "loss": 0.0369,
+      "reward": 0.41445315489545465,
+      "reward_std": 0.40713366749696434,
+      "rewards/end_of_conversation_reward_func": 0.08046875178115442,
+      "rewards/end_rm_reward_func": 0.341796875,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 193
+    },
+    {
+      "clip_ratio": 0.010416857578093186,
+      "epoch": 0.1552,
+      "grad_norm": 4.641303725834707,
+      "kl": 79.856689453125,
+      "learning_rate": 5e-07,
+      "loss": 0.0318,
+      "step": 194
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2337.7265625,
+      "epoch": 0.156,
+      "grad_norm": 7.337511224807291,
+      "kl": 121.304443359375,
+      "learning_rate": 5e-07,
+      "loss": 0.0442,
+      "reward": 0.3572265817783773,
+      "reward_std": 0.44290671218186617,
+      "rewards/end_of_conversation_reward_func": 0.07890625100117177,
+      "rewards/end_rm_reward_func": 0.2822265625,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 195
+    },
+    {
+      "clip_ratio": 0.01027121691731736,
+      "epoch": 0.1568,
+      "grad_norm": 5.231827919249724,
+      "kl": 134.763427734375,
+      "learning_rate": 5e-07,
+      "loss": 0.0394,
+      "step": 196
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2069.0546875,
+      "epoch": 0.1576,
+      "grad_norm": 7.402698135110819,
+      "kl": 52.291015625,
+      "learning_rate": 5e-07,
+      "loss": 0.0311,
+      "reward": 0.38632813934236765,
+      "reward_std": 0.3869206882081926,
+      "rewards/end_of_conversation_reward_func": 0.07187500083819032,
+      "rewards/end_rm_reward_func": 0.318359375,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 197
+    },
+    {
+      "clip_ratio": 0.006990124413277954,
+      "epoch": 0.1584,
+      "grad_norm": 113.50707154622745,
+      "kl": 21.02978515625,
+      "learning_rate": 5e-07,
+      "loss": 0.0278,
+      "step": 198
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2180.6640625,
+      "epoch": 0.1592,
+      "grad_norm": 8.268979237999838,
+      "kl": 146.75,
+      "learning_rate": 5e-07,
+      "loss": 0.0363,
+      "reward": 0.5003906521014869,
+      "reward_std": 0.4240156547166407,
+      "rewards/end_of_conversation_reward_func": 0.0804687519557774,
+      "rewards/end_rm_reward_func": 0.419921875,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 199
+    },
+    {
+      "clip_ratio": 0.0194309915532358,
+      "epoch": 0.16,
+      "grad_norm": 8.843678882129485,
+      "kl": 351.8544921875,
+      "learning_rate": 5e-07,
+      "loss": 0.0337,
+      "step": 200
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2056.3515625,
+      "epoch": 0.1608,
+      "grad_norm": 6.868955922992306,
+      "kl": 53.767578125,
+      "learning_rate": 5e-07,
+      "loss": 0.036,
+      "reward": 0.45488282945007086,
+      "reward_std": 0.41126722283661366,
+      "rewards/end_of_conversation_reward_func": 0.0867187506519258,
+      "rewards/end_rm_reward_func": 0.3681640625,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 201
+    },
+    {
+      "clip_ratio": 0.004734704503789544,
+      "epoch": 0.1616,
+      "grad_norm": 4.719162650271088,
+      "kl": 35.9853515625,
+      "learning_rate": 5e-07,
+      "loss": 0.0269,
+      "step": 202
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2449.8671875,
+      "epoch": 0.1624,
+      "grad_norm": 9.848426982264245,
+      "kl": 104.44140625,
+      "learning_rate": 5e-07,
+      "loss": 0.0108,
+      "reward": 0.3212890766444616,
+      "reward_std": 0.34017279790714383,
+      "rewards/end_of_conversation_reward_func": 0.06640625069849193,
+      "rewards/end_rm_reward_func": 0.2587890625,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 203
+    },
+    {
+      "clip_ratio": 0.012477430194849148,
+      "epoch": 0.1632,
+      "grad_norm": 5.484298870932829,
+      "kl": 146.384765625,
+      "learning_rate": 5e-07,
+      "loss": 0.0006,
+      "step": 204
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1957.1015625,
+      "epoch": 0.164,
+      "grad_norm": 11.459849219468296,
+      "kl": 317.099609375,
+      "learning_rate": 5e-07,
+      "loss": 0.0305,
+      "reward": 0.48828127491287887,
+      "reward_std": 0.5096489517018199,
+      "rewards/end_of_conversation_reward_func": 0.07812500046566129,
+      "rewards/end_rm_reward_func": 0.41015625,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 205
+    },
+    {
+      "clip_ratio": 0.008903506706701592,
+      "epoch": 0.1648,
+      "grad_norm": 6.73218235011968,
+      "kl": 414.3935546875,
+      "learning_rate": 5e-07,
+      "loss": 0.018,
+      "step": 206
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1970.984375,
+      "epoch": 0.1656,
+      "grad_norm": 8.656738486892705,
+      "kl": 246.80859375,
+      "learning_rate": 5e-07,
+      "loss": 0.0585,
+      "reward": 0.5083984585944563,
+      "reward_std": 0.4745393330231309,
+      "rewards/end_of_conversation_reward_func": 0.07578125153668225,
+      "rewards/end_rm_reward_func": 0.4365234375,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 207
+    },
+    {
+      "clip_ratio": 0.04178383277030662,
+      "epoch": 0.1664,
+      "grad_norm": 20.922629331798802,
+      "kl": 279.943359375,
+      "learning_rate": 5e-07,
+      "loss": 0.0645,
+      "step": 208
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1793.3828125,
+      "epoch": 0.1672,
+      "grad_norm": 10.449885777859176,
+      "kl": 294.4990234375,
+      "learning_rate": 5e-07,
+      "loss": 0.0182,
+      "reward": 0.5263672126457095,
+      "reward_std": 0.4001456112600863,
+      "rewards/end_of_conversation_reward_func": 0.08203125139698386,
+      "rewards/end_rm_reward_func": 0.4443359375,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 209
+    },
+    {
+      "clip_ratio": 0.025160407996736467,
+      "epoch": 0.168,
+      "grad_norm": 17.92069739644721,
+      "kl": 305.3837890625,
+      "learning_rate": 5e-07,
+      "loss": 0.0173,
+      "step": 210
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2268.9609375,
+      "epoch": 0.1688,
+      "grad_norm": 73.96836941822615,
+      "kl": 747.4951171875,
+      "learning_rate": 5e-07,
+      "loss": 0.1343,
+      "reward": 0.36914064013399184,
+      "reward_std": 0.3273033651057631,
+      "rewards/end_of_conversation_reward_func": 0.07812500093132257,
+      "rewards/end_rm_reward_func": 0.291015625,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 211
+    },
+    {
+      "clip_ratio": 0.012499632517574355,
+      "epoch": 0.1696,
+      "grad_norm": 18.974576313134882,
+      "kl": 611.49267578125,
+      "learning_rate": 5e-07,
+      "loss": 0.0721,
+      "step": 212
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2230.0,
+      "epoch": 0.1704,
+      "grad_norm": 6.681940389271876,
+      "kl": 14805.2890625,
+      "learning_rate": 5e-07,
+      "loss": 0.0276,
+      "reward": 0.3503906426485628,
+      "reward_std": 0.4293831647373736,
+      "rewards/end_of_conversation_reward_func": 0.06718750135041773,
+      "rewards/end_rm_reward_func": 0.283203125,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 213
+    },
+    {
+      "clip_ratio": 0.036163119453703985,
+      "epoch": 0.1712,
+      "grad_norm": 20.214531495705373,
+      "kl": 30367.22509765625,
+      "learning_rate": 5e-07,
+      "loss": 0.0295,
+      "step": 214
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2377.1953125,
+      "epoch": 0.172,
+      "grad_norm": 7.270360214395911,
+      "kl": 457.0078125,
+      "learning_rate": 5e-07,
+      "loss": 0.0059,
+      "reward": 0.3910156455822289,
+      "reward_std": 0.3999810148961842,
+      "rewards/end_of_conversation_reward_func": 0.08046875172294676,
+      "rewards/end_rm_reward_func": 0.310546875,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 215
+    },
+    {
+      "clip_ratio": 0.006911784948897548,
+      "epoch": 0.1728,
+      "grad_norm": 6.317486752283868,
+      "kl": 325.4482421875,
+      "learning_rate": 5e-07,
+      "loss": -0.0004,
+      "step": 216
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2922.046875,
+      "epoch": 0.1736,
+      "grad_norm": 6.82709648692313,
+      "kl": 3166.01025390625,
+      "learning_rate": 5e-07,
+      "loss": 0.0348,
+      "reward": 0.27285157423466444,
+      "reward_std": 0.4200313645415008,
+      "rewards/end_of_conversation_reward_func": 0.07265625067520887,
+      "rewards/end_rm_reward_func": 0.2041015625,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 217
+    },
+    {
+      "clip_ratio": 0.00761759614397306,
+      "epoch": 0.1744,
+      "grad_norm": 6.808138897589715,
+      "kl": 3725.60400390625,
+      "learning_rate": 5e-07,
+      "loss": 0.0255,
+      "step": 218
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1952.4453125,
+      "epoch": 0.1752,
+      "grad_norm": 8.424241871323705,
+      "kl": 1067.37109375,
+      "learning_rate": 5e-07,
+      "loss": 0.0463,
+      "reward": 0.34335939423181117,
+      "reward_std": 0.297838720260188,
+      "rewards/end_of_conversation_reward_func": 0.07578125176951289,
+      "rewards/end_rm_reward_func": 0.275390625,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 219
+    },
+    {
+      "clip_ratio": 0.010116227756952867,
+      "epoch": 0.176,
+      "grad_norm": 5.581257792673027,
+      "kl": 643.158203125,
+      "learning_rate": 5e-07,
+      "loss": 0.0394,
+      "step": 220
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2187.0078125,
+      "epoch": 0.1768,
+      "grad_norm": 8.701946640071931,
+      "kl": 44.845703125,
+      "learning_rate": 5e-07,
+      "loss": 0.0026,
+      "reward": 0.36250001611188054,
+      "reward_std": 0.4365540761500597,
+      "rewards/end_of_conversation_reward_func": 0.07734375179279596,
+      "rewards/end_rm_reward_func": 0.28515625,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 221
+    },
+    {
+      "clip_ratio": 0.012115928897401318,
+      "epoch": 0.1776,
+      "grad_norm": 6.840735832043148,
+      "kl": 76.4052734375,
+      "learning_rate": 5e-07,
+      "loss": -0.0039,
+      "step": 222
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2262.546875,
+      "epoch": 0.1784,
+      "grad_norm": 11.037911784681958,
+      "kl": 392.87890625,
+      "learning_rate": 5e-07,
+      "loss": 0.0408,
+      "reward": 0.3087890681345016,
+      "reward_std": 0.40431320294737816,
+      "rewards/end_of_conversation_reward_func": 0.07343750074505806,
+      "rewards/end_rm_reward_func": 0.2392578125,
+      "rewards/length_reward_func": -0.00390625,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 223
+    },
+    {
+      "clip_ratio": 0.020739365107147023,
+      "epoch": 0.1792,
+      "grad_norm": 10.349410625297242,
+      "kl": 446.98828125,
+      "learning_rate": 5e-07,
+      "loss": 0.0332,
+      "step": 224
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2353.828125,
+      "epoch": 0.18,
+      "grad_norm": 7.857636457593067,
+      "kl": 49.89697265625,
+      "learning_rate": 5e-07,
+      "loss": 0.0002,
+      "reward": 0.397070333128795,
+      "reward_std": 0.3746516448445618,
+      "rewards/end_of_conversation_reward_func": 0.08359375083819032,
+      "rewards/end_rm_reward_func": 0.3134765625,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 225
+    },
+    {
+      "clip_ratio": 0.04557889769785106,
+      "epoch": 0.1808,
+      "grad_norm": 32.46158867190515,
+      "kl": 143.626953125,
+      "learning_rate": 5e-07,
+      "loss": 0.008,
+      "step": 226
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1967.0078125,
+      "epoch": 0.1816,
+      "grad_norm": 7.690726728971795,
+      "kl": 168.8681640625,
+      "learning_rate": 5e-07,
+      "loss": 0.0427,
+      "reward": 0.4964843970956281,
+      "reward_std": 0.4297056058421731,
+      "rewards/end_of_conversation_reward_func": 0.08046875172294676,
+      "rewards/end_rm_reward_func": 0.416015625,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 227
+    },
+    {
+      "clip_ratio": 0.020050687453476712,
+      "epoch": 0.1824,
+      "grad_norm": 12.623134353539193,
+      "kl": 229.0390625,
+      "learning_rate": 5e-07,
+      "loss": 0.0382,
+      "step": 228
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2303.3203125,
+      "epoch": 0.1832,
+      "grad_norm": 8.985778261923576,
+      "kl": 1521.31640625,
+      "learning_rate": 5e-07,
+      "loss": 0.0609,
+      "reward": 0.34335938887670636,
+      "reward_std": 0.4365329034626484,
+      "rewards/end_of_conversation_reward_func": 0.07187500095460564,
+      "rewards/end_rm_reward_func": 0.271484375,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 229
+    },
+    {
+      "clip_ratio": 0.046887944103218615,
+      "epoch": 0.184,
+      "grad_norm": 34.903609321208826,
+      "kl": 1591.25390625,
+      "learning_rate": 5e-07,
+      "loss": 0.0655,
+      "step": 230
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2103.6015625,
+      "epoch": 0.1848,
+      "grad_norm": 7.8152371982851045,
+      "kl": 93.3984375,
+      "learning_rate": 5e-07,
+      "loss": 0.021,
+      "reward": 0.41718751774169505,
+      "reward_std": 0.4234715709462762,
+      "rewards/end_of_conversation_reward_func": 0.07734375167638063,
+      "rewards/end_rm_reward_func": 0.33984375,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 231
+    },
+    {
+      "clip_ratio": 0.01693671362590976,
+      "epoch": 0.1856,
+      "grad_norm": 15.100433483062815,
+      "kl": 119.0732421875,
+      "learning_rate": 5e-07,
+      "loss": 0.0166,
+      "step": 232
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2187.609375,
+      "epoch": 0.1864,
+      "grad_norm": 10.645873330116869,
+      "kl": 176.8994140625,
+      "learning_rate": 5e-07,
+      "loss": 0.0302,
+      "reward": 0.39726564241573215,
+      "reward_std": 0.3991687847301364,
+      "rewards/end_of_conversation_reward_func": 0.07109375053551048,
+      "rewards/end_rm_reward_func": 0.333984375,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 233
+    },
+    {
+      "clip_ratio": 0.009701315342681482,
+      "epoch": 0.1872,
+      "grad_norm": 9.681021841986935,
+      "kl": 173.751953125,
+      "learning_rate": 5e-07,
+      "loss": 0.0213,
+      "step": 234
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2018.2421875,
+      "epoch": 0.188,
+      "grad_norm": 9.837481125291612,
+      "kl": 250.2099609375,
+      "learning_rate": 5e-07,
+      "loss": 0.0135,
+      "reward": 0.3800781453028321,
+      "reward_std": 0.302262753713876,
+      "rewards/end_of_conversation_reward_func": 0.08125000144354999,
+      "rewards/end_rm_reward_func": 0.306640625,
+      "rewards/length_reward_func": -0.0078125,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 235
+    },
+    {
+      "clip_ratio": 0.02462595998076722,
+      "epoch": 0.1888,
+      "grad_norm": 18.237167770716425,
+      "kl": 475.810546875,
+      "learning_rate": 5e-07,
+      "loss": 0.0139,
+      "step": 236
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 2196.796875,
+      "epoch": 0.1896,
+      "grad_norm": 10.707362677641337,
+      "kl": 2000.958984375,
+      "learning_rate": 5e-07,
+      "loss": 0.0229,
+      "reward": 0.38046876690350473,
+      "reward_std": 0.30730303283780813,
+      "rewards/end_of_conversation_reward_func": 0.07578125095460564,
+      "rewards/end_rm_reward_func": 0.3046875,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 237
+    },
+    {
+      "clip_ratio": 0.00941651189350523,
+      "epoch": 0.1904,
+      "grad_norm": 7.341564288985746,
+      "kl": 3464.6708984375,
+      "learning_rate": 5e-07,
+      "loss": 0.0123,
+      "step": 238
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1952.6328125,
+      "epoch": 0.1912,
+      "grad_norm": 9.746913854316412,
+      "kl": 106.2119140625,
+      "learning_rate": 5e-07,
+      "loss": 0.0185,
+      "reward": 0.37890626420266926,
+      "reward_std": 0.44265066692605615,
+      "rewards/end_of_conversation_reward_func": 0.07031250128056854,
+      "rewards/end_rm_reward_func": 0.30859375,
+      "rewards/length_reward_func": 0.0,
+      "rewards/thinking_reward_func": 0.0,
+      "step": 239
+    },
+    {
+      "clip_ratio": 0.01394331140909344,
+      "epoch": 0.192,
+      "grad_norm": 10.146789021120865,
+      "kl": 102.3330078125,
+      "learning_rate": 5e-07,
+      "loss": 0.0135,
+      "step": 240
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1250,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 10,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6891687f44ecc1193350e947b9566f73161cba7a6ff057e809b1550a9d401e4
+size 7800