Model save

README.md

@@ -0,0 +1,68 @@
+---
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+library_name: transformers
+model_name: Qwen2.5-1.5B-Open-R1-GRPO
+tags:
+- generated_from_trainer
+- trl
+- grpo
+licence: license
+---
+
+# Model Card for Qwen2.5-1.5B-Open-R1-GRPO
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="bruel/Qwen2.5-1.5B-Open-R1-GRPO", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/unboxmagic/huggingface/runs/smvz4ow1) 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.15.0.dev0
+- Transformers: 4.49.0.dev0
+- Pytorch: 2.5.1
+- Datasets: 3.2.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 379.5353878517854,
+    "train_runtime": 232403.7931,
+    "train_samples": 72441,
+    "train_samples_per_second": 0.312,
+    "train_steps_per_second": 0.003
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.49.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97efeaf4c66077e459963a977201234187bf76301ead3dd8e0224e0f26b514d5
+size 1531449344

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 379.5353878517854,
+    "train_runtime": 232403.7931,
+    "train_samples": 72441,
+    "train_samples_per_second": 0.312,
+    "train_steps_per_second": 0.003
+}

trainer_state.json

@@ -0,0 +1,2081 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9987438399845395,
+  "eval_steps": 100,
+  "global_step": 646,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 393.6643056869507,
+      "epoch": 0.007730215479756498,
+      "grad_norm": 0.8319699187669469,
+      "kl": 0.00013954639434814452,
+      "learning_rate": 1.5384615384615387e-06,
+      "loss": 0.0,
+      "reward": 0.7195449007675052,
+      "reward_std": 0.5224083874374628,
+      "rewards/accuracy_reward": 0.16517857974395156,
+      "rewards/cosine_scaled_reward": -0.09027656889520586,
+      "rewards/format_reward": 0.46517859138548373,
+      "rewards/reasoning_steps_reward": 0.17946429918520151,
+      "step": 5
+    },
+    {
+      "completion_length": 388.05269584655764,
+      "epoch": 0.015460430959512996,
+      "grad_norm": 0.8123763229790176,
+      "kl": 0.0022653818130493166,
+      "learning_rate": 3.0769230769230774e-06,
+      "loss": 0.0001,
+      "reward": 0.7006159001961351,
+      "reward_std": 0.5306106515228748,
+      "rewards/accuracy_reward": 0.1410714359022677,
+      "rewards/cosine_scaled_reward": -0.11307462062686682,
+      "rewards/format_reward": 0.5080357354134322,
+      "rewards/reasoning_steps_reward": 0.16458334433846175,
+      "step": 10
+    },
+    {
+      "completion_length": 248.5607237815857,
+      "epoch": 0.023190646439269495,
+      "grad_norm": 1.699419290702465,
+      "kl": 0.03936920166015625,
+      "learning_rate": 4.615384615384616e-06,
+      "loss": 0.0016,
+      "reward": 0.8049403864890337,
+      "reward_std": 0.40321367387659846,
+      "rewards/accuracy_reward": 0.09642857667058706,
+      "rewards/cosine_scaled_reward": -0.10696442169137299,
+      "rewards/format_reward": 0.7455357499420643,
+      "rewards/reasoning_steps_reward": 0.06994048110209405,
+      "step": 15
+    },
+    {
+      "completion_length": 233.43661785125732,
+      "epoch": 0.03092086191902599,
+      "grad_norm": 1.0224092602520027,
+      "kl": 0.0746673583984375,
+      "learning_rate": 6.153846153846155e-06,
+      "loss": 0.003,
+      "reward": 0.8915791682898998,
+      "reward_std": 0.3877835442312062,
+      "rewards/accuracy_reward": 0.10625000568106771,
+      "rewards/cosine_scaled_reward": -0.04800420138417394,
+      "rewards/format_reward": 0.7437500357627869,
+      "rewards/reasoning_steps_reward": 0.08958333982154727,
+      "step": 20
+    },
+    {
+      "completion_length": 212.74733037948607,
+      "epoch": 0.03865107739878249,
+      "grad_norm": 1.240709005575142,
+      "kl": 0.0882232666015625,
+      "learning_rate": 7.692307692307694e-06,
+      "loss": 0.0035,
+      "reward": 1.002081935852766,
+      "reward_std": 0.3706091455183923,
+      "rewards/accuracy_reward": 0.12589286351576448,
+      "rewards/cosine_scaled_reward": -0.004763351219298784,
+      "rewards/format_reward": 0.7625000335276126,
+      "rewards/reasoning_steps_reward": 0.11845238907262683,
+      "step": 25
+    },
+    {
+      "completion_length": 65.06071722507477,
+      "epoch": 0.04638129287853899,
+      "grad_norm": 1.981164938553927,
+      "kl": 0.3583251953125,
+      "learning_rate": 9.230769230769232e-06,
+      "loss": 0.0143,
+      "reward": 1.1063394784927367,
+      "reward_std": 0.1653201764522237,
+      "rewards/accuracy_reward": 0.057142860069870946,
+      "rewards/cosine_scaled_reward": 0.023006107861874625,
+      "rewards/format_reward": 0.9598214417695999,
+      "rewards/reasoning_steps_reward": 0.06636905297636986,
+      "step": 30
+    },
+    {
+      "completion_length": 68.25714633464813,
+      "epoch": 0.054111508358295486,
+      "grad_norm": 2.6378409932487905,
+      "kl": 0.6129638671875,
+      "learning_rate": 1.076923076923077e-05,
+      "loss": 0.0246,
+      "reward": 1.175253478437662,
+      "reward_std": 0.18643303209682927,
+      "rewards/accuracy_reward": 0.025000001210719346,
+      "rewards/cosine_scaled_reward": -0.025341810117242857,
+      "rewards/format_reward": 0.9607143014669418,
+      "rewards/reasoning_steps_reward": 0.214880966860801,
+      "step": 35
+    },
+    {
+      "completion_length": 57.43035991191864,
+      "epoch": 0.06184172383805198,
+      "grad_norm": 4.956506661704911,
+      "kl": 0.47823486328125,
+      "learning_rate": 1.230769230769231e-05,
+      "loss": 0.0191,
+      "reward": 1.2816980078816413,
+      "reward_std": 0.1065220304124523,
+      "rewards/accuracy_reward": 0.008035714644938708,
+      "rewards/cosine_scaled_reward": -0.02871871000970714,
+      "rewards/format_reward": 0.975000011920929,
+      "rewards/reasoning_steps_reward": 0.3273809777572751,
+      "step": 40
+    },
+    {
+      "completion_length": 61.206253266334535,
+      "epoch": 0.06957193931780849,
+      "grad_norm": 1.1434158564850665,
+      "kl": 0.69990234375,
+      "learning_rate": 1.3846153846153847e-05,
+      "loss": 0.028,
+      "reward": 1.4386275261640549,
+      "reward_std": 0.14483234668150544,
+      "rewards/accuracy_reward": 0.004464285913854837,
+      "rewards/cosine_scaled_reward": -0.028932074527256192,
+      "rewards/format_reward": 0.9660714432597161,
+      "rewards/reasoning_steps_reward": 0.4970238521695137,
+      "step": 45
+    },
+    {
+      "completion_length": 373.93662371635435,
+      "epoch": 0.07730215479756498,
+      "grad_norm": 0.7317925308360725,
+      "kl": 0.3951904296875,
+      "learning_rate": 1.5384615384615387e-05,
+      "loss": 0.0158,
+      "reward": 1.4827586993575097,
+      "reward_std": 0.3412967847660184,
+      "rewards/accuracy_reward": 0.0053571430966258046,
+      "rewards/cosine_scaled_reward": -0.17259852699935435,
+      "rewards/format_reward": 0.703571456298232,
+      "rewards/reasoning_steps_reward": 0.9464286357164383,
+      "step": 50
+    },
+    {
+      "completion_length": 95.80268287658691,
+      "epoch": 0.08503237027732148,
+      "grad_norm": 1.2231255930992933,
+      "kl": 0.584619140625,
+      "learning_rate": 1.6923076923076924e-05,
+      "loss": 0.0234,
+      "reward": 1.8429067522287368,
+      "reward_std": 0.13412819448858498,
+      "rewards/accuracy_reward": 0.008928571827709675,
+      "rewards/cosine_scaled_reward": -0.07792668255860917,
+      "rewards/format_reward": 0.9875000052154064,
+      "rewards/reasoning_steps_reward": 0.924404826760292,
+      "step": 55
+    },
+    {
+      "completion_length": 45.93393068313598,
+      "epoch": 0.09276258575707798,
+      "grad_norm": 1.2290196834793985,
+      "kl": 0.862744140625,
+      "learning_rate": 1.8461538461538465e-05,
+      "loss": 0.0345,
+      "reward": 1.8843340575695038,
+      "reward_std": 0.09844313520006835,
+      "rewards/accuracy_reward": 0.0026785715483129023,
+      "rewards/cosine_scaled_reward": -0.02995176060067024,
+      "rewards/format_reward": 1.0,
+      "rewards/reasoning_steps_reward": 0.9116072118282318,
+      "step": 60
+    },
+    {
+      "completion_length": 41.71875171661377,
+      "epoch": 0.10049280123683448,
+      "grad_norm": 4.86495762622666,
+      "kl": 1.1458984375,
+      "learning_rate": 2e-05,
+      "loss": 0.0458,
+      "reward": 1.9429006069898604,
+      "reward_std": 0.09017826845229138,
+      "rewards/accuracy_reward": 0.008035714644938708,
+      "rewards/cosine_scaled_reward": -0.015730434632860123,
+      "rewards/format_reward": 0.9919642895460129,
+      "rewards/reasoning_steps_reward": 0.9586309991776943,
+      "step": 65
+    },
+    {
+      "completion_length": 33.29643015861511,
+      "epoch": 0.10822301671659097,
+      "grad_norm": 1.1702023462933628,
+      "kl": 1.3630859375,
+      "learning_rate": 1.999634547413886e-05,
+      "loss": 0.0545,
+      "reward": 1.919108833372593,
+      "reward_std": 0.09037394287151983,
+      "rewards/accuracy_reward": 0.0008928571827709675,
+      "rewards/cosine_scaled_reward": -0.009165053069591523,
+      "rewards/format_reward": 0.9910714328289032,
+      "rewards/reasoning_steps_reward": 0.9363095819950104,
+      "step": 70
+    },
+    {
+      "completion_length": 28.31250133514404,
+      "epoch": 0.11595323219634747,
+      "grad_norm": 0.723298648050913,
+      "kl": 1.65380859375,
+      "learning_rate": 1.9985384567667278e-05,
+      "loss": 0.0662,
+      "reward": 1.971532365679741,
+      "reward_std": 0.03292575376344757,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.005551052745431662,
+      "rewards/format_reward": 0.9955357164144516,
+      "rewards/reasoning_steps_reward": 0.9815476641058922,
+      "step": 75
+    },
+    {
+      "completion_length": 34.47589445114136,
+      "epoch": 0.12368344767610397,
+      "grad_norm": 1.0683163847397983,
+      "kl": 2.5994140625,
+      "learning_rate": 1.9967125291968495e-05,
+      "loss": 0.104,
+      "reward": 1.9481785863637924,
+      "reward_std": 0.06038723666652004,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.008369124261662364,
+      "rewards/format_reward": 0.9767857253551483,
+      "rewards/reasoning_steps_reward": 0.9797619417309761,
+      "step": 80
+    },
+    {
+      "completion_length": 65.65803842544555,
+      "epoch": 0.13141366315586048,
+      "grad_norm": 3.8166154325669464,
+      "kl": 2.8294921875,
+      "learning_rate": 1.9941580992841562e-05,
+      "loss": 0.1132,
+      "reward": 1.883014965057373,
+      "reward_std": 0.1193199211884803,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.020556552152265796,
+      "rewards/format_reward": 0.9205357365310192,
+      "rewards/reasoning_steps_reward": 0.9830357439815998,
+      "step": 85
+    },
+    {
+      "completion_length": 106.13304002285004,
+      "epoch": 0.13914387863561697,
+      "grad_norm": 2.4782906565877263,
+      "kl": 2.51455078125,
+      "learning_rate": 1.990877034074683e-05,
+      "loss": 0.1006,
+      "reward": 1.8108871415257455,
+      "reward_std": 0.22878142609918087,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.04179151313728653,
+      "rewards/format_reward": 0.891964317113161,
+      "rewards/reasoning_steps_reward": 0.9607143364846706,
+      "step": 90
+    },
+    {
+      "completion_length": 24.216965198516846,
+      "epoch": 0.14687409411537347,
+      "grad_norm": 16.3288297356888,
+      "kl": 3.2685546875,
+      "learning_rate": 1.9868717317159617e-05,
+      "loss": 0.1308,
+      "reward": 1.846986098587513,
+      "reward_std": 0.10216094429056284,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.007180654056719504,
+      "rewards/format_reward": 0.8803571516647934,
+      "rewards/reasoning_steps_reward": 0.9738095529377461,
+      "step": 95
+    },
+    {
+      "completion_length": 19.997322106361388,
+      "epoch": 0.15460430959512997,
+      "grad_norm": 0.6680377475774779,
+      "kl": 3.801171875,
+      "learning_rate": 1.9821451197042028e-05,
+      "loss": 0.152,
+      "reward": 1.79707273542881,
+      "reward_std": 0.13152264674754405,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.007391588386963121,
+      "rewards/format_reward": 0.9080357346683741,
+      "rewards/reasoning_steps_reward": 0.89642863124609,
+      "step": 100
+    },
+    {
+      "epoch": 0.15460430959512997,
+      "eval_completion_length": 19.953274488449097,
+      "eval_kl": 3.958984375,
+      "eval_loss": 0.1587880253791809,
+      "eval_reward": 1.967043936252594,
+      "eval_reward_std": 0.04172060855364634,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.004682280734414235,
+      "eval_rewards/format_reward": 0.977678582072258,
+      "eval_rewards/reasoning_steps_reward": 0.9940476343035698,
+      "eval_runtime": 88.5436,
+      "eval_samples_per_second": 1.118,
+      "eval_steps_per_second": 0.045,
+      "step": 100
+    },
+    {
+      "completion_length": 36.03392972946167,
+      "epoch": 0.16233452507488647,
+      "grad_norm": 1.0457278656163418,
+      "kl": 4.2841796875,
+      "learning_rate": 1.9767006527445728e-05,
+      "loss": 0.1714,
+      "reward": 1.8444557696580888,
+      "reward_std": 0.17751287204523578,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.022806165312067604,
+      "rewards/format_reward": 0.8982143141329288,
+      "rewards/reasoning_steps_reward": 0.9690476536750794,
+      "step": 105
+    },
+    {
+      "completion_length": 19.386607813835145,
+      "epoch": 0.17006474055464296,
+      "grad_norm": 3.975769496074024,
+      "kl": 4.3798828125,
+      "learning_rate": 1.9705423102261324e-05,
+      "loss": 0.1752,
+      "reward": 1.848235307633877,
+      "reward_std": 0.1854919496602861,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003848067167564295,
+      "rewards/format_reward": 0.9071428917348385,
+      "rewards/reasoning_steps_reward": 0.9449405193328857,
+      "step": 110
+    },
+    {
+      "completion_length": 39.196430110931395,
+      "epoch": 0.17779495603439946,
+      "grad_norm": 629.8751115115074,
+      "kl": 1221448.98359375,
+      "learning_rate": 1.9636745933132807e-05,
+      "loss": 49017.025,
+      "reward": 1.7226364403963088,
+      "reward_std": 0.26729307810414865,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.015161267505027354,
+      "rewards/format_reward": 0.8187500331550837,
+      "rewards/reasoning_steps_reward": 0.919047673791647,
+      "step": 115
+    },
+    {
+      "completion_length": 69.21607434749603,
+      "epoch": 0.18552517151415596,
+      "grad_norm": 1.2688648377958267,
+      "kl": 4.220703125,
+      "learning_rate": 1.956102521655831e-05,
+      "loss": 0.1688,
+      "reward": 1.8117250353097916,
+      "reward_std": 0.23047553618447408,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02309649466187693,
+      "rewards/format_reward": 0.8973214633762836,
+      "rewards/reasoning_steps_reward": 0.9375000439584256,
+      "step": 120
+    },
+    {
+      "completion_length": 60.73125305175781,
+      "epoch": 0.19325538699391245,
+      "grad_norm": 0.2129319651640345,
+      "kl": 4.1267578125,
+      "learning_rate": 1.9478316297201218e-05,
+      "loss": 0.1651,
+      "reward": 1.9027385860681534,
+      "reward_std": 0.13217100637344856,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.021666298160562293,
+      "rewards/format_reward": 0.949107164144516,
+      "rewards/reasoning_steps_reward": 0.9752976492047309,
+      "step": 125
+    },
+    {
+      "completion_length": 82.8946475982666,
+      "epoch": 0.20098560247366895,
+      "grad_norm": 0.4842709355139345,
+      "kl": 4.0013671875,
+      "learning_rate": 1.9388679627438486e-05,
+      "loss": 0.1601,
+      "reward": 1.8676396518945695,
+      "reward_std": 0.16894543728906228,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.02968190241372213,
+      "rewards/format_reward": 0.9276786014437676,
+      "rewards/reasoning_steps_reward": 0.9696428924798965,
+      "step": 130
+    },
+    {
+      "completion_length": 56.571431851387025,
+      "epoch": 0.20871581795342545,
+      "grad_norm": 0.3307371522282908,
+      "kl": 4.093359375,
+      "learning_rate": 1.9292180723175656e-05,
+      "loss": 0.1637,
+      "reward": 1.9362090989947318,
+      "reward_std": 0.0856858525637108,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.007243422014289536,
+      "rewards/format_reward": 0.9598214469850064,
+      "rewards/reasoning_steps_reward": 0.9836309775710106,
+      "step": 135
+    },
+    {
+      "completion_length": 53.09821681976318,
+      "epoch": 0.21644603343318194,
+      "grad_norm": 0.20158892085634514,
+      "kl": 4.10859375,
+      "learning_rate": 1.9188890115960967e-05,
+      "loss": 0.1643,
+      "reward": 1.9483717486262322,
+      "reward_std": 0.05921500856607054,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004902200927608647,
+      "rewards/format_reward": 0.965178582072258,
+      "rewards/reasoning_steps_reward": 0.9880952581763267,
+      "step": 140
+    },
+    {
+      "completion_length": 64.77232451438904,
+      "epoch": 0.22417624891293844,
+      "grad_norm": 0.03644543056993346,
+      "kl": 4.0466796875,
+      "learning_rate": 1.9078883301433488e-05,
+      "loss": 0.1618,
+      "reward": 1.9349053993821144,
+      "reward_std": 0.07295309662978297,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004678068493376486,
+      "rewards/format_reward": 0.9535714440047741,
+      "rewards/reasoning_steps_reward": 0.9860119305551052,
+      "step": 145
+    },
+    {
+      "completion_length": 38.74285895824433,
+      "epoch": 0.23190646439269494,
+      "grad_norm": 0.0659210905213367,
+      "kl": 4.1435546875,
+      "learning_rate": 1.8962240684142923e-05,
+      "loss": 0.1657,
+      "reward": 1.9672578498721123,
+      "reward_std": 0.04177629348894243,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0032780089415609835,
+      "rewards/format_reward": 0.980357152223587,
+      "rewards/reasoning_steps_reward": 0.9901785835623741,
+      "step": 150
+    },
+    {
+      "completion_length": 66.56161048412324,
+      "epoch": 0.23963667987245144,
+      "grad_norm": 0.1437599738341925,
+      "kl": 3.9880859375,
+      "learning_rate": 1.883904751878156e-05,
+      "loss": 0.1595,
+      "reward": 1.9231001794338227,
+      "reward_std": 0.09157901574684502,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0033880500152008606,
+      "rewards/format_reward": 0.9526785887777806,
+      "rewards/reasoning_steps_reward": 0.9738095432519913,
+      "step": 155
+    },
+    {
+      "completion_length": 59.35446717739105,
+      "epoch": 0.24736689535220793,
+      "grad_norm": 0.09815925409063538,
+      "kl": 4.066796875,
+      "learning_rate": 1.8709393847871146e-05,
+      "loss": 0.1627,
+      "reward": 1.8962562099099158,
+      "reward_std": 0.0967012564579818,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0034463008109014482,
+      "rewards/format_reward": 0.924107164889574,
+      "rewards/reasoning_steps_reward": 0.975595261156559,
+      "step": 160
+    },
+    {
+      "completion_length": 67.4553611755371,
+      "epoch": 0.25509711083196446,
+      "grad_norm": 0.057252157927286634,
+      "kl": 4.0279296875,
+      "learning_rate": 1.857337443595034e-05,
+      "loss": 0.1611,
+      "reward": 1.9181514993309974,
+      "reward_std": 0.10700749011431299,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035748237452935426,
+      "rewards/format_reward": 0.9517857328057289,
+      "rewards/reasoning_steps_reward": 0.9699404999613762,
+      "step": 165
+    },
+    {
+      "completion_length": 91.7125051498413,
+      "epoch": 0.26282732631172095,
+      "grad_norm": 445.82999475155805,
+      "kl": 6.0328125,
+      "learning_rate": 1.8431088700310846e-05,
+      "loss": 0.2412,
+      "reward": 1.879951538145542,
+      "reward_std": 0.15761058254630597,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0039771679730620235,
+      "rewards/format_reward": 0.9258928872644901,
+      "rewards/reasoning_steps_reward": 0.9580357454717159,
+      "step": 170
+    },
+    {
+      "completion_length": 152.7116146326065,
+      "epoch": 0.27055754179147745,
+      "grad_norm": 0.025658109413280724,
+      "kl": 3.7509765625,
+      "learning_rate": 1.8282640638332773e-05,
+      "loss": 0.15,
+      "reward": 1.828264494240284,
+      "reward_std": 0.19012156879272196,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004176116726011969,
+      "rewards/format_reward": 0.8651785988360643,
+      "rewards/reasoning_steps_reward": 0.9672619320452214,
+      "step": 175
+    },
+    {
+      "completion_length": 181.5080426454544,
+      "epoch": 0.27828775727123395,
+      "grad_norm": 0.3807030611089835,
+      "kl": 3.526416015625,
+      "learning_rate": 1.8128138751472432e-05,
+      "loss": 0.141,
+      "reward": 1.7915376737713813,
+      "reward_std": 0.1342850628308952,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.005783874096232466,
+      "rewards/format_reward": 0.8160714489407838,
+      "rewards/reasoning_steps_reward": 0.9812500163912773,
+      "step": 180
+    },
+    {
+      "completion_length": 49.52053849697113,
+      "epoch": 0.28601797275099045,
+      "grad_norm": 0.051073057360027686,
+      "kl": 4.1109375,
+      "learning_rate": 1.7967695965958044e-05,
+      "loss": 0.1644,
+      "reward": 1.9462698236107827,
+      "reward_std": 0.06717885590007881,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003432691577472724,
+      "rewards/format_reward": 0.9696428716182709,
+      "rewards/reasoning_steps_reward": 0.9800595372915268,
+      "step": 185
+    },
+    {
+      "completion_length": 92.98125462532043,
+      "epoch": 0.29374818823074694,
+      "grad_norm": 0.08491282273318096,
+      "kl": 3.8732421875,
+      "learning_rate": 1.780142955025139e-05,
+      "loss": 0.1549,
+      "reward": 1.8725923761725425,
+      "reward_std": 0.14104052223704003,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004193474692874588,
+      "rewards/format_reward": 0.9258928783237934,
+      "rewards/reasoning_steps_reward": 0.9508928813040256,
+      "step": 190
+    },
+    {
+      "completion_length": 53.98839597702026,
+      "epoch": 0.30147840371050344,
+      "grad_norm": 0.0407502209798921,
+      "kl": 4.1095703125,
+      "learning_rate": 1.7629461029335683e-05,
+      "loss": 0.1644,
+      "reward": 1.9376063704490663,
+      "reward_std": 0.07948707955421241,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003465201653307304,
+      "rewards/format_reward": 0.9642857313156128,
+      "rewards/reasoning_steps_reward": 0.9767857275903225,
+      "step": 195
+    },
+    {
+      "completion_length": 76.24018263816833,
+      "epoch": 0.30920861919025994,
+      "grad_norm": 0.04711777773542715,
+      "kl": 4.0279296875,
+      "learning_rate": 1.745191609589231e-05,
+      "loss": 0.1612,
+      "reward": 1.896301306784153,
+      "reward_std": 0.1302720103650472,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0048892945284023884,
+      "rewards/format_reward": 0.9383928827941418,
+      "rewards/reasoning_steps_reward": 0.9627976477146148,
+      "step": 200
+    },
+    {
+      "epoch": 0.30920861919025994,
+      "eval_completion_length": 62.13958692550659,
+      "eval_kl": 3.982421875,
+      "eval_loss": 0.15937010943889618,
+      "eval_reward": 1.9047856628894806,
+      "eval_reward_std": 0.12965827994048595,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.007416868465952575,
+      "eval_rewards/format_reward": 0.9419643133878708,
+      "eval_rewards/reasoning_steps_reward": 0.9702381640672684,
+      "eval_runtime": 130.8866,
+      "eval_samples_per_second": 0.756,
+      "eval_steps_per_second": 0.031,
+      "step": 200
+    },
+    {
+      "completion_length": 30.83035817146301,
+      "epoch": 0.31693883467001643,
+      "grad_norm": 0.30874335671632164,
+      "kl": 4.1349609375,
+      "learning_rate": 1.7268924518431437e-05,
+      "loss": 0.1654,
+      "reward": 1.9143808513879776,
+      "reward_std": 0.1070815003528196,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.008238333073677495,
+      "rewards/format_reward": 0.9491071663796902,
+      "rewards/reasoning_steps_reward": 0.9735119335353375,
+      "step": 205
+    },
+    {
+      "completion_length": 33.57767968177795,
+      "epoch": 0.32466905014977293,
+      "grad_norm": 1.4283379016888456,
+      "kl": 4.1873046875,
+      "learning_rate": 1.7080620046443503e-05,
+      "loss": 0.1674,
+      "reward": 1.9376077562570573,
+      "reward_std": 0.08286752559746446,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.009118573818705044,
+      "rewards/format_reward": 0.9651785872876644,
+      "rewards/reasoning_steps_reward": 0.9815476432442665,
+      "step": 210
+    },
+    {
+      "completion_length": 84.65982589721679,
+      "epoch": 0.33239926562952943,
+      "grad_norm": 0.17126527752625184,
+      "kl": 3.955859375,
+      "learning_rate": 1.6887140312641036e-05,
+      "loss": 0.1582,
+      "reward": 1.8850552409887313,
+      "reward_std": 0.1453381786996033,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.021194879824179225,
+      "rewards/format_reward": 0.9214286006987095,
+      "rewards/reasoning_steps_reward": 0.9848214447498321,
+      "step": 215
+    },
+    {
+      "completion_length": 43.63214511871338,
+      "epoch": 0.3401294811092859,
+      "grad_norm": 0.17340162500849332,
+      "kl": 4.155078125,
+      "learning_rate": 1.6688626732362192e-05,
+      "loss": 0.1662,
+      "reward": 1.947451178729534,
+      "reward_std": 0.06978715559945385,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.015048965471214615,
+      "rewards/format_reward": 0.9732142984867096,
+      "rewards/reasoning_steps_reward": 0.9892857283353805,
+      "step": 220
+    },
+    {
+      "completion_length": 101.56250622272492,
+      "epoch": 0.3478596965890424,
+      "grad_norm": 3.0284434140101624,
+      "kl": 3.828515625,
+      "learning_rate": 1.6485224400209557e-05,
+      "loss": 0.1532,
+      "reward": 1.8568125352263452,
+      "reward_std": 0.15324103019194127,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.04348521199135576,
+      "rewards/format_reward": 0.9151785984635353,
+      "rewards/reasoning_steps_reward": 0.985119067877531,
+      "step": 225
+    },
+    {
+      "completion_length": 72.90893268585205,
+      "epoch": 0.3555899120687989,
+      "grad_norm": 0.1007758395372248,
+      "kl": 4.01953125,
+      "learning_rate": 1.6277081983999742e-05,
+      "loss": 0.1607,
+      "reward": 1.90594839155674,
+      "reward_std": 0.12096411035238361,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.029468410287518055,
+      "rewards/format_reward": 0.9455357372760773,
+      "rewards/reasoning_steps_reward": 0.9898809589445591,
+      "step": 230
+    },
+    {
+      "completion_length": 79.20536160469055,
+      "epoch": 0.3633201275485554,
+      "grad_norm": 1.5503652714803593,
+      "kl": 3.966015625,
+      "learning_rate": 1.6064351616101318e-05,
+      "loss": 0.1587,
+      "reward": 1.8908424541354178,
+      "reward_std": 0.1385169033126772,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.03266958048625383,
+      "rewards/format_reward": 0.9392857372760772,
+      "rewards/reasoning_steps_reward": 0.984226206690073,
+      "step": 235
+    },
+    {
+      "completion_length": 58.220539116859435,
+      "epoch": 0.3710503430283119,
+      "grad_norm": 0.6541256672880773,
+      "kl": 4.5427734375,
+      "learning_rate": 1.5847188782240473e-05,
+      "loss": 0.1817,
+      "reward": 1.938517163693905,
+      "reward_std": 0.07940695661046675,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.012971065321471542,
+      "rewards/format_reward": 0.958035732805729,
+      "rewards/reasoning_steps_reward": 0.9934523917734623,
+      "step": 240
+    },
+    {
+      "completion_length": 86.35982661247253,
+      "epoch": 0.3787805585080684,
+      "grad_norm": 0.1809364354222118,
+      "kl": 4.1990234375,
+      "learning_rate": 1.562575220785569e-05,
+      "loss": 0.168,
+      "reward": 1.9029038920998573,
+      "reward_std": 0.1221888413362194,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.016143867559731007,
+      "rewards/format_reward": 0.9330357424914837,
+      "rewards/reasoning_steps_reward": 0.9860119186341763,
+      "step": 245
+    },
+    {
+      "completion_length": 78.21786177158356,
+      "epoch": 0.3865107739878249,
+      "grad_norm": 0.3531746901376527,
+      "kl": 4.165234375,
+      "learning_rate": 1.5400203742084508e-05,
+      "loss": 0.1666,
+      "reward": 1.909020183980465,
+      "reward_std": 0.11898586064601205,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.006158514012349769,
+      "rewards/format_reward": 0.9392857395112515,
+      "rewards/reasoning_steps_reward": 0.9758928813040256,
+      "step": 250
+    },
+    {
+      "completion_length": 67.57500426769256,
+      "epoch": 0.3942409894675814,
+      "grad_norm": 0.8061477992341036,
+      "kl": 4.11875,
+      "learning_rate": 1.5170708239467143e-05,
+      "loss": 0.1647,
+      "reward": 1.9224899157881736,
+      "reward_std": 0.10508770456108323,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.009653077411348932,
+      "rewards/format_reward": 0.9473214529454708,
+      "rewards/reasoning_steps_reward": 0.9848214507102966,
+      "step": 255
+    },
+    {
+      "completion_length": 78.22857594490051,
+      "epoch": 0.4019712049473379,
+      "grad_norm": 4.411813587166956,
+      "kl": 4.2771484375,
+      "learning_rate": 1.4937433439453465e-05,
+      "loss": 0.1711,
+      "reward": 1.9104647740721703,
+      "reward_std": 0.1049793604788988,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.021975849731825293,
+      "rewards/format_reward": 0.9410714536905289,
+      "rewards/reasoning_steps_reward": 0.9913690604269505,
+      "step": 260
+    },
+    {
+      "completion_length": 69.1669682264328,
+      "epoch": 0.4097014204270944,
+      "grad_norm": 0.48352144768621647,
+      "kl": 4.066015625,
+      "learning_rate": 1.4700549843801359e-05,
+      "loss": 0.1627,
+      "reward": 1.9152694940567017,
+      "reward_std": 0.10864624861775382,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.022528255853103474,
+      "rewards/format_reward": 0.946428594738245,
+      "rewards/reasoning_steps_reward": 0.9913690611720085,
+      "step": 265
+    },
+    {
+      "completion_length": 37.12946565151215,
+      "epoch": 0.4174316359068509,
+      "grad_norm": 0.6714538930597738,
+      "kl": 4.706640625,
+      "learning_rate": 1.4460230591956097e-05,
+      "loss": 0.1883,
+      "reward": 1.9362671867012977,
+      "reward_std": 0.08168737509336096,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.015518663177499548,
+      "rewards/format_reward": 0.9785714387893677,
+      "rewards/reasoning_steps_reward": 0.9732143096625805,
+      "step": 270
+    },
+    {
+      "completion_length": 52.492859625816344,
+      "epoch": 0.4251618513866074,
+      "grad_norm": 0.9245446719505419,
+      "kl": 4.1919921875,
+      "learning_rate": 1.421665133450184e-05,
+      "loss": 0.1676,
+      "reward": 1.9015057906508446,
+      "reward_std": 0.13137552942173442,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.016946710253250785,
+      "rewards/format_reward": 0.946428594738245,
+      "rewards/reasoning_steps_reward": 0.9720238372683525,
+      "step": 275
+    },
+    {
+      "completion_length": 69.02589702606201,
+      "epoch": 0.4328920668663639,
+      "grad_norm": 0.11042447450412597,
+      "kl": 4.1,
+      "learning_rate": 1.3969990104777712e-05,
+      "loss": 0.164,
+      "reward": 1.9225183099508285,
+      "reward_std": 0.10247833729500826,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0063508788269246,
+      "rewards/format_reward": 0.9500000223517417,
+      "rewards/reasoning_steps_reward": 0.9788690708577633,
+      "step": 280
+    },
+    {
+      "completion_length": 71.41964704990387,
+      "epoch": 0.4406222823461204,
+      "grad_norm": 0.16895486430636023,
+      "kl": 4.07265625,
+      "learning_rate": 1.3720427188752306e-05,
+      "loss": 0.1629,
+      "reward": 1.9211889013648034,
+      "reward_std": 0.1069276062933909,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004406464856583625,
+      "rewards/format_reward": 0.9482143066823483,
+      "rewards/reasoning_steps_reward": 0.977380970865488,
+      "step": 285
+    },
+    {
+      "completion_length": 73.2026825428009,
+      "epoch": 0.4483524978258769,
+      "grad_norm": 0.23089713625880126,
+      "kl": 4.0236328125,
+      "learning_rate": 1.3468144993251735e-05,
+      "loss": 0.1609,
+      "reward": 1.9197445258498191,
+      "reward_std": 0.08582717737808707,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.006446088643861004,
+      "rewards/format_reward": 0.945535734295845,
+      "rewards/reasoning_steps_reward": 0.9806547850370407,
+      "step": 290
+    },
+    {
+      "completion_length": 54.97232456207276,
+      "epoch": 0.4560827133056334,
+      "grad_norm": 0.3314704351486173,
+      "kl": 4.11015625,
+      "learning_rate": 1.3213327912637563e-05,
+      "loss": 0.1644,
+      "reward": 1.9451483145356179,
+      "reward_std": 0.068802025315199,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004256584955146536,
+      "rewards/format_reward": 0.9669643014669418,
+      "rewards/reasoning_steps_reward": 0.9824404947459697,
+      "step": 295
+    },
+    {
+      "completion_length": 89.13750543594361,
+      "epoch": 0.4638129287853899,
+      "grad_norm": 0.056025297442606876,
+      "kl": 3.9078125,
+      "learning_rate": 1.295616219403197e-05,
+      "loss": 0.1563,
+      "reward": 1.8981666922569276,
+      "reward_std": 0.13097392180061435,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.008976291766157374,
+      "rewards/format_reward": 0.935714314877987,
+      "rewards/reasoning_steps_reward": 0.9714286036789417,
+      "step": 300
+    },
+    {
+      "epoch": 0.4638129287853899,
+      "eval_completion_length": 39.84375238418579,
+      "eval_kl": 4.07421875,
+      "eval_loss": 0.16346079111099243,
+      "eval_reward": 1.960254266858101,
+      "eval_reward_std": 0.051680851727724075,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.007007778622210026,
+      "eval_rewards/format_reward": 0.977678582072258,
+      "eval_rewards/reasoning_steps_reward": 0.9895833656191826,
+      "eval_runtime": 115.8419,
+      "eval_samples_per_second": 0.855,
+      "eval_steps_per_second": 0.035,
+      "step": 300
+    },
+    {
+      "completion_length": 76.06429018974305,
+      "epoch": 0.4715431442651464,
+      "grad_norm": 0.18209581705265776,
+      "kl": 3.9677734375,
+      "learning_rate": 1.2696835801188816e-05,
+      "loss": 0.1587,
+      "reward": 1.9158331274986267,
+      "reward_std": 0.10005336800240912,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.007678911578841508,
+      "rewards/format_reward": 0.9464285925030709,
+      "rewards/reasoning_steps_reward": 0.9770833596587181,
+      "step": 305
+    },
+    {
+      "completion_length": 67.11250400543213,
+      "epoch": 0.47927335974490287,
+      "grad_norm": 0.10256530306756913,
+      "kl": 4.0390625,
+      "learning_rate": 1.2435538277109919e-05,
+      "loss": 0.1616,
+      "reward": 1.9289911210536956,
+      "reward_std": 0.09589594877152194,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004639970138669014,
+      "rewards/format_reward": 0.9544643066823483,
+      "rewards/reasoning_steps_reward": 0.9791666835546493,
+      "step": 310
+    },
+    {
+      "completion_length": 74.25000467300416,
+      "epoch": 0.48700357522465937,
+      "grad_norm": 0.22469716748320895,
+      "kl": 4.0345703125,
+      "learning_rate": 1.2172460605507126e-05,
+      "loss": 0.1613,
+      "reward": 1.9088378116488456,
+      "reward_std": 0.12017987889821598,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003959942577057518,
+      "rewards/format_reward": 0.9455357387661933,
+      "rewards/reasoning_steps_reward": 0.9672619320452214,
+      "step": 315
+    },
+    {
+      "completion_length": 73.52054042816162,
+      "epoch": 0.49473379070441587,
+      "grad_norm": 0.055833624142444874,
+      "kl": 4.0427734375,
+      "learning_rate": 1.19077950712113e-05,
+      "loss": 0.1617,
+      "reward": 1.9088953778147697,
+      "reward_std": 0.11588089512188163,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0039023776946123688,
+      "rewards/format_reward": 0.9446428827941418,
+      "rewards/reasoning_steps_reward": 0.968154788017273,
+      "step": 320
+    },
+    {
+      "completion_length": 70.25625467300415,
+      "epoch": 0.5024640061841724,
+      "grad_norm": 0.052235556184819606,
+      "kl": 4.0248046875,
+      "learning_rate": 1.1641735119630373e-05,
+      "loss": 0.1609,
+      "reward": 1.9149396985769271,
+      "reward_std": 0.10732133641477049,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0038104403211036697,
+      "rewards/format_reward": 0.9482143104076386,
+      "rewards/reasoning_steps_reward": 0.9705357439815998,
+      "step": 325
+    },
+    {
+      "completion_length": 87.53929159641265,
+      "epoch": 0.5101942216639289,
+      "grad_norm": 0.06219274350103104,
+      "kl": 3.944921875,
+      "learning_rate": 1.137447521535908e-05,
+      "loss": 0.1578,
+      "reward": 1.885709960758686,
+      "reward_std": 0.15287861328179134,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.00417111199349165,
+      "rewards/format_reward": 0.9312500312924386,
+      "rewards/reasoning_steps_reward": 0.9586309857666493,
+      "step": 330
+    },
+    {
+      "completion_length": 63.86786105632782,
+      "epoch": 0.5179244371436854,
+      "grad_norm": 0.08929289186123827,
+      "kl": 4.05546875,
+      "learning_rate": 1.110621070004378e-05,
+      "loss": 0.1622,
+      "reward": 1.9251599743962289,
+      "reward_std": 0.10131168051884742,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0034115967486286537,
+      "rewards/format_reward": 0.955357163399458,
+      "rewards/reasoning_steps_reward": 0.973214303702116,
+      "step": 335
+    },
+    {
+      "completion_length": 59.61786041259766,
+      "epoch": 0.5256546526234419,
+      "grad_norm": 0.09420380111856261,
+      "kl": 4.08125,
+      "learning_rate": 1.0837137649606241e-05,
+      "loss": 0.1633,
+      "reward": 1.9315883651375771,
+      "reward_std": 0.0922205568291247,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0038284392998320983,
+      "rewards/format_reward": 0.9589285895228385,
+      "rewards/reasoning_steps_reward": 0.9764881134033203,
+      "step": 340
+    },
+    {
+      "completion_length": 80.26875534057618,
+      "epoch": 0.5333848681031984,
+      "grad_norm": 0.06973853747292577,
+      "kl": 3.965625,
+      "learning_rate": 1.0567452730930743e-05,
+      "loss": 0.1585,
+      "reward": 1.9035059094429017,
+      "reward_std": 0.12083175969310105,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0048275615088641645,
+      "rewards/format_reward": 0.9392857410013675,
+      "rewards/reasoning_steps_reward": 0.9690476469695568,
+      "step": 345
+    },
+    {
+      "completion_length": 86.70536255836487,
+      "epoch": 0.5411150835829549,
+      "grad_norm": 0.2670010274286229,
+      "kl": 14.13515625,
+      "learning_rate": 1.0297353058119209e-05,
+      "loss": 0.5658,
+      "reward": 1.8832309633493423,
+      "reward_std": 0.14878732510696863,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004269177743117325,
+      "rewards/format_reward": 0.9303571738302707,
+      "rewards/reasoning_steps_reward": 0.9571428880095482,
+      "step": 350
+    },
+    {
+      "completion_length": 66.56428985595703,
+      "epoch": 0.5488452990627114,
+      "grad_norm": 0.5662585781110105,
+      "kl": 4.0787109375,
+      "learning_rate": 1.0027036048419514e-05,
+      "loss": 0.1631,
+      "reward": 1.920632439851761,
+      "reward_std": 0.09926460331845419,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003772465680958703,
+      "rewards/format_reward": 0.9526785925030709,
+      "rewards/reasoning_steps_reward": 0.9717262141406536,
+      "step": 355
+    },
+    {
+      "completion_length": 105.22679147720336,
+      "epoch": 0.5565755145424679,
+      "grad_norm": 0.16159590077584135,
+      "kl": 3.834765625,
+      "learning_rate": 9.756699277932196e-06,
+      "loss": 0.1534,
+      "reward": 1.856449380517006,
+      "reward_std": 0.1816349770885836,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003669783708755858,
+      "rewards/format_reward": 0.9133928887546062,
+      "rewards/reasoning_steps_reward": 0.9467262342572212,
+      "step": 360
+    },
+    {
+      "completion_length": 61.28750357627869,
+      "epoch": 0.5643057300222244,
+      "grad_norm": 0.09838211954717037,
+      "kl": 4.0859375,
+      "learning_rate": 9.486540337201046e-06,
+      "loss": 0.1635,
+      "reward": 1.9208381652832032,
+      "reward_std": 0.10742362968286442,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003864350024377927,
+      "rewards/format_reward": 0.9535714499652386,
+      "rewards/reasoning_steps_reward": 0.9711309716105461,
+      "step": 365
+    },
+    {
+      "completion_length": 40.5357162475586,
+      "epoch": 0.5720359455019809,
+      "grad_norm": 0.045030859854867156,
+      "kl": 4.1826171875,
+      "learning_rate": 9.216756686793163e-06,
+      "loss": 0.1673,
+      "reward": 1.9622542053461074,
+      "reward_std": 0.04885252295061946,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035197534918552263,
+      "rewards/format_reward": 0.9785714387893677,
+      "rewards/reasoning_steps_reward": 0.9872023954987525,
+      "step": 370
+    },
+    {
+      "completion_length": 51.31071715354919,
+      "epoch": 0.5797661609817374,
+      "grad_norm": 0.05451944691861018,
+      "kl": 4.1115234375,
+      "learning_rate": 8.94754551297402e-06,
+      "loss": 0.1645,
+      "reward": 1.946009913086891,
+      "reward_std": 0.07182540869818013,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0033949891163501888,
+      "rewards/format_reward": 0.9678571581840515,
+      "rewards/reasoning_steps_reward": 0.9815476380288601,
+      "step": 375
+    },
+    {
+      "completion_length": 131.2000051498413,
+      "epoch": 0.5874963764614939,
+      "grad_norm": 6.7166440282552555,
+      "kl": 3.77578125,
+      "learning_rate": 8.67910358358298e-06,
+      "loss": 0.151,
+      "reward": 1.844117023050785,
+      "reward_std": 0.1535771109163761,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035021472489461304,
+      "rewards/format_reward": 0.8883928790688514,
+      "rewards/reasoning_steps_reward": 0.9592262215912342,
+      "step": 380
+    },
+    {
+      "completion_length": 70.14732570648194,
+      "epoch": 0.5952265919412504,
+      "grad_norm": 0.05964035031954431,
+      "kl": 4.0572265625,
+      "learning_rate": 8.411627104214675e-06,
+      "loss": 0.1623,
+      "reward": 1.9135051429271699,
+      "reward_std": 0.1051442735389287,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0034592857031384484,
+      "rewards/format_reward": 0.9491071656346322,
+      "rewards/reasoning_steps_reward": 0.9678571671247482,
+      "step": 385
+    },
+    {
+      "completion_length": 71.67768251895905,
+      "epoch": 0.6029568074210069,
+      "grad_norm": 0.07095713474628179,
+      "kl": 4.0458984375,
+      "learning_rate": 8.145311574811325e-06,
+      "loss": 0.1618,
+      "reward": 1.9104698538780212,
+      "reward_std": 0.11787946913391352,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0038160031515872108,
+      "rewards/format_reward": 0.9473214507102966,
+      "rewards/reasoning_steps_reward": 0.9669643051922321,
+      "step": 390
+    },
+    {
+      "completion_length": 59.12500314712524,
+      "epoch": 0.6106870229007634,
+      "grad_norm": 0.22475776321486607,
+      "kl": 4.0669921875,
+      "learning_rate": 7.880351646770824e-06,
+      "loss": 0.1626,
+      "reward": 1.9262871831655501,
+      "reward_std": 0.09550657907720961,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.00407009502814617,
+      "rewards/format_reward": 0.958035732805729,
+      "rewards/reasoning_steps_reward": 0.9723214544355869,
+      "step": 395
+    },
+    {
+      "completion_length": 70.24464654922485,
+      "epoch": 0.6184172383805199,
+      "grad_norm": 0.09092889716408575,
+      "kl": 4.0275390625,
+      "learning_rate": 7.616940980675004e-06,
+      "loss": 0.1611,
+      "reward": 1.909550815820694,
+      "reward_std": 0.11917392232409156,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0038421809673309325,
+      "rewards/format_reward": 0.9473214507102966,
+      "rewards/reasoning_steps_reward": 0.966071455180645,
+      "step": 400
+    },
+    {
+      "epoch": 0.6184172383805199,
+      "eval_completion_length": 85.97321724891663,
+      "eval_kl": 3.9296875,
+      "eval_loss": 0.15645448863506317,
+      "eval_reward": 1.8864707350730896,
+      "eval_reward_std": 0.15602644719183445,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.003410293662454933,
+      "eval_rewards/format_reward": 0.933035746216774,
+      "eval_rewards/reasoning_steps_reward": 0.9568452909588814,
+      "eval_runtime": 132.6383,
+      "eval_samples_per_second": 0.746,
+      "eval_steps_per_second": 0.03,
+      "step": 400
+    },
+    {
+      "completion_length": 78.62679014205932,
+      "epoch": 0.6261474538602764,
+      "grad_norm": 0.10311342926080141,
+      "kl": 3.9962890625,
+      "learning_rate": 7.355272104742132e-06,
+      "loss": 0.1599,
+      "reward": 1.8960644856095314,
+      "reward_std": 0.13990092346015218,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.00423326367745176,
+      "rewards/format_reward": 0.9383928827941418,
+      "rewards/reasoning_steps_reward": 0.9619047924876213,
+      "step": 405
+    },
+    {
+      "completion_length": 77.0848256111145,
+      "epoch": 0.6338776693400329,
+      "grad_norm": 0.12373506403704226,
+      "kl": 3.9802734375,
+      "learning_rate": 7.095536274107046e-06,
+      "loss": 0.1592,
+      "reward": 1.898416666686535,
+      "reward_std": 0.11678378849976526,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0048572868574410675,
+      "rewards/format_reward": 0.9410714522004128,
+      "rewards/reasoning_steps_reward": 0.9622024163603783,
+      "step": 410
+    },
+    {
+      "completion_length": 77.84375290870666,
+      "epoch": 0.6416078848197894,
+      "grad_norm": 0.09434877670325093,
+      "kl": 3.988671875,
+      "learning_rate": 6.837923331031761e-06,
+      "loss": 0.1595,
+      "reward": 1.9000333428382874,
+      "reward_std": 0.12504753311442302,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004431077223853208,
+      "rewards/format_reward": 0.9401785969734192,
+      "rewards/reasoning_steps_reward": 0.9642857477068901,
+      "step": 415
+    },
+    {
+      "completion_length": 84.20625410079955,
+      "epoch": 0.6493381002995459,
+      "grad_norm": 0.09331120868084523,
+      "kl": 3.9787109375,
+      "learning_rate": 6.58262156614881e-06,
+      "loss": 0.1592,
+      "reward": 1.8847931444644928,
+      "reward_std": 0.1432155600269084,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0056831808760762215,
+      "rewards/format_reward": 0.9321428827941418,
+      "rewards/reasoning_steps_reward": 0.9583333604037761,
+      "step": 420
+    },
+    {
+      "completion_length": 66.88036062717438,
+      "epoch": 0.6570683157793024,
+      "grad_norm": 0.19772149539193312,
+      "kl": 4.0763671875,
+      "learning_rate": 6.3298175808386284e-06,
+      "loss": 0.1631,
+      "reward": 1.9214427471160889,
+      "reward_std": 0.10656861763076222,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004152622053516098,
+      "rewards/format_reward": 0.9526785932481289,
+      "rewards/reasoning_steps_reward": 0.9729166924953461,
+      "step": 425
+    },
+    {
+      "completion_length": 71.4062525510788,
+      "epoch": 0.6647985312590589,
+      "grad_norm": 0.17489903890816574,
+      "kl": 4.015234375,
+      "learning_rate": 6.079696150841634e-06,
+      "loss": 0.1606,
+      "reward": 1.9176838472485542,
+      "reward_std": 0.10934554782797931,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0037448538321768867,
+      "rewards/format_reward": 0.9464285939931869,
+      "rewards/reasoning_steps_reward": 0.9750000186264515,
+      "step": 430
+    },
+    {
+      "completion_length": 82.87410979270935,
+      "epoch": 0.6725287467388154,
+      "grad_norm": 0.12536338202728264,
+      "kl": 3.985546875,
+      "learning_rate": 5.832440091204698e-06,
+      "loss": 0.1594,
+      "reward": 1.898548111319542,
+      "reward_std": 0.13894653706357757,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0041305988444946705,
+      "rewards/format_reward": 0.9339285999536514,
+      "rewards/reasoning_steps_reward": 0.9687500208616256,
+      "step": 435
+    },
+    {
+      "completion_length": 73.71339569091796,
+      "epoch": 0.6802589622185718,
+      "grad_norm": 0.0831511845118778,
+      "kl": 4.0279296875,
+      "learning_rate": 5.588230122660672e-06,
+      "loss": 0.1612,
+      "reward": 1.910518865287304,
+      "reward_std": 0.11439205608330667,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0034693721594521777,
+      "rewards/format_reward": 0.9446428798139095,
+      "rewards/reasoning_steps_reward": 0.9693452589213848,
+      "step": 440
+    },
+    {
+      "completion_length": 59.29910898208618,
+      "epoch": 0.6879891776983283,
+      "grad_norm": 0.06302346191293476,
+      "kl": 4.091796875,
+      "learning_rate": 5.347244739538677e-06,
+      "loss": 0.1637,
+      "reward": 1.9330023020505904,
+      "reward_std": 0.08597645508125425,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0039025974809192123,
+      "rewards/format_reward": 0.9589285895228385,
+      "rewards/reasoning_steps_reward": 0.977976205945015,
+      "step": 445
+    },
+    {
+      "completion_length": 56.962501764297485,
+      "epoch": 0.6957193931780848,
+      "grad_norm": 0.0790724324832407,
+      "kl": 4.0904296875,
+      "learning_rate": 5.109660079301668e-06,
+      "loss": 0.1636,
+      "reward": 1.9378462925553321,
+      "reward_std": 0.0833705105818808,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0038205120887141674,
+      "rewards/format_reward": 0.9616071589291095,
+      "rewards/reasoning_steps_reward": 0.980059539526701,
+      "step": 450
+    },
+    {
+      "completion_length": 115.70804004669189,
+      "epoch": 0.7034496086578413,
+      "grad_norm": 0.07587596082957317,
+      "kl": 3.97265625,
+      "learning_rate": 4.875649793806655e-06,
+      "loss": 0.1589,
+      "reward": 1.8709102511405944,
+      "reward_std": 0.1623626801001933,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.005280344319180586,
+      "rewards/format_reward": 0.9000000312924386,
+      "rewards/reasoning_steps_reward": 0.9761904969811439,
+      "step": 455
+    },
+    {
+      "completion_length": 57.54821660518646,
+      "epoch": 0.7111798241375978,
+      "grad_norm": 0.04214670683442245,
+      "kl": 4.1099609375,
+      "learning_rate": 4.64538492238166e-06,
+      "loss": 0.1644,
+      "reward": 1.9352339580655098,
+      "reward_std": 0.08453528358004406,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0034566513902973385,
+      "rewards/format_reward": 0.9598214477300644,
+      "rewards/reasoning_steps_reward": 0.9788690611720086,
+      "step": 460
+    },
+    {
+      "completion_length": 51.64375264644623,
+      "epoch": 0.7189100396173543,
+      "grad_norm": 0.055977738202792546,
+      "kl": 4.12734375,
+      "learning_rate": 4.4190337668121964e-06,
+      "loss": 0.165,
+      "reward": 1.9431429341435433,
+      "reward_std": 0.07587992586195469,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003881012083729729,
+      "rewards/format_reward": 0.9669643014669418,
+      "rewards/reasoning_steps_reward": 0.980059539526701,
+      "step": 465
+    },
+    {
+      "completion_length": 56.71964542865753,
+      "epoch": 0.7266402550971108,
+      "grad_norm": 0.1346185358347846,
+      "kl": 4.091015625,
+      "learning_rate": 4.196761768328599e-06,
+      "loss": 0.1636,
+      "reward": 1.9341410562396049,
+      "reward_std": 0.08432842884212732,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003954318125033751,
+      "rewards/format_reward": 0.9616071596741677,
+      "rewards/reasoning_steps_reward": 0.9764881141483783,
+      "step": 470
+    },
+    {
+      "completion_length": 57.53839483261108,
+      "epoch": 0.7343704705768673,
+      "grad_norm": 0.06837566720060198,
+      "kl": 4.121484375,
+      "learning_rate": 3.978731386684206e-06,
+      "loss": 0.1648,
+      "reward": 1.9330390319228172,
+      "reward_std": 0.08763236850500107,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.00356825259514153,
+      "rewards/format_reward": 0.9616071604192257,
+      "rewards/reasoning_steps_reward": 0.9750000216066838,
+      "step": 475
+    },
+    {
+      "completion_length": 58.821430277824405,
+      "epoch": 0.7421006860566238,
+      "grad_norm": 0.0870444764801619,
+      "kl": 4.094140625,
+      "learning_rate": 3.7651019814126656e-06,
+      "loss": 0.1639,
+      "reward": 1.9318299040198326,
+      "reward_std": 0.0834509583259205,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003884529904462397,
+      "rewards/format_reward": 0.960714303702116,
+      "rewards/reasoning_steps_reward": 0.9750000201165676,
+      "step": 480
+    },
+    {
+      "completion_length": 67.40535995960235,
+      "epoch": 0.7498309015363803,
+      "grad_norm": 0.07235089035047353,
+      "kl": 4.0345703125,
+      "learning_rate": 3.5560296953512296e-06,
+      "loss": 0.1614,
+      "reward": 1.9167783677577972,
+      "reward_std": 0.10895022895697366,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003459858492715284,
+      "rewards/format_reward": 0.951785734295845,
+      "rewards/reasoning_steps_reward": 0.9684524081647397,
+      "step": 485
+    },
+    {
+      "completion_length": 72.09464554786682,
+      "epoch": 0.7575611170161368,
+      "grad_norm": 0.1027155942227925,
+      "kl": 4.054296875,
+      "learning_rate": 3.3516673405151546e-06,
+      "loss": 0.1621,
+      "reward": 1.9102395072579383,
+      "reward_std": 0.11398139604827975,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0037487239431357013,
+      "rewards/format_reward": 0.9464285969734192,
+      "rewards/reasoning_steps_reward": 0.9675595439970494,
+      "step": 490
+    },
+    {
+      "completion_length": 79.98036060333251,
+      "epoch": 0.7652913324958933,
+      "grad_norm": 0.3055341411144215,
+      "kl": 3.969140625,
+      "learning_rate": 3.1521642864065905e-06,
+      "loss": 0.1588,
+      "reward": 1.8994512423872947,
+      "reward_std": 0.1334554604154846,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035250794346211476,
+      "rewards/format_reward": 0.9392857417464257,
+      "rewards/reasoning_steps_reward": 0.9636905036866665,
+      "step": 495
+    },
+    {
+      "completion_length": 79.49732532501221,
+      "epoch": 0.7730215479756498,
+      "grad_norm": 0.0481075530133631,
+      "kl": 3.970703125,
+      "learning_rate": 2.957666350839663e-06,
+      "loss": 0.1587,
+      "reward": 1.8963672295212746,
+      "reward_std": 0.1335994067238971,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003930515790125355,
+      "rewards/format_reward": 0.9383928842842579,
+      "rewards/reasoning_steps_reward": 0.961904788017273,
+      "step": 500
+    },
+    {
+      "epoch": 0.7730215479756498,
+      "eval_completion_length": 79.98661208152771,
+      "eval_kl": 3.966796875,
+      "eval_loss": 0.1587902307510376,
+      "eval_reward": 1.9129911959171295,
+      "eval_reward_std": 0.11852092316900098,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.0036756078188773245,
+      "eval_rewards/format_reward": 0.9464285969734192,
+      "eval_rewards/reasoning_steps_reward": 0.970238134264946,
+      "eval_runtime": 125.6388,
+      "eval_samples_per_second": 0.788,
+      "eval_steps_per_second": 0.032,
+      "step": 500
+    },
+    {
+      "completion_length": 87.8714322090149,
+      "epoch": 0.7807517634554063,
+      "grad_norm": 0.13866239123933727,
+      "kl": 3.9505859375,
+      "learning_rate": 2.768315693361474e-06,
+      "loss": 0.158,
+      "reward": 1.8883270829916001,
+      "reward_std": 0.15002182250412943,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003637332082143985,
+      "rewards/format_reward": 0.9321428842842578,
+      "rewards/reasoning_steps_reward": 0.9598214574158191,
+      "step": 505
+    },
+    {
+      "completion_length": 76.42232499122619,
+      "epoch": 0.7884819789351628,
+      "grad_norm": 0.06252134672865321,
+      "kl": 4.0162109375,
+      "learning_rate": 2.5842507113469307e-06,
+      "loss": 0.1606,
+      "reward": 1.9065988451242446,
+      "reward_std": 0.12157938196323811,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035203441220801324,
+      "rewards/format_reward": 0.942857164889574,
+      "rewards/reasoning_steps_reward": 0.9672619260847568,
+      "step": 510
+    },
+    {
+      "completion_length": 71.84464635849,
+      "epoch": 0.7962121944149193,
+      "grad_norm": 0.053547609323593474,
+      "kl": 4.02265625,
+      "learning_rate": 2.405605938843416e-06,
+      "loss": 0.1609,
+      "reward": 1.913142444193363,
+      "reward_std": 0.10145501654681084,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003524366064812057,
+      "rewards/format_reward": 0.9473214514553547,
+      "rewards/reasoning_steps_reward": 0.9693452551960945,
+      "step": 515
+    },
+    {
+      "completion_length": 72.07143197059631,
+      "epoch": 0.8039424098946758,
+      "grad_norm": 0.11014893043681045,
+      "kl": 4.02265625,
+      "learning_rate": 2.2325119482391466e-06,
+      "loss": 0.1609,
+      "reward": 1.912400482594967,
+      "reward_std": 0.11260003176839746,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0036710815300466495,
+      "rewards/format_reward": 0.9473214492201805,
+      "rewards/reasoning_steps_reward": 0.9687500201165676,
+      "step": 520
+    },
+    {
+      "completion_length": 71.91250350475312,
+      "epoch": 0.8116726253744323,
+      "grad_norm": 0.06342775337824043,
+      "kl": 4.0349609375,
+      "learning_rate": 2.065095254827133e-06,
+      "loss": 0.1614,
+      "reward": 1.9105623066425323,
+      "reward_std": 0.11352632529434459,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003425922698806971,
+      "rewards/format_reward": 0.946428594738245,
+      "rewards/reasoning_steps_reward": 0.9675595439970494,
+      "step": 525
+    },
+    {
+      "completion_length": 74.61964704990388,
+      "epoch": 0.8194028408541888,
+      "grad_norm": 0.052026170437193264,
+      "kl": 4.01875,
+      "learning_rate": 1.9034782243345074e-06,
+      "loss": 0.1607,
+      "reward": 1.9095401376485825,
+      "reward_std": 0.11663368406508426,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035552371642552315,
+      "rewards/format_reward": 0.9446428820490838,
+      "rewards/reasoning_steps_reward": 0.9684524036943912,
+      "step": 530
+    },
+    {
+      "completion_length": 77.84553978443145,
+      "epoch": 0.8271330563339453,
+      "grad_norm": 0.09858238225370353,
+      "kl": 4.009765625,
+      "learning_rate": 1.7477789834847835e-06,
+      "loss": 0.1604,
+      "reward": 1.902077753841877,
+      "reward_std": 0.12957435930147768,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003874768817331642,
+      "rewards/format_reward": 0.9419643104076385,
+      "rewards/reasoning_steps_reward": 0.9639881193637848,
+      "step": 535
+    },
+    {
+      "completion_length": 82.68750412464142,
+      "epoch": 0.8348632718137018,
+      "grad_norm": 0.065574476351722,
+      "kl": 3.98203125,
+      "learning_rate": 1.5981113336584041e-06,
+      "loss": 0.1593,
+      "reward": 1.89682334959507,
+      "reward_std": 0.13294254373256537,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003474415006348863,
+      "rewards/format_reward": 0.9375000275671482,
+      "rewards/reasoning_steps_reward": 0.9627976469695568,
+      "step": 540
+    },
+    {
+      "completion_length": 70.13036069869995,
+      "epoch": 0.8425934872934583,
+      "grad_norm": 0.1318813583738261,
+      "kl": 4.043359375,
+      "learning_rate": 1.4545846677147446e-06,
+      "loss": 0.1617,
+      "reward": 1.920035783946514,
+      "reward_std": 0.10434096483513713,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003476260631578043,
+      "rewards/format_reward": 0.9491071656346322,
+      "rewards/reasoning_steps_reward": 0.9744047850370408,
+      "step": 545
+    },
+    {
+      "completion_length": 60.432145619392394,
+      "epoch": 0.8503237027732148,
+      "grad_norm": 0.09318876295576914,
+      "kl": 4.0876953125,
+      "learning_rate": 1.3173038900362977e-06,
+      "loss": 0.1635,
+      "reward": 1.9312258705496788,
+      "reward_std": 0.08848651480149101,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035956974781583993,
+      "rewards/format_reward": 0.958035733550787,
+      "rewards/reasoning_steps_reward": 0.9767857387661933,
+      "step": 550
+    },
+    {
+      "completion_length": 70.33036000728607,
+      "epoch": 0.8580539182529713,
+      "grad_norm": 0.07392017210161962,
+      "kl": 4.029296875,
+      "learning_rate": 1.1863693398535115e-06,
+      "loss": 0.1611,
+      "reward": 1.9152584731578828,
+      "reward_std": 0.11109586956590647,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0037892865744652226,
+      "rewards/format_reward": 0.9482143081724643,
+      "rewards/reasoning_steps_reward": 0.9708333566784859,
+      "step": 555
+    },
+    {
+      "completion_length": 75.85178906917572,
+      "epoch": 0.8657841337327278,
+      "grad_norm": 0.055625451749199084,
+      "kl": 4.0359375,
+      "learning_rate": 1.0618767179063416e-06,
+      "loss": 0.1615,
+      "reward": 1.9078429341316223,
+      "reward_std": 0.11737079951482202,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004061957640806213,
+      "rewards/format_reward": 0.9428571663796902,
+      "rewards/reasoning_steps_reward": 0.9690476417541504,
+      "step": 560
+    },
+    {
+      "completion_length": 83.18482513427735,
+      "epoch": 0.8735143492124843,
+      "grad_norm": 0.0325554385194626,
+      "kl": 3.9732421875,
+      "learning_rate": 9.439170164960765e-07,
+      "loss": 0.1589,
+      "reward": 1.8949774771928787,
+      "reward_std": 0.13219705742287716,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0038321896485285834,
+      "rewards/format_reward": 0.9357143126428127,
+      "rewards/reasoning_steps_reward": 0.9630952678620815,
+      "step": 565
+    },
+    {
+      "completion_length": 74.5973245382309,
+      "epoch": 0.8812445646922408,
+      "grad_norm": 0.07216264151387287,
+      "kl": 3.9919921875,
+      "learning_rate": 8.325764529785851e-07,
+      "loss": 0.1596,
+      "reward": 1.9114135801792145,
+      "reward_std": 0.11653647294224356,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003467519613332115,
+      "rewards/format_reward": 0.9446428813040256,
+      "rewards/reasoning_steps_reward": 0.9702381134033203,
+      "step": 570
+    },
+    {
+      "completion_length": 80.90625429153442,
+      "epoch": 0.8889747801719973,
+      "grad_norm": 0.1218928141161543,
+      "kl": 4.0041015625,
+      "learning_rate": 7.279364067476247e-07,
+      "loss": 0.1601,
+      "reward": 1.9003287717700004,
+      "reward_std": 0.13390125300320505,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.00383802902942989,
+      "rewards/format_reward": 0.9375000275671482,
+      "rewards/reasoning_steps_reward": 0.9666666924953461,
+      "step": 575
+    },
+    {
+      "completion_length": 73.71339654922485,
+      "epoch": 0.8967049956517538,
+      "grad_norm": 0.0779054928956573,
+      "kl": 4.0125,
+      "learning_rate": 6.300733597542086e-07,
+      "loss": 0.1604,
+      "reward": 1.9101678639650346,
+      "reward_std": 0.11828550670562663,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003522748072282411,
+      "rewards/format_reward": 0.9455357365310192,
+      "rewards/reasoning_steps_reward": 0.9681547850370407,
+      "step": 580
+    },
+    {
+      "completion_length": 87.17946894168854,
+      "epoch": 0.9044352111315103,
+      "grad_norm": 0.28673289049569806,
+      "kl": 3.9888671875,
+      "learning_rate": 5.390588406055497e-07,
+      "loss": 0.1595,
+      "reward": 1.8926132425665856,
+      "reward_std": 0.14313040670480176,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035178393678506835,
+      "rewards/format_reward": 0.9330357432365417,
+      "rewards/reasoning_steps_reward": 0.9630952633917331,
+      "step": 585
+    },
+    {
+      "completion_length": 65.64018182754516,
+      "epoch": 0.9121654266112668,
+      "grad_norm": 0.09094243510356628,
+      "kl": 4.05390625,
+      "learning_rate": 4.549593722844492e-07,
+      "loss": 0.1621,
+      "reward": 1.9212124049663544,
+      "reward_std": 0.10267741186544299,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0034901109320344403,
+      "rewards/format_reward": 0.9535714477300644,
+      "rewards/reasoning_steps_reward": 0.9711309760808945,
+      "step": 590
+    },
+    {
+      "completion_length": 73.72411081790924,
+      "epoch": 0.9198956420910233,
+      "grad_norm": 0.042966362813253126,
+      "kl": 4.0140625,
+      "learning_rate": 3.77836423527278e-07,
+      "loss": 0.1606,
+      "reward": 1.9099439069628716,
+      "reward_std": 0.11524384366857703,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003449089830974117,
+      "rewards/format_reward": 0.9455357365310192,
+      "rewards/reasoning_steps_reward": 0.967857164144516,
+      "step": 595
+    },
+    {
+      "completion_length": 76.99911091327667,
+      "epoch": 0.9276258575707798,
+      "grad_norm": 0.10859950376158761,
+      "kl": 4.0298828125,
+      "learning_rate": 3.0774636389618196e-07,
+      "loss": 0.1611,
+      "reward": 1.9045991107821465,
+      "reward_std": 0.11265480399888475,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.004031982950982638,
+      "rewards/format_reward": 0.9410714529454708,
+      "rewards/reasoning_steps_reward": 0.9675595439970494,
+      "step": 600
+    },
+    {
+      "epoch": 0.9276258575707798,
+      "eval_completion_length": 54.86607503890991,
+      "eval_kl": 4.048828125,
+      "eval_loss": 0.1622004508972168,
+      "eval_reward": 1.9295674413442612,
+      "eval_reward_std": 0.09507856704294682,
+      "eval_rewards/accuracy_reward": 0.0,
+      "eval_rewards/cosine_scaled_reward": -0.0034684068523347378,
+      "eval_rewards/format_reward": 0.9598214477300644,
+      "eval_rewards/reasoning_steps_reward": 0.9732143208384514,
+      "eval_runtime": 125.081,
+      "eval_samples_per_second": 0.791,
+      "eval_steps_per_second": 0.032,
+      "step": 600
+    },
+    {
+      "completion_length": 70.12500309944153,
+      "epoch": 0.9353560730505363,
+      "grad_norm": 0.05179823112310112,
+      "kl": 4.0185546875,
+      "learning_rate": 2.44740422578269e-07,
+      "loss": 0.1607,
+      "reward": 1.9167007893323897,
+      "reward_std": 0.10983568504452705,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035374575527384877,
+      "rewards/format_reward": 0.9482143089175225,
+      "rewards/reasoning_steps_reward": 0.9720238253474236,
+      "step": 605
+    },
+    {
+      "completion_length": 71.05714664459228,
+      "epoch": 0.9430862885302927,
+      "grad_norm": 0.04825879971717623,
+      "kl": 4.0408203125,
+      "learning_rate": 1.8886465094192895e-07,
+      "loss": 0.1616,
+      "reward": 1.916950461268425,
+      "reward_std": 0.10439383089745888,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035853900481015444,
+      "rewards/format_reward": 0.9491071656346322,
+      "rewards/reasoning_steps_reward": 0.9714285925030708,
+      "step": 610
+    },
+    {
+      "completion_length": 73.90893230438232,
+      "epoch": 0.9508165040100492,
+      "grad_norm": 0.07733704565763884,
+      "kl": 4.0107421875,
+      "learning_rate": 1.401598888776523e-07,
+      "loss": 0.1604,
+      "reward": 1.9141234815120698,
+      "reward_std": 0.11271129585802556,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0037338059541070835,
+      "rewards/format_reward": 0.9455357372760773,
+      "rewards/reasoning_steps_reward": 0.9723214514553546,
+      "step": 615
+    },
+    {
+      "completion_length": 74.61875405311585,
+      "epoch": 0.9585467194898057,
+      "grad_norm": 0.09563400241916642,
+      "kl": 4.0271484375,
+      "learning_rate": 9.866173494794462e-08,
+      "loss": 0.1611,
+      "reward": 1.9092511892318726,
+      "reward_std": 0.12381009180098772,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035465647582896055,
+      "rewards/format_reward": 0.9446428820490838,
+      "rewards/reasoning_steps_reward": 0.9681547835469246,
+      "step": 620
+    },
+    {
+      "completion_length": 74.12857549190521,
+      "epoch": 0.9662769349695622,
+      "grad_norm": 0.06433206274792466,
+      "kl": 4.0037109375,
+      "learning_rate": 6.440052036815081e-08,
+      "loss": 0.1602,
+      "reward": 1.9127392619848251,
+      "reward_std": 0.11887687893087104,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003927528584608808,
+      "rewards/format_reward": 0.9473214522004128,
+      "rewards/reasoning_steps_reward": 0.9693452633917332,
+      "step": 625
+    },
+    {
+      "completion_length": 79.99643235206604,
+      "epoch": 0.9740071504493187,
+      "grad_norm": 0.031726665005581765,
+      "kl": 4.0005859375,
+      "learning_rate": 3.7401286837214224e-08,
+      "loss": 0.16,
+      "reward": 1.900482177734375,
+      "reward_std": 0.12850792148201434,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003684622357832268,
+      "rewards/format_reward": 0.9392857387661934,
+      "rewards/reasoning_steps_reward": 0.9648809768259525,
+      "step": 630
+    },
+    {
+      "completion_length": 79.1017897605896,
+      "epoch": 0.9817373659290752,
+      "grad_norm": 0.07055867261765238,
+      "kl": 4.007421875,
+      "learning_rate": 1.7683768234568745e-08,
+      "loss": 0.1603,
+      "reward": 1.9047423645853996,
+      "reward_std": 0.1259796370670415,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0035911083628889172,
+      "rewards/format_reward": 0.9401785977184772,
+      "rewards/reasoning_steps_reward": 0.9681547895073891,
+      "step": 635
+    },
+    {
+      "completion_length": 70.12857460975647,
+      "epoch": 0.9894675814088317,
+      "grad_norm": 0.03510443981029856,
+      "kl": 4.028515625,
+      "learning_rate": 5.262376196544239e-09,
+      "loss": 0.1611,
+      "reward": 1.9158729910850525,
+      "reward_std": 0.11106844576099775,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003472377223079093,
+      "rewards/format_reward": 0.949107164144516,
+      "rewards/reasoning_steps_reward": 0.9702381178736686,
+      "step": 640
+    },
+    {
+      "completion_length": 66.0607172012329,
+      "epoch": 0.9971977968885882,
+      "grad_norm": 0.1041554063271925,
+      "kl": 4.050390625,
+      "learning_rate": 1.461895828280824e-10,
+      "loss": 0.162,
+      "reward": 1.9202266588807106,
+      "reward_std": 0.10828843610361219,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.0038806214783107863,
+      "rewards/format_reward": 0.9526785917580127,
+      "rewards/reasoning_steps_reward": 0.9714285910129548,
+      "step": 645
+    },
+    {
+      "completion_length": 59.3750034570694,
+      "epoch": 0.9987438399845395,
+      "kl": 4.0791015625,
+      "reward": 1.9341124221682549,
+      "reward_std": 0.0886426616698941,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/cosine_scaled_reward": -0.003387685399502516,
+      "rewards/format_reward": 0.9598214477300644,
+      "rewards/reasoning_steps_reward": 0.977678582072258,
+      "step": 646,
+      "total_flos": 0.0,
+      "train_loss": 379.5353878517854,
+      "train_runtime": 232403.7931,
+      "train_samples_per_second": 0.312,
+      "train_steps_per_second": 0.003
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 646,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e81028790fb3d3f5d6f376245c71297fed6ab72454aa24b3f36c69f7bc9ce2d8
+size 7480