Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,67 @@
+---
+library_name: transformers
+model_name: DeepSeek-R1-Distill-Qwen-7B-S
+tags:
+- generated_from_trainer
+- grpo
+- trl
+licence: license
+---
+
+# Model Card for DeepSeek-R1-Distill-Qwen-7B-S
+
+This model is a fine-tuned version of [None](https://huggingface.co/None).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="leonMW/DeepSeek-R1-Distill-Qwen-7B-S", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/leonwenderoth-tu-darmstadt/huggingface/runs/b2dqumry) 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.23.0
+- Transformers: 4.56.2
+- Pytorch: 2.7.1
+- Datasets: 4.1.1
+- Tokenizers: 0.22.1
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{shao2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.0,
+    "train_runtime": 2.3434,
+    "train_samples": 3000,
+    "train_samples_per_second": 2560.359,
+    "train_steps_per_second": 78.518
+}

chat_template.jinja

@@ -0,0 +1 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}

config.json

@@ -0,0 +1,60 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151646,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.56.2",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151646,
+  "eos_token_id": 151643,
+  "pad_token_id": 151643,
+  "transformers_version": "4.56.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b8f752344974194fc8a1f24f8f542da9ac75a1f0a32fe28b0ba00e149768894
+size 4877660776

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a43d15eaf06e17594c86809c958511bd178ce798ee87563438e06329d2ef4367
+size 4932751008

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f3a66181f459450828bee809b24b88eb4c26b7583f897f75b2d040c72fa5478
+size 4330865200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab24b3572f03b03a507add4574f4c23e0fa67746c4e108310ec3c06e63f813e8
+size 1089994880

model.safetensors.index.json

@@ -0,0 +1,347 @@
+{
+  "metadata": {
+    "total_parameters": 333312,
+    "total_size": 15231233024
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4256422650d141f228fe954acee98679da412984c29a569877eefd3af69315a
+size 11422959

tokenizer_config.json

@@ -0,0 +1,198 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<｜User｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151645": {
+      "content": "<｜Assistant｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151646": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|EOT|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151648": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151649": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "max_length": null,
+  "model_max_length": 16384,
+  "pad_to_multiple_of": null,
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizerFast",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.0,
+    "train_runtime": 2.3434,
+    "train_samples": 3000,
+    "train_samples_per_second": 2560.359,
+    "train_steps_per_second": 78.518
+}

trainer_state.json

@@ -0,0 +1,2026 @@
+{
+  "best_global_step": 184,
+  "best_metric": 0.003358551999554038,
+  "best_model_checkpoint": "data/DeepSeek-R1-Distill-Qwen-7B-Staged-4/checkpoint-184",
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 184,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1531.0,
+      "completions/max_terminated_length": 1531.0,
+      "completions/mean_length": 423.75732421875,
+      "completions/mean_terminated_length": 423.75732421875,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
+      "entropy": 1.90119668841362,
+      "epoch": 0.010869565217391304,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11881979554891586,
+      "learning_rate": 1e-05,
+      "loss": 0.0039,
+      "num_tokens": 2917775.0,
+      "reward": 3.519592761993408,
+      "reward_std": 0.18993031978607178,
+      "rewards/ngram_repetition2/mean": 0.9466348886489868,
+      "rewards/ngram_repetition2/std": 0.01929597184062004,
+      "rewards/ngram_repetition3/mean": 0.9919997453689575,
+      "rewards/ngram_repetition3/std": 0.010940907523036003,
+      "rewards/symbolic_reward_accuracy/mean": 0.7822265625,
+      "rewards/symbolic_reward_accuracy/std": 0.4128333628177643,
+      "rewards/symbolic_reward_partial_score/mean": 0.9275716543197632,
+      "rewards/symbolic_reward_partial_score/std": 0.1604749709367752,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.02703022211790085,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9646764397621155,
+      "rewards/thinking_answer_ratio_reward/std": 0.055691447108983994,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.5215855836868286,
+      "sampling/importance_sampling_ratio/min": 0.0016820691525936127,
+      "sampling/sampling_logp_difference/max": 6.387730598449707,
+      "sampling/sampling_logp_difference/mean": 0.6701650023460388,
+      "step": 1
+    },
+    {
+      "clip_ratio/high_max": 0.11458333333333333,
+      "clip_ratio/high_mean": 0.037760416666666664,
+      "clip_ratio/low_mean": 0.359375,
+      "clip_ratio/low_min": 0.19270833333333334,
+      "clip_ratio/region_mean": 0.3971354166666667,
+      "entropy": 1.9287260274092357,
+      "epoch": 0.043478260869565216,
+      "grad_norm": 0.2112465351819992,
+      "learning_rate": 1e-05,
+      "loss": 0.0079,
+      "step": 4
+    },
+    {
+      "clip_ratio/high_max": 0.16015625,
+      "clip_ratio/high_mean": 0.0673828125,
+      "clip_ratio/low_mean": 0.27392578125,
+      "clip_ratio/low_min": 0.140625,
+      "clip_ratio/region_mean": 0.34130859375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 620.0,
+      "completions/max_terminated_length": 620.0,
+      "completions/mean_length": 426.1572265625,
+      "completions/mean_terminated_length": 426.1572265625,
+      "completions/min_length": 69.0,
+      "completions/min_terminated_length": 69.0,
+      "entropy": 1.884685069322586,
+      "epoch": 0.08695652173913043,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10315027832984924,
+      "learning_rate": 1e-05,
+      "loss": 0.003,
+      "num_tokens": 5881649.0,
+      "reward": 3.206584930419922,
+      "reward_std": 0.24610982835292816,
+      "rewards/ngram_repetition2/mean": 0.9437640309333801,
+      "rewards/ngram_repetition2/std": 0.01659868098795414,
+      "rewards/ngram_repetition3/mean": 0.9914915561676025,
+      "rewards/ngram_repetition3/std": 0.006500926800072193,
+      "rewards/symbolic_reward_accuracy/mean": 0.64599609375,
+      "rewards/symbolic_reward_accuracy/std": 0.4783271551132202,
+      "rewards/symbolic_reward_partial_score/mean": 0.8863118886947632,
+      "rewards/symbolic_reward_partial_score/std": 0.18328505754470825,
+      "rewards/tag_count_reward/mean": 0.999267578125,
+      "rewards/tag_count_reward/std": 0.019127286970615387,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9660772085189819,
+      "rewards/thinking_answer_ratio_reward/std": 0.0389963835477829,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.5227073431015015,
+      "sampling/importance_sampling_ratio/min": 0.0010220027761533856,
+      "sampling/sampling_logp_difference/max": 6.885991096496582,
+      "sampling/sampling_logp_difference/mean": 0.6709789037704468,
+      "step": 8
+    },
+    {
+      "clip_ratio/high_max": 0.2109375,
+      "clip_ratio/high_mean": 0.107421875,
+      "clip_ratio/low_mean": 0.26953125,
+      "clip_ratio/low_min": 0.16015625,
+      "clip_ratio/region_mean": 0.376953125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 724.0,
+      "completions/max_terminated_length": 724.0,
+      "completions/mean_length": 440.37939453125,
+      "completions/mean_terminated_length": 440.37939453125,
+      "completions/min_length": 215.0,
+      "completions/min_terminated_length": 215.0,
+      "entropy": 1.9614799618721008,
+      "epoch": 0.13043478260869565,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11278107017278671,
+      "learning_rate": 1e-05,
+      "loss": 0.0064,
+      "num_tokens": 8830298.0,
+      "reward": 3.415938138961792,
+      "reward_std": 0.14413174986839294,
+      "rewards/ngram_repetition2/mean": 0.9401968717575073,
+      "rewards/ngram_repetition2/std": 0.016658587381243706,
+      "rewards/ngram_repetition3/mean": 0.9910030364990234,
+      "rewards/ngram_repetition3/std": 0.006113104056566954,
+      "rewards/symbolic_reward_accuracy/mean": 0.7392578125,
+      "rewards/symbolic_reward_accuracy/std": 0.4391467273235321,
+      "rewards/symbolic_reward_partial_score/mean": 0.9086507558822632,
+      "rewards/symbolic_reward_partial_score/std": 0.18448227643966675,
+      "rewards/tag_count_reward/mean": 0.999755859375,
+      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "rewards/thinking_answer_ratio_reward/mean": 0.970394492149353,
+      "rewards/thinking_answer_ratio_reward/std": 0.023110121488571167,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.530914068222046,
+      "sampling/importance_sampling_ratio/min": 0.0017453260952606797,
+      "sampling/sampling_logp_difference/max": 6.350813865661621,
+      "sampling/sampling_logp_difference/mean": 0.6836334466934204,
+      "step": 12
+    },
+    {
+      "clip_ratio/high_max": 0.203125,
+      "clip_ratio/high_mean": 0.1064453125,
+      "clip_ratio/low_mean": 0.259765625,
+      "clip_ratio/low_min": 0.140625,
+      "clip_ratio/region_mean": 0.3662109375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 792.0,
+      "completions/max_terminated_length": 792.0,
+      "completions/mean_length": 433.44384765625,
+      "completions/mean_terminated_length": 433.44384765625,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
+      "entropy": 1.9702377393841743,
+      "epoch": 0.17391304347826086,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08772186189889908,
+      "learning_rate": 1e-05,
+      "loss": 0.0077,
+      "num_tokens": 11771079.0,
+      "reward": 3.4083731174468994,
+      "reward_std": 0.2164277881383896,
+      "rewards/ngram_repetition2/mean": 0.9409148693084717,
+      "rewards/ngram_repetition2/std": 0.016893817111849785,
+      "rewards/ngram_repetition3/mean": 0.9911813735961914,
+      "rewards/ngram_repetition3/std": 0.006109961774200201,
+      "rewards/symbolic_reward_accuracy/mean": 0.73974609375,
+      "rewards/symbolic_reward_accuracy/std": 0.43888023495674133,
+      "rewards/symbolic_reward_partial_score/mean": 0.902099609375,
+      "rewards/symbolic_reward_partial_score/std": 0.1949901431798935,
+      "rewards/tag_count_reward/mean": 0.997802734375,
+      "rewards/tag_count_reward/std": 0.033080797642469406,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9657655954360962,
+      "rewards/thinking_answer_ratio_reward/std": 0.06475918740034103,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.5316100120544434,
+      "sampling/importance_sampling_ratio/min": 0.001513556926511228,
+      "sampling/sampling_logp_difference/max": 6.493292808532715,
+      "sampling/sampling_logp_difference/mean": 0.6902951598167419,
+      "step": 16
+    },
+    {
+      "clip_ratio/high_max": 0.18359375,
+      "clip_ratio/high_mean": 0.07568359375,
+      "clip_ratio/low_mean": 0.27001953125,
+      "clip_ratio/low_min": 0.15234375,
+      "clip_ratio/region_mean": 0.345703125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 789.0,
+      "completions/max_terminated_length": 789.0,
+      "completions/mean_length": 463.08349609375,
+      "completions/mean_terminated_length": 463.08349609375,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
+      "entropy": 2.0984890162944794,
+      "epoch": 0.21739130434782608,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.29433581233024597,
+      "learning_rate": 1e-05,
+      "loss": 0.009,
+      "num_tokens": 14759890.0,
+      "reward": 3.492382526397705,
+      "reward_std": 0.12577903270721436,
+      "rewards/ngram_repetition2/mean": 0.9376952052116394,
+      "rewards/ngram_repetition2/std": 0.01754874922335148,
+      "rewards/ngram_repetition3/mean": 0.991070568561554,
+      "rewards/ngram_repetition3/std": 0.005940499249845743,
+      "rewards/symbolic_reward_accuracy/mean": 0.767578125,
+      "rewards/symbolic_reward_accuracy/std": 0.4224797189235687,
+      "rewards/symbolic_reward_partial_score/mean": 0.9296875,
+      "rewards/symbolic_reward_partial_score/std": 0.14929406344890594,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.02703022211790085,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9716020226478577,
+      "rewards/thinking_answer_ratio_reward/std": 0.052979789674282074,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.547365427017212,
+      "sampling/importance_sampling_ratio/min": 0.0007647222955711186,
+      "sampling/sampling_logp_difference/max": 7.175997734069824,
+      "sampling/sampling_logp_difference/mean": 0.7198488712310791,
+      "step": 20
+    },
+    {
+      "clip_ratio/high_max": 0.09375,
+      "clip_ratio/high_mean": 0.037109375,
+      "clip_ratio/low_mean": 0.31298828125,
+      "clip_ratio/low_min": 0.16015625,
+      "clip_ratio/region_mean": 0.35009765625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 830.0,
+      "completions/max_terminated_length": 830.0,
+      "completions/mean_length": 469.103515625,
+      "completions/mean_terminated_length": 469.103515625,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
+      "entropy": 2.1055676490068436,
+      "epoch": 0.2608695652173913,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20278385281562805,
+      "learning_rate": 1e-05,
+      "loss": 0.0095,
+      "num_tokens": 17748358.0,
+      "reward": 3.392237424850464,
+      "reward_std": 0.13710367679595947,
+      "rewards/ngram_repetition2/mean": 0.9388803243637085,
+      "rewards/ngram_repetition2/std": 0.016749296337366104,
+      "rewards/ngram_repetition3/mean": 0.9914165139198303,
+      "rewards/ngram_repetition3/std": 0.005852194037288427,
+      "rewards/symbolic_reward_accuracy/mean": 0.71728515625,
+      "rewards/symbolic_reward_accuracy/std": 0.45042893290519714,
+      "rewards/symbolic_reward_partial_score/mean": 0.9291177988052368,
+      "rewards/symbolic_reward_partial_score/std": 0.13959749042987823,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.015621182508766651,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9734582304954529,
+      "rewards/thinking_answer_ratio_reward/std": 0.030977273359894753,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.5528595447540283,
+      "sampling/importance_sampling_ratio/min": 0.0014880819944664836,
+      "sampling/sampling_logp_difference/max": 6.51026725769043,
+      "sampling/sampling_logp_difference/mean": 0.728722095489502,
+      "step": 24
+    },
+    {
+      "clip_ratio/high_max": 0.1640625,
+      "clip_ratio/high_mean": 0.08447265625,
+      "clip_ratio/low_mean": 0.27197265625,
+      "clip_ratio/low_min": 0.13671875,
+      "clip_ratio/region_mean": 0.3564453125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 791.0,
+      "completions/max_terminated_length": 791.0,
+      "completions/mean_length": 479.78955078125,
+      "completions/mean_terminated_length": 479.78955078125,
+      "completions/min_length": 83.0,
+      "completions/min_terminated_length": 83.0,
+      "entropy": 2.219125986099243,
+      "epoch": 0.30434782608695654,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.29682013392448425,
+      "learning_rate": 1e-05,
+      "loss": 0.0102,
+      "num_tokens": 20780887.0,
+      "reward": 3.550210475921631,
+      "reward_std": 0.06435245275497437,
+      "rewards/ngram_repetition2/mean": 0.9357079267501831,
+      "rewards/ngram_repetition2/std": 0.017338356003165245,
+      "rewards/ngram_repetition3/mean": 0.9910642504692078,
+      "rewards/ngram_repetition3/std": 0.005752938333898783,
+      "rewards/symbolic_reward_accuracy/mean": 0.787109375,
+      "rewards/symbolic_reward_accuracy/std": 0.4094509482383728,
+      "rewards/symbolic_reward_partial_score/mean": 0.9469807744026184,
+      "rewards/symbolic_reward_partial_score/std": 0.12396769970655441,
+      "rewards/tag_count_reward/mean": 1.0,
+      "rewards/tag_count_reward/std": 0.0,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9743147492408752,
+      "rewards/thinking_answer_ratio_reward/std": 0.00800173357129097,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.5690585374832153,
+      "sampling/importance_sampling_ratio/min": 0.00028703955467790365,
+      "sampling/sampling_logp_difference/max": 8.155890464782715,
+      "sampling/sampling_logp_difference/mean": 0.764785885810852,
+      "step": 28
+    },
+    {
+      "clip_ratio/high_max": 0.12109375,
+      "clip_ratio/high_mean": 0.04931640625,
+      "clip_ratio/low_mean": 0.2880859375,
+      "clip_ratio/low_min": 0.1640625,
+      "clip_ratio/region_mean": 0.33740234375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 807.0,
+      "completions/max_terminated_length": 807.0,
+      "completions/mean_length": 496.53515625,
+      "completions/mean_terminated_length": 496.53515625,
+      "completions/min_length": 81.0,
+      "completions/min_terminated_length": 81.0,
+      "entropy": 2.3269284814596176,
+      "epoch": 0.34782608695652173,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.42718204855918884,
+      "learning_rate": 1e-05,
+      "loss": 0.0099,
+      "num_tokens": 23854047.0,
+      "reward": 3.542815685272217,
+      "reward_std": 0.06283775717020035,
+      "rewards/ngram_repetition2/mean": 0.935409665107727,
+      "rewards/ngram_repetition2/std": 0.01653331145644188,
+      "rewards/ngram_repetition3/mean": 0.991692304611206,
+      "rewards/ngram_repetition3/std": 0.005664789583534002,
+      "rewards/symbolic_reward_accuracy/mean": 0.783203125,
+      "rewards/symbolic_reward_accuracy/std": 0.41216373443603516,
+      "rewards/symbolic_reward_partial_score/mean": 0.9476318359375,
+      "rewards/symbolic_reward_partial_score/std": 0.11877194046974182,
+      "rewards/tag_count_reward/mean": 0.999755859375,
+      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "rewards/thinking_answer_ratio_reward/mean": 0.975084662437439,
+      "rewards/thinking_answer_ratio_reward/std": 0.023241188377141953,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.5795289278030396,
+      "sampling/importance_sampling_ratio/min": 0.0014081046683713794,
+      "sampling/sampling_logp_difference/max": 6.5655107498168945,
+      "sampling/sampling_logp_difference/mean": 0.7878707051277161,
+      "step": 32
+    },
+    {
+      "clip_ratio/high_max": 0.05078125,
+      "clip_ratio/high_mean": 0.01904296875,
+      "clip_ratio/low_mean": 0.33984375,
+      "clip_ratio/low_min": 0.21484375,
+      "clip_ratio/region_mean": 0.35888671875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 792.0,
+      "completions/max_terminated_length": 792.0,
+      "completions/mean_length": 469.27099609375,
+      "completions/mean_terminated_length": 469.27099609375,
+      "completions/min_length": 94.0,
+      "completions/min_terminated_length": 94.0,
+      "entropy": 2.4070171415805817,
+      "epoch": 0.391304347826087,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4410569965839386,
+      "learning_rate": 1e-05,
+      "loss": 0.0054,
+      "num_tokens": 26880874.0,
+      "reward": 3.329148530960083,
+      "reward_std": 0.05563488230109215,
+      "rewards/ngram_repetition2/mean": 0.9407572746276855,
+      "rewards/ngram_repetition2/std": 0.015095624141395092,
+      "rewards/ngram_repetition3/mean": 0.9932428598403931,
+      "rewards/ngram_repetition3/std": 0.004793159198015928,
+      "rewards/symbolic_reward_accuracy/mean": 0.69189453125,
+      "rewards/symbolic_reward_accuracy/std": 0.46182316541671753,
+      "rewards/symbolic_reward_partial_score/mean": 0.9170328974723816,
+      "rewards/symbolic_reward_partial_score/std": 0.14379549026489258,
+      "rewards/tag_count_reward/mean": 0.999267578125,
+      "rewards/tag_count_reward/std": 0.019127286970615387,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9718977808952332,
+      "rewards/thinking_answer_ratio_reward/std": 0.03811168670654297,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.5838468074798584,
+      "sampling/importance_sampling_ratio/min": 0.00034505638177506626,
+      "sampling/sampling_logp_difference/max": 7.971802711486816,
+      "sampling/sampling_logp_difference/mean": 0.808716893196106,
+      "step": 36
+    },
+    {
+      "clip_ratio/high_max": 0.09765625,
+      "clip_ratio/high_mean": 0.0341796875,
+      "clip_ratio/low_mean": 0.31787109375,
+      "clip_ratio/low_min": 0.16796875,
+      "clip_ratio/region_mean": 0.35205078125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 737.0,
+      "completions/max_terminated_length": 737.0,
+      "completions/mean_length": 489.2646484375,
+      "completions/mean_terminated_length": 489.2646484375,
+      "completions/min_length": 42.0,
+      "completions/min_terminated_length": 42.0,
+      "entropy": 2.6946236938238144,
+      "epoch": 0.43478260869565216,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2574540674686432,
+      "learning_rate": 1e-05,
+      "loss": 0.0076,
+      "num_tokens": 29920136.0,
+      "reward": 3.6383514404296875,
+      "reward_std": 0.0791553258895874,
+      "rewards/ngram_repetition2/mean": 0.9311598539352417,
+      "rewards/ngram_repetition2/std": 0.01622355915606022,
+      "rewards/ngram_repetition3/mean": 0.9929859638214111,
+      "rewards/ngram_repetition3/std": 0.004797095898538828,
+      "rewards/symbolic_reward_accuracy/mean": 0.826171875,
+      "rewards/symbolic_reward_accuracy/std": 0.37905415892601013,
+      "rewards/symbolic_reward_partial_score/mean": 0.957275390625,
+      "rewards/symbolic_reward_partial_score/std": 0.10868100076913834,
+      "rewards/tag_count_reward/mean": 0.999755859375,
+      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9735076427459717,
+      "rewards/thinking_answer_ratio_reward/std": 0.024763144552707672,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.596329927444458,
+      "sampling/importance_sampling_ratio/min": 6.51177906547673e-05,
+      "sampling/sampling_logp_difference/max": 9.639312744140625,
+      "sampling/sampling_logp_difference/mean": 0.8470304012298584,
+      "step": 40
+    },
+    {
+      "clip_ratio/high_max": 0.046875,
+      "clip_ratio/high_mean": 0.01611328125,
+      "clip_ratio/low_mean": 0.3125,
+      "clip_ratio/low_min": 0.1953125,
+      "clip_ratio/region_mean": 0.32861328125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1433.0,
+      "completions/max_terminated_length": 1433.0,
+      "completions/mean_length": 503.10205078125,
+      "completions/mean_terminated_length": 503.10205078125,
+      "completions/min_length": 70.0,
+      "completions/min_terminated_length": 70.0,
+      "entropy": 2.872429236769676,
+      "epoch": 0.4782608695652174,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.493705153465271,
+      "learning_rate": 1e-05,
+      "loss": 0.0106,
+      "num_tokens": 33009913.0,
+      "reward": 3.4076955318450928,
+      "reward_std": 0.07953877747058868,
+      "rewards/ngram_repetition2/mean": 0.9200476408004761,
+      "rewards/ngram_repetition2/std": 0.02228526771068573,
+      "rewards/ngram_repetition3/mean": 0.9917500615119934,
+      "rewards/ngram_repetition3/std": 0.012813129462301731,
+      "rewards/symbolic_reward_accuracy/mean": 0.72802734375,
+      "rewards/symbolic_reward_accuracy/std": 0.4450845420360565,
+      "rewards/symbolic_reward_partial_score/mean": 0.9240315556526184,
+      "rewards/symbolic_reward_partial_score/std": 0.14966855943202972,
+      "rewards/tag_count_reward/mean": 0.998779296875,
+      "rewards/tag_count_reward/std": 0.02468114346265793,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9712032079696655,
+      "rewards/thinking_answer_ratio_reward/std": 0.0486665777862072,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6160120964050293,
+      "sampling/importance_sampling_ratio/min": 0.0009256865596398711,
+      "sampling/sampling_logp_difference/max": 6.9849748611450195,
+      "sampling/sampling_logp_difference/mean": 0.8949941396713257,
+      "step": 44
+    },
+    {
+      "clip_ratio/high_max": 0.0859375,
+      "clip_ratio/high_mean": 0.0400390625,
+      "clip_ratio/low_mean": 0.306640625,
+      "clip_ratio/low_min": 0.1875,
+      "clip_ratio/region_mean": 0.3466796875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 939.0,
+      "completions/max_terminated_length": 939.0,
+      "completions/mean_length": 571.37060546875,
+      "completions/mean_terminated_length": 571.37060546875,
+      "completions/min_length": 60.0,
+      "completions/min_terminated_length": 60.0,
+      "entropy": 2.9497868567705154,
+      "epoch": 0.5217391304347826,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6648740172386169,
+      "learning_rate": 1e-05,
+      "loss": 0.0113,
+      "num_tokens": 36239504.0,
+      "reward": 3.5769097805023193,
+      "reward_std": 0.07018020749092102,
+      "rewards/ngram_repetition2/mean": 0.914193868637085,
+      "rewards/ngram_repetition2/std": 0.017817478626966476,
+      "rewards/ngram_repetition3/mean": 0.9914629459381104,
+      "rewards/ngram_repetition3/std": 0.0048860348761081696,
+      "rewards/symbolic_reward_accuracy/mean": 0.80029296875,
+      "rewards/symbolic_reward_accuracy/std": 0.39987775683403015,
+      "rewards/symbolic_reward_partial_score/mean": 0.9482421875,
+      "rewards/symbolic_reward_partial_score/std": 0.1259223222732544,
+      "rewards/tag_count_reward/mean": 0.999267578125,
+      "rewards/tag_count_reward/std": 0.019127286970615387,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9757556915283203,
+      "rewards/thinking_answer_ratio_reward/std": 0.03801291063427925,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6147382259368896,
+      "sampling/importance_sampling_ratio/min": 0.0013765409821644425,
+      "sampling/sampling_logp_difference/max": 6.588181495666504,
+      "sampling/sampling_logp_difference/mean": 0.8927680850028992,
+      "step": 48
+    },
+    {
+      "clip_ratio/high_max": 0.1640625,
+      "clip_ratio/high_mean": 0.09228515625,
+      "clip_ratio/low_mean": 0.267578125,
+      "clip_ratio/low_min": 0.140625,
+      "clip_ratio/region_mean": 0.35986328125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1066.0,
+      "completions/max_terminated_length": 1066.0,
+      "completions/mean_length": 594.3876953125,
+      "completions/mean_terminated_length": 594.3876953125,
+      "completions/min_length": 80.0,
+      "completions/min_terminated_length": 80.0,
+      "entropy": 3.021150380373001,
+      "epoch": 0.5652173913043478,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3681764006614685,
+      "learning_rate": 1e-05,
+      "loss": 0.0141,
+      "num_tokens": 39532074.0,
+      "reward": 3.3776330947875977,
+      "reward_std": 0.10503610223531723,
+      "rewards/ngram_repetition2/mean": 0.908423900604248,
+      "rewards/ngram_repetition2/std": 0.02048499695956707,
+      "rewards/ngram_repetition3/mean": 0.9907451272010803,
+      "rewards/ngram_repetition3/std": 0.005756227765232325,
+      "rewards/symbolic_reward_accuracy/mean": 0.7138671875,
+      "rewards/symbolic_reward_accuracy/std": 0.45206260681152344,
+      "rewards/symbolic_reward_partial_score/mean": 0.921630859375,
+      "rewards/symbolic_reward_partial_score/std": 0.15090703964233398,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.015621182508766651,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9764349460601807,
+      "rewards/thinking_answer_ratio_reward/std": 0.03162452206015587,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6216703653335571,
+      "sampling/importance_sampling_ratio/min": 0.001497772173024714,
+      "sampling/sampling_logp_difference/max": 6.503776550292969,
+      "sampling/sampling_logp_difference/mean": 0.9096119999885559,
+      "step": 52
+    },
+    {
+      "clip_ratio/high_max": 0.125,
+      "clip_ratio/high_mean": 0.0537109375,
+      "clip_ratio/low_mean": 0.30859375,
+      "clip_ratio/low_min": 0.1953125,
+      "clip_ratio/region_mean": 0.3623046875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 908.0,
+      "completions/max_terminated_length": 908.0,
+      "completions/mean_length": 545.55126953125,
+      "completions/mean_terminated_length": 545.55126953125,
+      "completions/min_length": 53.0,
+      "completions/min_terminated_length": 53.0,
+      "entropy": 3.076643869280815,
+      "epoch": 0.6086956521739131,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5528954267501831,
+      "learning_rate": 1e-05,
+      "loss": 0.0124,
+      "num_tokens": 42708787.0,
+      "reward": 3.5160012245178223,
+      "reward_std": 0.06761978566646576,
+      "rewards/ngram_repetition2/mean": 0.919230043888092,
+      "rewards/ngram_repetition2/std": 0.019755244255065918,
+      "rewards/ngram_repetition3/mean": 0.9927445650100708,
+      "rewards/ngram_repetition3/std": 0.004674999509006739,
+      "rewards/symbolic_reward_accuracy/mean": 0.77490234375,
+      "rewards/symbolic_reward_accuracy/std": 0.4177486300468445,
+      "rewards/symbolic_reward_partial_score/mean": 0.9378255605697632,
+      "rewards/symbolic_reward_partial_score/std": 0.13589942455291748,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.015621182508766651,
+      "rewards/thinking_answer_ratio_reward/mean": 0.973955512046814,
+      "rewards/thinking_answer_ratio_reward/std": 0.03269569203257561,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6185786724090576,
+      "sampling/importance_sampling_ratio/min": 0.0006516074645332992,
+      "sampling/sampling_logp_difference/max": 7.336068153381348,
+      "sampling/sampling_logp_difference/mean": 0.9117289781570435,
+      "step": 56
+    },
+    {
+      "clip_ratio/high_max": 0.1015625,
+      "clip_ratio/high_mean": 0.0439453125,
+      "clip_ratio/low_mean": 0.2900390625,
+      "clip_ratio/low_min": 0.16015625,
+      "clip_ratio/region_mean": 0.333984375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 868.0,
+      "completions/max_terminated_length": 868.0,
+      "completions/mean_length": 484.09326171875,
+      "completions/mean_terminated_length": 484.09326171875,
+      "completions/min_length": 22.0,
+      "completions/min_terminated_length": 22.0,
+      "entropy": 3.069231167435646,
+      "epoch": 0.6521739130434783,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7982656359672546,
+      "learning_rate": 1e-05,
+      "loss": 0.0098,
+      "num_tokens": 45756466.0,
+      "reward": 3.442070960998535,
+      "reward_std": 0.07040801644325256,
+      "rewards/ngram_repetition2/mean": 0.9283775091171265,
+      "rewards/ngram_repetition2/std": 0.020430130884051323,
+      "rewards/ngram_repetition3/mean": 0.9933985471725464,
+      "rewards/ngram_repetition3/std": 0.004864003509283066,
+      "rewards/symbolic_reward_accuracy/mean": 0.73974609375,
+      "rewards/symbolic_reward_accuracy/std": 0.43888023495674133,
+      "rewards/symbolic_reward_partial_score/mean": 0.9348958134651184,
+      "rewards/symbolic_reward_partial_score/std": 0.13156187534332275,
+      "rewards/tag_count_reward/mean": 0.998779296875,
+      "rewards/tag_count_reward/std": 0.02468114346265793,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9685888290405273,
+      "rewards/thinking_answer_ratio_reward/std": 0.051747798919677734,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.621800422668457,
+      "sampling/importance_sampling_ratio/min": 0.000417891307733953,
+      "sampling/sampling_logp_difference/max": 7.780289173126221,
+      "sampling/sampling_logp_difference/mean": 0.9235653877258301,
+      "step": 60
+    },
+    {
+      "clip_ratio/high_max": 0.03515625,
+      "clip_ratio/high_mean": 0.00830078125,
+      "clip_ratio/low_mean": 0.3193359375,
+      "clip_ratio/low_min": 0.1953125,
+      "clip_ratio/region_mean": 0.32763671875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 813.0,
+      "completions/max_terminated_length": 813.0,
+      "completions/mean_length": 467.482421875,
+      "completions/mean_terminated_length": 467.482421875,
+      "completions/min_length": 80.0,
+      "completions/min_terminated_length": 80.0,
+      "entropy": 3.1883307099342346,
+      "epoch": 0.6956521739130435,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.9010258316993713,
+      "learning_rate": 1e-05,
+      "loss": 0.0079,
+      "num_tokens": 48776462.0,
+      "reward": 3.5352914333343506,
+      "reward_std": 0.11082939803600311,
+      "rewards/ngram_repetition2/mean": 0.9302128553390503,
+      "rewards/ngram_repetition2/std": 0.01884542964398861,
+      "rewards/ngram_repetition3/mean": 0.9941788911819458,
+      "rewards/ngram_repetition3/std": 0.0045182653702795506,
+      "rewards/symbolic_reward_accuracy/mean": 0.7822265625,
+      "rewards/symbolic_reward_accuracy/std": 0.4128333628177643,
+      "rewards/symbolic_reward_partial_score/mean": 0.9423828125,
+      "rewards/symbolic_reward_partial_score/std": 0.13051293790340424,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.015621182508766651,
+      "rewards/thinking_answer_ratio_reward/mean": 0.969980776309967,
+      "rewards/thinking_answer_ratio_reward/std": 0.03189215064048767,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.626150369644165,
+      "sampling/importance_sampling_ratio/min": 0.00047154395724646747,
+      "sampling/sampling_logp_difference/max": 7.65949821472168,
+      "sampling/sampling_logp_difference/mean": 0.9483247995376587,
+      "step": 64
+    },
+    {
+      "clip_ratio/high_max": 0.0234375,
+      "clip_ratio/high_mean": 0.00390625,
+      "clip_ratio/low_mean": 0.333984375,
+      "clip_ratio/low_min": 0.203125,
+      "clip_ratio/region_mean": 0.337890625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1044.0,
+      "completions/max_terminated_length": 1044.0,
+      "completions/mean_length": 568.92041015625,
+      "completions/mean_terminated_length": 568.92041015625,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 3.2027508467435837,
+      "epoch": 0.7391304347826086,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.4200383424758911,
+      "learning_rate": 1e-05,
+      "loss": 0.0152,
+      "num_tokens": 51997867.0,
+      "reward": 3.482755422592163,
+      "reward_std": 0.08074051141738892,
+      "rewards/ngram_repetition2/mean": 0.9018445611000061,
+      "rewards/ngram_repetition2/std": 0.020495234057307243,
+      "rewards/ngram_repetition3/mean": 0.9901250600814819,
+      "rewards/ngram_repetition3/std": 0.005463259294629097,
+      "rewards/symbolic_reward_accuracy/mean": 0.7578125,
+      "rewards/symbolic_reward_accuracy/std": 0.4285118281841278,
+      "rewards/symbolic_reward_partial_score/mean": 0.939697265625,
+      "rewards/symbolic_reward_partial_score/std": 0.12308314442634583,
+      "rewards/tag_count_reward/mean": 0.998779296875,
+      "rewards/tag_count_reward/std": 0.02468114346265793,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9734160304069519,
+      "rewards/thinking_answer_ratio_reward/std": 0.052874695509672165,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6354997158050537,
+      "sampling/importance_sampling_ratio/min": 0.000564142013899982,
+      "sampling/sampling_logp_difference/max": 7.4802045822143555,
+      "sampling/sampling_logp_difference/mean": 0.9833019971847534,
+      "step": 68
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.33203125,
+      "clip_ratio/low_min": 0.1953125,
+      "clip_ratio/region_mean": 0.33203125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1407.0,
+      "completions/max_terminated_length": 1407.0,
+      "completions/mean_length": 611.984375,
+      "completions/mean_terminated_length": 611.984375,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
+      "entropy": 3.311859115958214,
+      "epoch": 0.782608695652174,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.4716572761535645,
+      "learning_rate": 1e-05,
+      "loss": 0.0173,
+      "num_tokens": 55275787.0,
+      "reward": 3.511713981628418,
+      "reward_std": 0.18764734268188477,
+      "rewards/ngram_repetition2/mean": 0.8970193862915039,
+      "rewards/ngram_repetition2/std": 0.020160134881734848,
+      "rewards/ngram_repetition3/mean": 0.9902719259262085,
+      "rewards/ngram_repetition3/std": 0.005153110716491938,
+      "rewards/symbolic_reward_accuracy/mean": 0.7763671875,
+      "rewards/symbolic_reward_accuracy/std": 0.4167805016040802,
+      "rewards/symbolic_reward_partial_score/mean": 0.9328206181526184,
+      "rewards/symbolic_reward_partial_score/std": 0.15078683197498322,
+      "rewards/tag_count_reward/mean": 0.99755859375,
+      "rewards/tag_count_reward/std": 0.034861668944358826,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9727209806442261,
+      "rewards/thinking_answer_ratio_reward/std": 0.06795475631952286,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6320133209228516,
+      "sampling/importance_sampling_ratio/min": 0.0007096517365425825,
+      "sampling/sampling_logp_difference/max": 7.250736236572266,
+      "sampling/sampling_logp_difference/mean": 0.9720152616500854,
+      "step": 72
+    },
+    {
+      "clip_ratio/high_max": 0.00390625,
+      "clip_ratio/high_mean": 0.0009765625,
+      "clip_ratio/low_mean": 0.31982421875,
+      "clip_ratio/low_min": 0.17578125,
+      "clip_ratio/region_mean": 0.32080078125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1373.0,
+      "completions/max_terminated_length": 1373.0,
+      "completions/mean_length": 853.9833984375,
+      "completions/mean_terminated_length": 853.9833984375,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 3.3660423159599304,
+      "epoch": 0.8260869565217391,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.4343006610870361,
+      "learning_rate": 1e-05,
+      "loss": 0.0235,
+      "num_tokens": 59081001.0,
+      "reward": 3.422502279281616,
+      "reward_std": 0.1829456090927124,
+      "rewards/ngram_repetition2/mean": 0.8537134528160095,
+      "rewards/ngram_repetition2/std": 0.024440823122859,
+      "rewards/ngram_repetition3/mean": 0.9847572445869446,
+      "rewards/ngram_repetition3/std": 0.005931735038757324,
+      "rewards/symbolic_reward_accuracy/mean": 0.73828125,
+      "rewards/symbolic_reward_accuracy/std": 0.4396776556968689,
+      "rewards/symbolic_reward_partial_score/mean": 0.9226888418197632,
+      "rewards/symbolic_reward_partial_score/std": 0.16657814383506775,
+      "rewards/tag_count_reward/mean": 0.9951171875,
+      "rewards/tag_count_reward/std": 0.04918074235320091,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9749013185501099,
+      "rewards/thinking_answer_ratio_reward/std": 0.0969509705901146,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6466174125671387,
+      "sampling/importance_sampling_ratio/min": 0.00026180053828284144,
+      "sampling/sampling_logp_difference/max": 8.24792766571045,
+      "sampling/sampling_logp_difference/mean": 1.043502688407898,
+      "step": 76
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.33642578125,
+      "clip_ratio/low_min": 0.19921875,
+      "clip_ratio/region_mean": 0.33642578125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2030.0,
+      "completions/max_terminated_length": 2030.0,
+      "completions/mean_length": 812.24609375,
+      "completions/mean_terminated_length": 812.24609375,
+      "completions/min_length": 5.0,
+      "completions/min_terminated_length": 5.0,
+      "entropy": 3.547543302178383,
+      "epoch": 0.8695652173913043,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.1804628372192383,
+      "learning_rate": 1e-05,
+      "loss": 0.025,
+      "num_tokens": 62765889.0,
+      "reward": 3.5936460494995117,
+      "reward_std": 0.13216999173164368,
+      "rewards/ngram_repetition2/mean": 0.8755757808685303,
+      "rewards/ngram_repetition2/std": 0.022492358461022377,
+      "rewards/ngram_repetition3/mean": 0.9866626262664795,
+      "rewards/ngram_repetition3/std": 0.005942893214523792,
+      "rewards/symbolic_reward_accuracy/mean": 0.8115234375,
+      "rewards/symbolic_reward_accuracy/std": 0.39118775725364685,
+      "rewards/symbolic_reward_partial_score/mean": 0.9463704228401184,
+      "rewards/symbolic_reward_partial_score/std": 0.1394457072019577,
+      "rewards/tag_count_reward/mean": 0.995849609375,
+      "rewards/tag_count_reward/std": 0.04537592828273773,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9756726026535034,
+      "rewards/thinking_answer_ratio_reward/std": 0.08393006771802902,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6469275951385498,
+      "sampling/importance_sampling_ratio/min": 0.0009282914106734097,
+      "sampling/sampling_logp_difference/max": 6.9821648597717285,
+      "sampling/sampling_logp_difference/mean": 1.0689876079559326,
+      "step": 80
+    },
+    {
+      "clip_ratio/high_max": 0.00390625,
+      "clip_ratio/high_mean": 0.00048828125,
+      "clip_ratio/low_mean": 0.33740234375,
+      "clip_ratio/low_min": 0.19921875,
+      "clip_ratio/region_mean": 0.337890625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1538.0,
+      "completions/max_terminated_length": 1538.0,
+      "completions/mean_length": 874.0634765625,
+      "completions/mean_terminated_length": 874.0634765625,
+      "completions/min_length": 6.0,
+      "completions/min_terminated_length": 6.0,
+      "entropy": 3.464124232530594,
+      "epoch": 0.9130434782608695,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.2991092205047607,
+      "learning_rate": 1e-05,
+      "loss": 0.0286,
+      "num_tokens": 66609059.0,
+      "reward": 3.4975390434265137,
+      "reward_std": 0.14730925858020782,
+      "rewards/ngram_repetition2/mean": 0.8646764755249023,
+      "rewards/ngram_repetition2/std": 0.02347402088344097,
+      "rewards/ngram_repetition3/mean": 0.9867684841156006,
+      "rewards/ngram_repetition3/std": 0.005439760163426399,
+      "rewards/symbolic_reward_accuracy/mean": 0.771484375,
+      "rewards/symbolic_reward_accuracy/std": 0.4199790060520172,
+      "rewards/symbolic_reward_partial_score/mean": 0.92919921875,
+      "rewards/symbolic_reward_partial_score/std": 0.1563497930765152,
+      "rewards/tag_count_reward/mean": 0.9970703125,
+      "rewards/tag_count_reward/std": 0.03817030414938927,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9786182641983032,
+      "rewards/thinking_answer_ratio_reward/std": 0.07284367829561234,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6411030292510986,
+      "sampling/importance_sampling_ratio/min": 0.002884730463847518,
+      "sampling/sampling_logp_difference/max": 5.848323822021484,
+      "sampling/sampling_logp_difference/mean": 1.0739994049072266,
+      "step": 84
+    },
+    {
+      "clip_ratio/high_max": 0.015625,
+      "clip_ratio/high_mean": 0.0029296875,
+      "clip_ratio/low_mean": 0.32421875,
+      "clip_ratio/low_min": 0.1875,
+      "clip_ratio/region_mean": 0.3271484375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1897.0,
+      "completions/max_terminated_length": 1897.0,
+      "completions/mean_length": 795.52587890625,
+      "completions/mean_terminated_length": 795.52587890625,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "entropy": 3.7464267015457153,
+      "epoch": 0.9565217391304348,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.355712652206421,
+      "learning_rate": 1e-05,
+      "loss": 0.0302,
+      "num_tokens": 70304056.0,
+      "reward": 3.435868501663208,
+      "reward_std": 0.1441032588481903,
+      "rewards/ngram_repetition2/mean": 0.8944746255874634,
+      "rewards/ngram_repetition2/std": 0.022508157417178154,
+      "rewards/ngram_repetition3/mean": 0.9901339411735535,
+      "rewards/ngram_repetition3/std": 0.005564424674957991,
+      "rewards/symbolic_reward_accuracy/mean": 0.74072265625,
+      "rewards/symbolic_reward_accuracy/std": 0.4383451044559479,
+      "rewards/symbolic_reward_partial_score/mean": 0.9297281503677368,
+      "rewards/symbolic_reward_partial_score/std": 0.1441221833229065,
+      "rewards/tag_count_reward/mean": 0.99609375,
+      "rewards/tag_count_reward/std": 0.04403195157647133,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9755286574363708,
+      "rewards/thinking_answer_ratio_reward/std": 0.08032744377851486,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6346502304077148,
+      "sampling/importance_sampling_ratio/min": 5.58673445993918e-06,
+      "sampling/sampling_logp_difference/max": 12.095115661621094,
+      "sampling/sampling_logp_difference/mean": 1.0778782367706299,
+      "step": 88
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.3232421875,
+      "clip_ratio/low_min": 0.20703125,
+      "clip_ratio/region_mean": 0.3232421875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1791.0,
+      "completions/max_terminated_length": 1791.0,
+      "completions/mean_length": 827.90234375,
+      "completions/mean_terminated_length": 827.90234375,
+      "completions/min_length": 6.0,
+      "completions/min_terminated_length": 6.0,
+      "entropy": 3.5320966094732285,
+      "epoch": 1.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.9927502870559692,
+      "learning_rate": 1e-05,
+      "loss": 0.0361,
+      "num_tokens": 74071696.0,
+      "reward": 3.5067601203918457,
+      "reward_std": 0.13380564749240875,
+      "rewards/ngram_repetition2/mean": 0.8853453397750854,
+      "rewards/ngram_repetition2/std": 0.022779002785682678,
+      "rewards/ngram_repetition3/mean": 0.9901328682899475,
+      "rewards/ngram_repetition3/std": 0.005021216347813606,
+      "rewards/symbolic_reward_accuracy/mean": 0.7734375,
+      "rewards/symbolic_reward_accuracy/std": 0.4187093675136566,
+      "rewards/symbolic_reward_partial_score/mean": 0.9352620840072632,
+      "rewards/symbolic_reward_partial_score/std": 0.1416569948196411,
+      "rewards/tag_count_reward/mean": 0.99609375,
+      "rewards/tag_count_reward/std": 0.04403195157647133,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9774473905563354,
+      "rewards/thinking_answer_ratio_reward/std": 0.06120087578892708,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6360163688659668,
+      "sampling/importance_sampling_ratio/min": 1.9057076769968262e-06,
+      "sampling/sampling_logp_difference/max": 13.17065715789795,
+      "sampling/sampling_logp_difference/mean": 1.0849487781524658,
+      "step": 92
+    },
+    {
+      "epoch": 1.0,
+      "eval_clip_ratio/high_max": 0.0,
+      "eval_clip_ratio/high_mean": 0.0,
+      "eval_clip_ratio/low_mean": 0.0,
+      "eval_clip_ratio/low_min": 0.0,
+      "eval_clip_ratio/region_mean": 0.0,
+      "eval_completions/clipped_ratio": 0.0,
+      "eval_completions/max_length": 1317.4736842105262,
+      "eval_completions/max_terminated_length": 1317.4736842105262,
+      "eval_completions/mean_length": 848.6509046052631,
+      "eval_completions/mean_terminated_length": 848.6509046052631,
+      "eval_completions/min_length": 408.2105263157895,
+      "eval_completions/min_terminated_length": 408.2105263157895,
+      "eval_entropy": 3.616316644768966,
+      "eval_frac_reward_zero_std": 0.0,
+      "eval_loss": 0.008092080242931843,
+      "eval_num_tokens": 74071696.0,
+      "eval_reward": 3.544994504828202,
+      "eval_reward_std": 0.13573118780914228,
+      "eval_rewards/ngram_repetition2/mean": 0.8825263663342124,
+      "eval_rewards/ngram_repetition2/std": 0.021967738376636254,
+      "eval_rewards/ngram_repetition3/mean": 0.9900523204552499,
+      "eval_rewards/ngram_repetition3/std": 0.00497045870380182,
+      "eval_rewards/symbolic_reward_accuracy/mean": 0.7894736842105263,
+      "eval_rewards/symbolic_reward_accuracy/std": 0.36093331324426753,
+      "eval_rewards/symbolic_reward_partial_score/mean": 0.9397957826915541,
+      "eval_rewards/symbolic_reward_partial_score/std": 0.12393280491232872,
+      "eval_rewards/tag_count_reward/mean": 0.9977384868421053,
+      "eval_rewards/tag_count_reward/std": 0.022835058405211096,
+      "eval_rewards/thinking_answer_ratio_reward/mean": 0.9787100898592096,
+      "eval_rewards/thinking_answer_ratio_reward/std": 0.04720702974468862,
+      "eval_runtime": 701.8881,
+      "eval_samples_per_second": 0.214,
+      "eval_sampling/importance_sampling_ratio/max": 2.0,
+      "eval_sampling/importance_sampling_ratio/mean": 1.632221924631219,
+      "eval_sampling/importance_sampling_ratio/min": 0.004045612331920941,
+      "eval_sampling/sampling_logp_difference/max": 5.546152215254934,
+      "eval_sampling/sampling_logp_difference/mean": 1.0775353092896311,
+      "eval_steps_per_second": 0.003,
+      "step": 92
+    },
+    {
+      "clip_ratio/high_max": 0.0703125,
+      "clip_ratio/high_mean": 0.02294921875,
+      "clip_ratio/low_mean": 0.35302734375,
+      "clip_ratio/low_min": 0.21875,
+      "clip_ratio/region_mean": 0.3759765625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1500.0,
+      "completions/max_terminated_length": 1500.0,
+      "completions/mean_length": 857.35693359375,
+      "completions/mean_terminated_length": 857.35693359375,
+      "completions/min_length": 216.0,
+      "completions/min_terminated_length": 216.0,
+      "entropy": 3.729607880115509,
+      "epoch": 1.0434782608695652,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.33806663751602173,
+      "learning_rate": 1e-05,
+      "loss": 0.0364,
+      "num_tokens": 77877483.0,
+      "reward": 3.549053192138672,
+      "reward_std": 0.09654223173856735,
+      "rewards/ngram_repetition2/mean": 0.8818784952163696,
+      "rewards/ngram_repetition2/std": 0.022474652156233788,
+      "rewards/ngram_repetition3/mean": 0.9899985790252686,
+      "rewards/ngram_repetition3/std": 0.005039301700890064,
+      "rewards/symbolic_reward_accuracy/mean": 0.78662109375,
+      "rewards/symbolic_reward_accuracy/std": 0.40979304909706116,
+      "rewards/symbolic_reward_partial_score/mean": 0.94775390625,
+      "rewards/symbolic_reward_partial_score/std": 0.11653388291597366,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.015621182508766651,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9826548099517822,
+      "rewards/thinking_answer_ratio_reward/std": 0.024480855092406273,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6328657865524292,
+      "sampling/importance_sampling_ratio/min": 0.0006916878628544509,
+      "sampling/sampling_logp_difference/max": 7.276375770568848,
+      "sampling/sampling_logp_difference/mean": 1.0774481296539307,
+      "step": 96
+    },
+    {
+      "clip_ratio/high_max": 0.00390625,
+      "clip_ratio/high_mean": 0.00048828125,
+      "clip_ratio/low_mean": 0.3291015625,
+      "clip_ratio/low_min": 0.20703125,
+      "clip_ratio/region_mean": 0.32958984375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1438.0,
+      "completions/max_terminated_length": 1438.0,
+      "completions/mean_length": 829.27978515625,
+      "completions/mean_terminated_length": 829.27978515625,
+      "completions/min_length": 286.0,
+      "completions/min_terminated_length": 286.0,
+      "entropy": 3.5110934525728226,
+      "epoch": 1.0869565217391304,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.883220672607422,
+      "learning_rate": 1e-05,
+      "loss": 0.0346,
+      "num_tokens": 81641608.0,
+      "reward": 3.5197534561157227,
+      "reward_std": 0.09288936108350754,
+      "rewards/ngram_repetition2/mean": 0.882154107093811,
+      "rewards/ngram_repetition2/std": 0.02149037830531597,
+      "rewards/ngram_repetition3/mean": 0.9907884001731873,
+      "rewards/ngram_repetition3/std": 0.004652977455407381,
+      "rewards/symbolic_reward_accuracy/mean": 0.77392578125,
+      "rewards/symbolic_reward_accuracy/std": 0.4183899462223053,
+      "rewards/symbolic_reward_partial_score/mean": 0.944091796875,
+      "rewards/symbolic_reward_partial_score/std": 0.11925595253705978,
+      "rewards/tag_count_reward/mean": 0.999267578125,
+      "rewards/tag_count_reward/std": 0.019127286970615387,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9813138246536255,
+      "rewards/thinking_answer_ratio_reward/std": 0.031017476692795753,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6330723762512207,
+      "sampling/importance_sampling_ratio/min": 0.002558924490585923,
+      "sampling/sampling_logp_difference/max": 5.968168258666992,
+      "sampling/sampling_logp_difference/mean": 1.0812547206878662,
+      "step": 100
+    },
+    {
+      "clip_ratio/high_max": 0.0625,
+      "clip_ratio/high_mean": 0.02294921875,
+      "clip_ratio/low_mean": 0.36181640625,
+      "clip_ratio/low_min": 0.21875,
+      "clip_ratio/region_mean": 0.384765625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1466.0,
+      "completions/max_terminated_length": 1466.0,
+      "completions/mean_length": 860.47021484375,
+      "completions/mean_terminated_length": 860.47021484375,
+      "completions/min_length": 216.0,
+      "completions/min_terminated_length": 216.0,
+      "entropy": 3.9600134193897247,
+      "epoch": 1.1304347826086956,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.501100480556488,
+      "learning_rate": 1e-05,
+      "loss": 0.0304,
+      "num_tokens": 85434763.0,
+      "reward": 3.551880359649658,
+      "reward_std": 0.08972637355327606,
+      "rewards/ngram_repetition2/mean": 0.888477623462677,
+      "rewards/ngram_repetition2/std": 0.0196506567299366,
+      "rewards/ngram_repetition3/mean": 0.9930633306503296,
+      "rewards/ngram_repetition3/std": 0.004272008314728737,
+      "rewards/symbolic_reward_accuracy/mean": 0.7890625,
+      "rewards/symbolic_reward_accuracy/std": 0.408073753118515,
+      "rewards/symbolic_reward_partial_score/mean": 0.9453531503677368,
+      "rewards/symbolic_reward_partial_score/std": 0.12117670476436615,
+      "rewards/tag_count_reward/mean": 0.999755859375,
+      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9830958843231201,
+      "rewards/thinking_answer_ratio_reward/std": 0.022135263308882713,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.631166934967041,
+      "sampling/importance_sampling_ratio/min": 0.0002176268317271024,
+      "sampling/sampling_logp_difference/max": 8.43272876739502,
+      "sampling/sampling_logp_difference/mean": 1.10259211063385,
+      "step": 104
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.302734375,
+      "clip_ratio/low_min": 0.1796875,
+      "clip_ratio/region_mean": 0.302734375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1480.0,
+      "completions/max_terminated_length": 1480.0,
+      "completions/mean_length": 881.1708984375,
+      "completions/mean_terminated_length": 881.1708984375,
+      "completions/min_length": 5.0,
+      "completions/min_terminated_length": 5.0,
+      "entropy": 3.68235644698143,
+      "epoch": 1.1739130434782608,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.7818312644958496,
+      "learning_rate": 1e-05,
+      "loss": 0.0286,
+      "num_tokens": 89267145.0,
+      "reward": 3.489163398742676,
+      "reward_std": 0.21473455429077148,
+      "rewards/ngram_repetition2/mean": 0.8834545612335205,
+      "rewards/ngram_repetition2/std": 0.02104972116649151,
+      "rewards/ngram_repetition3/mean": 0.9921650886535645,
+      "rewards/ngram_repetition3/std": 0.004745583515614271,
+      "rewards/symbolic_reward_accuracy/mean": 0.77099609375,
+      "rewards/symbolic_reward_accuracy/std": 0.4202943742275238,
+      "rewards/symbolic_reward_partial_score/mean": 0.9245198965072632,
+      "rewards/symbolic_reward_partial_score/std": 0.16520294547080994,
+      "rewards/tag_count_reward/mean": 0.994140625,
+      "rewards/tag_count_reward/std": 0.05382164567708969,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9754455089569092,
+      "rewards/thinking_answer_ratio_reward/std": 0.08314761519432068,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6302025318145752,
+      "sampling/importance_sampling_ratio/min": 0.0031186703126877546,
+      "sampling/sampling_logp_difference/max": 5.77034854888916,
+      "sampling/sampling_logp_difference/mean": 1.0910520553588867,
+      "step": 108
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.32080078125,
+      "clip_ratio/low_min": 0.16015625,
+      "clip_ratio/region_mean": 0.32080078125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1600.0,
+      "completions/max_terminated_length": 1600.0,
+      "completions/mean_length": 1038.5830078125,
+      "completions/mean_terminated_length": 1038.5830078125,
+      "completions/min_length": 259.0,
+      "completions/min_terminated_length": 259.0,
+      "entropy": 3.8035393804311752,
+      "epoch": 1.2173913043478262,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.930414915084839,
+      "learning_rate": 1e-05,
+      "loss": 0.0317,
+      "num_tokens": 93412403.0,
+      "reward": 3.636284112930298,
+      "reward_std": 0.20913578569889069,
+      "rewards/ngram_repetition2/mean": 0.8412303924560547,
+      "rewards/ngram_repetition2/std": 0.019966477528214455,
+      "rewards/ngram_repetition3/mean": 0.9881943464279175,
+      "rewards/ngram_repetition3/std": 0.005055025219917297,
+      "rewards/symbolic_reward_accuracy/mean": 0.82958984375,
+      "rewards/symbolic_reward_accuracy/std": 0.376084566116333,
+      "rewards/symbolic_reward_partial_score/mean": 0.950439453125,
+      "rewards/symbolic_reward_partial_score/std": 0.12963470816612244,
+      "rewards/tag_count_reward/mean": 0.99853515625,
+      "rewards/tag_count_reward/std": 0.02703022211790085,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9835522770881653,
+      "rewards/thinking_answer_ratio_reward/std": 0.04572867602109909,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6397275924682617,
+      "sampling/importance_sampling_ratio/min": 1.4277229638537392e-06,
+      "sampling/sampling_logp_difference/max": 13.459429740905762,
+      "sampling/sampling_logp_difference/mean": 1.105949878692627,
+      "step": 112
+    },
+    {
+      "clip_ratio/high_max": 0.02734375,
+      "clip_ratio/high_mean": 0.00634765625,
+      "clip_ratio/low_mean": 0.32861328125,
+      "clip_ratio/low_min": 0.1953125,
+      "clip_ratio/region_mean": 0.3349609375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1575.0,
+      "completions/max_terminated_length": 1575.0,
+      "completions/mean_length": 1060.87890625,
+      "completions/mean_terminated_length": 1060.87890625,
+      "completions/min_length": 495.0,
+      "completions/min_terminated_length": 495.0,
+      "entropy": 3.8799190521240234,
+      "epoch": 1.2608695652173914,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.9784842729568481,
+      "learning_rate": 1e-05,
+      "loss": 0.0206,
+      "num_tokens": 97660347.0,
+      "reward": 3.4817919731140137,
+      "reward_std": 0.123238705098629,
+      "rewards/ngram_repetition2/mean": 0.8279547691345215,
+      "rewards/ngram_repetition2/std": 0.01901470310986042,
+      "rewards/ngram_repetition3/mean": 0.9874847531318665,
+      "rewards/ngram_repetition3/std": 0.004876126069575548,
+      "rewards/symbolic_reward_accuracy/mean": 0.765625,
+      "rewards/symbolic_reward_accuracy/std": 0.42371100187301636,
+      "rewards/symbolic_reward_partial_score/mean": 0.9267171621322632,
+      "rewards/symbolic_reward_partial_score/std": 0.16115300357341766,
+      "rewards/tag_count_reward/mean": 0.995849609375,
+      "rewards/tag_count_reward/std": 0.04537592828273773,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9820874333381653,
+      "rewards/thinking_answer_ratio_reward/std": 0.05360926315188408,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6399784088134766,
+      "sampling/importance_sampling_ratio/min": 0.00026858848286792636,
+      "sampling/sampling_logp_difference/max": 8.222330093383789,
+      "sampling/sampling_logp_difference/mean": 1.0847612619400024,
+      "step": 116
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.34912109375,
+      "clip_ratio/low_min": 0.22265625,
+      "clip_ratio/region_mean": 0.34912109375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1566.0,
+      "completions/max_terminated_length": 1566.0,
+      "completions/mean_length": 1056.3486328125,
+      "completions/mean_terminated_length": 1056.3486328125,
+      "completions/min_length": 396.0,
+      "completions/min_terminated_length": 396.0,
+      "entropy": 4.022845968604088,
+      "epoch": 1.3043478260869565,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.9172134399414062,
+      "learning_rate": 1e-05,
+      "loss": 0.0336,
+      "num_tokens": 101886341.0,
+      "reward": 3.4570488929748535,
+      "reward_std": 0.0775345116853714,
+      "rewards/ngram_repetition2/mean": 0.8566447496414185,
+      "rewards/ngram_repetition2/std": 0.017400307580828667,
+      "rewards/ngram_repetition3/mean": 0.991142988204956,
+      "rewards/ngram_repetition3/std": 0.004262521397322416,
+      "rewards/symbolic_reward_accuracy/mean": 0.74951171875,
+      "rewards/symbolic_reward_accuracy/std": 0.43340006470680237,
+      "rewards/symbolic_reward_partial_score/mean": 0.929931640625,
+      "rewards/symbolic_reward_partial_score/std": 0.146311417222023,
+      "rewards/tag_count_reward/mean": 0.999755859375,
+      "rewards/tag_count_reward/std": 0.011048543266952038,
+      "rewards/thinking_answer_ratio_reward/mean": 0.985991895198822,
+      "rewards/thinking_answer_ratio_reward/std": 0.007111483719199896,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6356934309005737,
+      "sampling/importance_sampling_ratio/min": 0.0025200077798217535,
+      "sampling/sampling_logp_difference/max": 5.983493328094482,
+      "sampling/sampling_logp_difference/mean": 1.1201770305633545,
+      "step": 120
+    },
+    {
+      "clip_ratio/high_max": 0.015625,
+      "clip_ratio/high_mean": 0.001953125,
+      "clip_ratio/low_mean": 0.36181640625,
+      "clip_ratio/low_min": 0.25,
+      "clip_ratio/region_mean": 0.36376953125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1387.0,
+      "completions/max_terminated_length": 1387.0,
+      "completions/mean_length": 929.70458984375,
+      "completions/mean_terminated_length": 929.70458984375,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 4.241550624370575,
+      "epoch": 1.3478260869565217,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.469667911529541,
+      "learning_rate": 1e-05,
+      "loss": 0.0194,
+      "num_tokens": 105843464.0,
+      "reward": 3.478210926055908,
+      "reward_std": 0.06497098505496979,
+      "rewards/ngram_repetition2/mean": 0.9079857468605042,
+      "rewards/ngram_repetition2/std": 0.014988835901021957,
+      "rewards/ngram_repetition3/mean": 0.995934009552002,
+      "rewards/ngram_repetition3/std": 0.0035896108020097017,
+      "rewards/symbolic_reward_accuracy/mean": 0.755859375,
+      "rewards/symbolic_reward_accuracy/std": 0.4296814203262329,
+      "rewards/symbolic_reward_partial_score/mean": 0.9385986328125,
+      "rewards/symbolic_reward_partial_score/std": 0.1285245567560196,
+      "rewards/tag_count_reward/mean": 0.9990234375,
+      "rewards/tag_count_reward/std": 0.022080888971686363,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9830856323242188,
+      "rewards/thinking_answer_ratio_reward/std": 0.04350229352712631,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.624282956123352,
+      "sampling/importance_sampling_ratio/min": 0.0017993793589994311,
+      "sampling/sampling_logp_difference/max": 6.320313453674316,
+      "sampling/sampling_logp_difference/mean": 1.111283779144287,
+      "step": 124
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.36279296875,
+      "clip_ratio/low_min": 0.234375,
+      "clip_ratio/region_mean": 0.36279296875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1281.0,
+      "completions/max_terminated_length": 1281.0,
+      "completions/mean_length": 887.55078125,
+      "completions/mean_terminated_length": 887.55078125,
+      "completions/min_length": 222.0,
+      "completions/min_terminated_length": 222.0,
+      "entropy": 4.1487970650196075,
+      "epoch": 1.391304347826087,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6331172585487366,
+      "learning_rate": 1e-05,
+      "loss": 0.018,
+      "num_tokens": 109714256.0,
+      "reward": 3.4955267906188965,
+      "reward_std": 0.07705140858888626,
+      "rewards/ngram_repetition2/mean": 0.9199254512786865,
+      "rewards/ngram_repetition2/std": 0.01465023122727871,
+      "rewards/ngram_repetition3/mean": 0.9966325163841248,
+      "rewards/ngram_repetition3/std": 0.0036702074576169252,
+      "rewards/symbolic_reward_accuracy/mean": 0.76416015625,
+      "rewards/symbolic_reward_accuracy/std": 0.4246262311935425,
+      "rewards/symbolic_reward_partial_score/mean": 0.940185546875,
+      "rewards/symbolic_reward_partial_score/std": 0.1274607628583908,
+      "rewards/tag_count_reward/mean": 0.998046875,
+      "rewards/tag_count_reward/std": 0.03119652159512043,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9808384776115417,
+      "rewards/thinking_answer_ratio_reward/std": 0.05348685011267662,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6253031492233276,
+      "sampling/importance_sampling_ratio/min": 0.0035080641973763704,
+      "sampling/sampling_logp_difference/max": 5.652690887451172,
+      "sampling/sampling_logp_difference/mean": 1.1522209644317627,
+      "step": 128
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.32470703125,
+      "clip_ratio/low_min": 0.1953125,
+      "clip_ratio/region_mean": 0.32470703125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1373.0,
+      "completions/max_terminated_length": 1373.0,
+      "completions/mean_length": 918.24755859375,
+      "completions/mean_terminated_length": 918.24755859375,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "entropy": 4.297388672828674,
+      "epoch": 1.434782608695652,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.1936229467391968,
+      "learning_rate": 1e-05,
+      "loss": 0.0167,
+      "num_tokens": 113647915.0,
+      "reward": 3.6550018787384033,
+      "reward_std": 0.12648092210292816,
+      "rewards/ngram_repetition2/mean": 0.9229152798652649,
+      "rewards/ngram_repetition2/std": 0.014652576297521591,
+      "rewards/ngram_repetition3/mean": 0.9968780279159546,
+      "rewards/ngram_repetition3/std": 0.004642123356461525,
+      "rewards/symbolic_reward_accuracy/mean": 0.837890625,
+      "rewards/symbolic_reward_accuracy/std": 0.3686411380767822,
+      "rewards/symbolic_reward_partial_score/mean": 0.9538980722427368,
+      "rewards/symbolic_reward_partial_score/std": 0.12867037951946259,
+      "rewards/tag_count_reward/mean": 0.996337890625,
+      "rewards/tag_count_reward/std": 0.04264424368739128,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9786805510520935,
+      "rewards/thinking_answer_ratio_reward/std": 0.07263471931219101,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6224591732025146,
+      "sampling/importance_sampling_ratio/min": 0.00012925604823976755,
+      "sampling/sampling_logp_difference/max": 8.953715324401855,
+      "sampling/sampling_logp_difference/mean": 1.143388032913208,
+      "step": 132
+    },
+    {
+      "clip_ratio/high_max": 0.00390625,
+      "clip_ratio/high_mean": 0.00048828125,
+      "clip_ratio/low_mean": 0.33984375,
+      "clip_ratio/low_min": 0.22265625,
+      "clip_ratio/region_mean": 0.34033203125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1310.0,
+      "completions/max_terminated_length": 1310.0,
+      "completions/mean_length": 871.56787109375,
+      "completions/mean_terminated_length": 871.56787109375,
+      "completions/min_length": 269.0,
+      "completions/min_terminated_length": 269.0,
+      "entropy": 4.470572978258133,
+      "epoch": 1.4782608695652173,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7518815398216248,
+      "learning_rate": 1e-05,
+      "loss": 0.0146,
+      "num_tokens": 117492310.0,
+      "reward": 3.562993049621582,
+      "reward_std": 0.1021745428442955,
+      "rewards/ngram_repetition2/mean": 0.9367643594741821,
+      "rewards/ngram_repetition2/std": 0.01393189001828432,
+      "rewards/ngram_repetition3/mean": 0.9970389604568481,
+      "rewards/ngram_repetition3/std": 0.0056902337819337845,
+      "rewards/symbolic_reward_accuracy/mean": 0.7998046875,
+      "rewards/symbolic_reward_accuracy/std": 0.4002441465854645,
+      "rewards/symbolic_reward_partial_score/mean": 0.9386392831802368,
+      "rewards/symbolic_reward_partial_score/std": 0.15098440647125244,
+      "rewards/tag_count_reward/mean": 0.99560546875,
+      "rewards/tag_count_reward/std": 0.04667995125055313,
+      "rewards/thinking_answer_ratio_reward/mean": 0.980078399181366,
+      "rewards/thinking_answer_ratio_reward/std": 0.04658709093928337,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.610716462135315,
+      "sampling/importance_sampling_ratio/min": 0.0008395969052799046,
+      "sampling/sampling_logp_difference/max": 7.0825886726379395,
+      "sampling/sampling_logp_difference/mean": 1.1626429557800293,
+      "step": 136
+    },
+    {
+      "clip_ratio/high_max": 0.00390625,
+      "clip_ratio/high_mean": 0.00048828125,
+      "clip_ratio/low_mean": 0.3232421875,
+      "clip_ratio/low_min": 0.19140625,
+      "clip_ratio/region_mean": 0.32373046875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1429.0,
+      "completions/max_terminated_length": 1429.0,
+      "completions/mean_length": 977.38232421875,
+      "completions/mean_terminated_length": 977.38232421875,
+      "completions/min_length": 74.0,
+      "completions/min_terminated_length": 74.0,
+      "entropy": 4.514361083507538,
+      "epoch": 1.5217391304347827,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8003511428833008,
+      "learning_rate": 1e-05,
+      "loss": 0.0119,
+      "num_tokens": 121528069.0,
+      "reward": 3.4866585731506348,
+      "reward_std": 0.09917771071195602,
+      "rewards/ngram_repetition2/mean": 0.9362497925758362,
+      "rewards/ngram_repetition2/std": 0.013396705500781536,
+      "rewards/ngram_repetition3/mean": 0.9971665143966675,
+      "rewards/ngram_repetition3/std": 0.00501285633072257,
+      "rewards/symbolic_reward_accuracy/mean": 0.76416015625,
+      "rewards/symbolic_reward_accuracy/std": 0.4246262311935425,
+      "rewards/symbolic_reward_partial_score/mean": 0.933837890625,
+      "rewards/symbolic_reward_partial_score/std": 0.15093302726745605,
+      "rewards/tag_count_reward/mean": 0.995361328125,
+      "rewards/tag_count_reward/std": 0.04794727638363838,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9805018901824951,
+      "rewards/thinking_answer_ratio_reward/std": 0.06314614415168762,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6080012321472168,
+      "sampling/importance_sampling_ratio/min": 0.00030363252153620124,
+      "sampling/sampling_logp_difference/max": 8.099692344665527,
+      "sampling/sampling_logp_difference/mean": 1.1383968591690063,
+      "step": 140
+    },
+    {
+      "clip_ratio/high_max": 0.03125,
+      "clip_ratio/high_mean": 0.00830078125,
+      "clip_ratio/low_mean": 0.2998046875,
+      "clip_ratio/low_min": 0.16796875,
+      "clip_ratio/region_mean": 0.30810546875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1363.0,
+      "completions/max_terminated_length": 1363.0,
+      "completions/mean_length": 898.19921875,
+      "completions/mean_terminated_length": 898.19921875,
+      "completions/min_length": 324.0,
+      "completions/min_terminated_length": 324.0,
+      "entropy": 4.418534815311432,
+      "epoch": 1.5652173913043477,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.44075483083724976,
+      "learning_rate": 1e-05,
+      "loss": 0.013,
+      "num_tokens": 125398493.0,
+      "reward": 3.593691825866699,
+      "reward_std": 0.1295282244682312,
+      "rewards/ngram_repetition2/mean": 0.9471422433853149,
+      "rewards/ngram_repetition2/std": 0.012752486392855644,
+      "rewards/ngram_repetition3/mean": 0.9974905252456665,
+      "rewards/ngram_repetition3/std": 0.005424594972282648,
+      "rewards/symbolic_reward_accuracy/mean": 0.81396484375,
+      "rewards/symbolic_reward_accuracy/std": 0.38923007249832153,
+      "rewards/symbolic_reward_partial_score/mean": 0.94189453125,
+      "rewards/symbolic_reward_partial_score/std": 0.1490928679704666,
+      "rewards/tag_count_reward/mean": 0.99462890625,
+      "rewards/tag_count_reward/std": 0.051555756479501724,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9792462587356567,
+      "rewards/thinking_answer_ratio_reward/std": 0.05640830844640732,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.607062816619873,
+      "sampling/importance_sampling_ratio/min": 0.0016754436073824763,
+      "sampling/sampling_logp_difference/max": 6.391677379608154,
+      "sampling/sampling_logp_difference/mean": 1.144271731376648,
+      "step": 144
+    },
+    {
+      "clip_ratio/high_max": 0.09375,
+      "clip_ratio/high_mean": 0.0263671875,
+      "clip_ratio/low_mean": 0.30810546875,
+      "clip_ratio/low_min": 0.16015625,
+      "clip_ratio/region_mean": 0.33447265625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1594.0,
+      "completions/max_terminated_length": 1594.0,
+      "completions/mean_length": 1067.33642578125,
+      "completions/mean_terminated_length": 1067.33642578125,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 4.551530241966248,
+      "epoch": 1.608695652173913,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3275977373123169,
+      "learning_rate": 1e-05,
+      "loss": 0.0142,
+      "num_tokens": 129669166.0,
+      "reward": 3.5267133712768555,
+      "reward_std": 0.0681791827082634,
+      "rewards/ngram_repetition2/mean": 0.9426812529563904,
+      "rewards/ngram_repetition2/std": 0.01187061332166195,
+      "rewards/ngram_repetition3/mean": 0.9978224039077759,
+      "rewards/ngram_repetition3/std": 0.00368481851182878,
+      "rewards/symbolic_reward_accuracy/mean": 0.77880859375,
+      "rewards/symbolic_reward_accuracy/std": 0.4151504933834076,
+      "rewards/symbolic_reward_partial_score/mean": 0.9430338144302368,
+      "rewards/symbolic_reward_partial_score/std": 0.1309022754430771,
+      "rewards/tag_count_reward/mean": 0.996826171875,
+      "rewards/tag_count_reward/std": 0.03971915319561958,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9831109046936035,
+      "rewards/thinking_answer_ratio_reward/std": 0.05386970564723015,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6074790954589844,
+      "sampling/importance_sampling_ratio/min": 9.599540499038994e-05,
+      "sampling/sampling_logp_difference/max": 9.25121021270752,
+      "sampling/sampling_logp_difference/mean": 1.1549028158187866,
+      "step": 148
+    },
+    {
+      "clip_ratio/high_max": 0.0859375,
+      "clip_ratio/high_mean": 0.029296875,
+      "clip_ratio/low_mean": 0.32666015625,
+      "clip_ratio/low_min": 0.1953125,
+      "clip_ratio/region_mean": 0.35595703125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1579.0,
+      "completions/max_terminated_length": 1579.0,
+      "completions/mean_length": 1040.93359375,
+      "completions/mean_terminated_length": 1040.93359375,
+      "completions/min_length": 565.0,
+      "completions/min_terminated_length": 565.0,
+      "entropy": 4.479687809944153,
+      "epoch": 1.6521739130434783,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3054217994213104,
+      "learning_rate": 1e-05,
+      "loss": 0.0146,
+      "num_tokens": 133847750.0,
+      "reward": 3.668701171875,
+      "reward_std": 0.0505342110991478,
+      "rewards/ngram_repetition2/mean": 0.9501175880432129,
+      "rewards/ngram_repetition2/std": 0.010720828548073769,
+      "rewards/ngram_repetition3/mean": 0.998192548751831,
+      "rewards/ngram_repetition3/std": 0.002623009728267789,
+      "rewards/symbolic_reward_accuracy/mean": 0.8369140625,
+      "rewards/symbolic_reward_accuracy/std": 0.36953428387641907,
+      "rewards/symbolic_reward_partial_score/mean": 0.9660237431526184,
+      "rewards/symbolic_reward_partial_score/std": 0.0860089510679245,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.015621182508766651,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9854624271392822,
+      "rewards/thinking_answer_ratio_reward/std": 0.02365659922361374,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.607342004776001,
+      "sampling/importance_sampling_ratio/min": 2.2357276975526474e-05,
+      "sampling/sampling_logp_difference/max": 10.708358764648438,
+      "sampling/sampling_logp_difference/mean": 1.1617491245269775,
+      "step": 152
+    },
+    {
+      "clip_ratio/high_max": 0.1484375,
+      "clip_ratio/high_mean": 0.0595703125,
+      "clip_ratio/low_mean": 0.2744140625,
+      "clip_ratio/low_min": 0.16015625,
+      "clip_ratio/region_mean": 0.333984375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1557.0,
+      "completions/max_terminated_length": 1557.0,
+      "completions/mean_length": 1038.66357421875,
+      "completions/mean_terminated_length": 1038.66357421875,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 4.511291533708572,
+      "epoch": 1.6956521739130435,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3336785137653351,
+      "learning_rate": 1e-05,
+      "loss": 0.0107,
+      "num_tokens": 138043861.0,
+      "reward": 3.3625810146331787,
+      "reward_std": 0.14225080609321594,
+      "rewards/ngram_repetition2/mean": 0.9543116092681885,
+      "rewards/ngram_repetition2/std": 0.01117774099111557,
+      "rewards/ngram_repetition3/mean": 0.998005211353302,
+      "rewards/ngram_repetition3/std": 0.004470388405025005,
+      "rewards/symbolic_reward_accuracy/mean": 0.7109375,
+      "rewards/symbolic_reward_accuracy/std": 0.4534377157688141,
+      "rewards/symbolic_reward_partial_score/mean": 0.916015625,
+      "rewards/symbolic_reward_partial_score/std": 0.16625778377056122,
+      "rewards/tag_count_reward/mean": 0.995361328125,
+      "rewards/tag_count_reward/std": 0.04794727638363838,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9806008338928223,
+      "rewards/thinking_answer_ratio_reward/std": 0.06250394135713577,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6025564670562744,
+      "sampling/importance_sampling_ratio/min": 5.173232784727588e-05,
+      "sampling/sampling_logp_difference/max": 9.869427680969238,
+      "sampling/sampling_logp_difference/mean": 1.1629631519317627,
+      "step": 156
+    },
+    {
+      "clip_ratio/high_max": 0.109375,
+      "clip_ratio/high_mean": 0.04296875,
+      "clip_ratio/low_mean": 0.2861328125,
+      "clip_ratio/low_min": 0.14453125,
+      "clip_ratio/region_mean": 0.3291015625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1716.0,
+      "completions/max_terminated_length": 1716.0,
+      "completions/mean_length": 1049.35009765625,
+      "completions/mean_terminated_length": 1049.35009765625,
+      "completions/min_length": 10.0,
+      "completions/min_terminated_length": 10.0,
+      "entropy": 4.4311443865299225,
+      "epoch": 1.7391304347826086,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22241854667663574,
+      "learning_rate": 1e-05,
+      "loss": 0.012,
+      "num_tokens": 142239682.0,
+      "reward": 3.6017355918884277,
+      "reward_std": 0.12942886352539062,
+      "rewards/ngram_repetition2/mean": 0.9581420421600342,
+      "rewards/ngram_repetition2/std": 0.01057825330644846,
+      "rewards/ngram_repetition3/mean": 0.9981793165206909,
+      "rewards/ngram_repetition3/std": 0.004097526893019676,
+      "rewards/symbolic_reward_accuracy/mean": 0.81884765625,
+      "rewards/symbolic_reward_accuracy/std": 0.3852384090423584,
+      "rewards/symbolic_reward_partial_score/mean": 0.9380696415901184,
+      "rewards/symbolic_reward_partial_score/std": 0.16267931461334229,
+      "rewards/tag_count_reward/mean": 0.99658203125,
+      "rewards/tag_count_reward/std": 0.04120838642120361,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9825441837310791,
+      "rewards/thinking_answer_ratio_reward/std": 0.04960091784596443,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6059880256652832,
+      "sampling/importance_sampling_ratio/min": 0.0016915490850806236,
+      "sampling/sampling_logp_difference/max": 6.382110595703125,
+      "sampling/sampling_logp_difference/mean": 1.1938023567199707,
+      "step": 160
+    },
+    {
+      "clip_ratio/high_max": 0.10546875,
+      "clip_ratio/high_mean": 0.046875,
+      "clip_ratio/low_mean": 0.2822265625,
+      "clip_ratio/low_min": 0.15234375,
+      "clip_ratio/region_mean": 0.3291015625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1622.0,
+      "completions/max_terminated_length": 1622.0,
+      "completions/mean_length": 1033.77587890625,
+      "completions/mean_terminated_length": 1033.77587890625,
+      "completions/min_length": 250.0,
+      "completions/min_terminated_length": 250.0,
+      "entropy": 4.365045428276062,
+      "epoch": 1.7826086956521738,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.21216052770614624,
+      "learning_rate": 1e-05,
+      "loss": 0.0114,
+      "num_tokens": 146419447.0,
+      "reward": 3.5902209281921387,
+      "reward_std": 0.1201181709766388,
+      "rewards/ngram_repetition2/mean": 0.9616591930389404,
+      "rewards/ngram_repetition2/std": 0.010357243940234184,
+      "rewards/ngram_repetition3/mean": 0.9983977675437927,
+      "rewards/ngram_repetition3/std": 0.004474774468690157,
+      "rewards/symbolic_reward_accuracy/mean": 0.8076171875,
+      "rewards/symbolic_reward_accuracy/std": 0.3942683935165405,
+      "rewards/symbolic_reward_partial_score/mean": 0.948486328125,
+      "rewards/symbolic_reward_partial_score/std": 0.12491386383771896,
+      "rewards/tag_count_reward/mean": 0.9970703125,
+      "rewards/tag_count_reward/std": 0.03817030414938927,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9829385876655579,
+      "rewards/thinking_answer_ratio_reward/std": 0.04094744101166725,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.609969139099121,
+      "sampling/importance_sampling_ratio/min": 3.6088283650315134e-06,
+      "sampling/sampling_logp_difference/max": 12.532127380371094,
+      "sampling/sampling_logp_difference/mean": 1.2227305173873901,
+      "step": 164
+    },
+    {
+      "clip_ratio/high_max": 0.09765625,
+      "clip_ratio/high_mean": 0.0400390625,
+      "clip_ratio/low_mean": 0.26953125,
+      "clip_ratio/low_min": 0.15625,
+      "clip_ratio/region_mean": 0.3095703125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1699.0,
+      "completions/max_terminated_length": 1699.0,
+      "completions/mean_length": 1053.5283203125,
+      "completions/mean_terminated_length": 1053.5283203125,
+      "completions/min_length": 42.0,
+      "completions/min_terminated_length": 42.0,
+      "entropy": 4.248802453279495,
+      "epoch": 1.8260869565217392,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.31685301661491394,
+      "learning_rate": 1e-05,
+      "loss": 0.0072,
+      "num_tokens": 150655505.0,
+      "reward": 3.371926784515381,
+      "reward_std": 0.20888543128967285,
+      "rewards/ngram_repetition2/mean": 0.9610307216644287,
+      "rewards/ngram_repetition2/std": 0.011032468639314175,
+      "rewards/ngram_repetition3/mean": 0.9981548190116882,
+      "rewards/ngram_repetition3/std": 0.005917286965996027,
+      "rewards/symbolic_reward_accuracy/mean": 0.71337890625,
+      "rewards/symbolic_reward_accuracy/std": 0.45229339599609375,
+      "rewards/symbolic_reward_partial_score/mean": 0.919189453125,
+      "rewards/symbolic_reward_partial_score/std": 0.157909095287323,
+      "rewards/tag_count_reward/mean": 0.99658203125,
+      "rewards/tag_count_reward/std": 0.04120838642120361,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9805506467819214,
+      "rewards/thinking_answer_ratio_reward/std": 0.06105503812432289,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6129553318023682,
+      "sampling/importance_sampling_ratio/min": 5.888082341698464e-07,
+      "sampling/sampling_logp_difference/max": 14.345165252685547,
+      "sampling/sampling_logp_difference/mean": 1.2391889095306396,
+      "step": 168
+    },
+    {
+      "clip_ratio/high_max": 0.0625,
+      "clip_ratio/high_mean": 0.02001953125,
+      "clip_ratio/low_mean": 0.32177734375,
+      "clip_ratio/low_min": 0.19140625,
+      "clip_ratio/region_mean": 0.341796875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1696.0,
+      "completions/max_terminated_length": 1696.0,
+      "completions/mean_length": 1089.537109375,
+      "completions/mean_terminated_length": 1089.537109375,
+      "completions/min_length": 465.0,
+      "completions/min_terminated_length": 465.0,
+      "entropy": 4.472538262605667,
+      "epoch": 1.8695652173913042,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8991731405258179,
+      "learning_rate": 1e-05,
+      "loss": 0.009,
+      "num_tokens": 154917789.0,
+      "reward": 3.514044761657715,
+      "reward_std": 0.12172482162714005,
+      "rewards/ngram_repetition2/mean": 0.9645071029663086,
+      "rewards/ngram_repetition2/std": 0.009678427129983902,
+      "rewards/ngram_repetition3/mean": 0.9983615875244141,
+      "rewards/ngram_repetition3/std": 0.004190252162516117,
+      "rewards/symbolic_reward_accuracy/mean": 0.77392578125,
+      "rewards/symbolic_reward_accuracy/std": 0.4183899462223053,
+      "rewards/symbolic_reward_partial_score/mean": 0.9379475712776184,
+      "rewards/symbolic_reward_partial_score/std": 0.13337303698062897,
+      "rewards/tag_count_reward/mean": 0.998779296875,
+      "rewards/tag_count_reward/std": 0.02468114346265793,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9837693572044373,
+      "rewards/thinking_answer_ratio_reward/std": 0.0436534620821476,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.6039378643035889,
+      "sampling/importance_sampling_ratio/min": 0.00011818981874966994,
+      "sampling/sampling_logp_difference/max": 9.043218612670898,
+      "sampling/sampling_logp_difference/mean": 1.231788158416748,
+      "step": 172
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.3046875,
+      "clip_ratio/low_min": 0.1875,
+      "clip_ratio/region_mean": 0.3046875,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1795.0,
+      "completions/max_terminated_length": 1795.0,
+      "completions/mean_length": 1130.46533203125,
+      "completions/mean_terminated_length": 1130.46533203125,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 4.910979479551315,
+      "epoch": 1.9130434782608696,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 3.6889185905456543,
+      "learning_rate": 1e-05,
+      "loss": 0.0104,
+      "num_tokens": 159301910.0,
+      "reward": 3.603275775909424,
+      "reward_std": 0.09857556223869324,
+      "rewards/ngram_repetition2/mean": 0.9671369791030884,
+      "rewards/ngram_repetition2/std": 0.009682528674602509,
+      "rewards/ngram_repetition3/mean": 0.9982260465621948,
+      "rewards/ngram_repetition3/std": 0.005538651719689369,
+      "rewards/symbolic_reward_accuracy/mean": 0.8125,
+      "rewards/symbolic_reward_accuracy/std": 0.3904076814651489,
+      "rewards/symbolic_reward_partial_score/mean": 0.9497476816177368,
+      "rewards/symbolic_reward_partial_score/std": 0.12718616425991058,
+      "rewards/tag_count_reward/mean": 0.9990234375,
+      "rewards/tag_count_reward/std": 0.022080888971686363,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9851157665252686,
+      "rewards/thinking_answer_ratio_reward/std": 0.04367439076304436,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.588046669960022,
+      "sampling/importance_sampling_ratio/min": 0.0005490838666446507,
+      "sampling/sampling_logp_difference/max": 7.507259368896484,
+      "sampling/sampling_logp_difference/mean": 1.1903247833251953,
+      "step": 176
+    },
+    {
+      "clip_ratio/high_max": 0.03125,
+      "clip_ratio/high_mean": 0.0068359375,
+      "clip_ratio/low_mean": 0.32373046875,
+      "clip_ratio/low_min": 0.203125,
+      "clip_ratio/region_mean": 0.33056640625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1933.0,
+      "completions/max_terminated_length": 1933.0,
+      "completions/mean_length": 1252.52880859375,
+      "completions/mean_terminated_length": 1252.52880859375,
+      "completions/min_length": 297.0,
+      "completions/min_terminated_length": 297.0,
+      "entropy": 4.105740413069725,
+      "epoch": 1.9565217391304348,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.827364206314087,
+      "learning_rate": 1e-05,
+      "loss": 0.019,
+      "num_tokens": 163936017.0,
+      "reward": 3.5113019943237305,
+      "reward_std": 0.056809887290000916,
+      "rewards/ngram_repetition2/mean": 0.9607850313186646,
+      "rewards/ngram_repetition2/std": 0.010354345664381981,
+      "rewards/ngram_repetition3/mean": 0.9982134699821472,
+      "rewards/ngram_repetition3/std": 0.004524180665612221,
+      "rewards/symbolic_reward_accuracy/mean": 0.76708984375,
+      "rewards/symbolic_reward_accuracy/std": 0.42278870940208435,
+      "rewards/symbolic_reward_partial_score/mean": 0.9481608271598816,
+      "rewards/symbolic_reward_partial_score/std": 0.1097652018070221,
+      "rewards/tag_count_reward/mean": 0.99951171875,
+      "rewards/tag_count_reward/std": 0.015621182508766651,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9859845638275146,
+      "rewards/thinking_answer_ratio_reward/std": 0.025435911491513252,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.619266390800476,
+      "sampling/importance_sampling_ratio/min": 0.00031076392042450607,
+      "sampling/sampling_logp_difference/max": 8.07647705078125,
+      "sampling/sampling_logp_difference/mean": 1.2865667343139648,
+      "step": 180
+    },
+    {
+      "clip_ratio/high_max": 0.03125,
+      "clip_ratio/high_mean": 0.00830078125,
+      "clip_ratio/low_mean": 0.33154296875,
+      "clip_ratio/low_min": 0.19921875,
+      "clip_ratio/region_mean": 0.33984375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2023.0,
+      "completions/max_terminated_length": 2023.0,
+      "completions/mean_length": 1276.42138671875,
+      "completions/mean_terminated_length": 1276.42138671875,
+      "completions/min_length": 565.0,
+      "completions/min_terminated_length": 565.0,
+      "entropy": 4.40667536854744,
+      "epoch": 2.0,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.1742569208145142,
+      "learning_rate": 1e-05,
+      "loss": 0.007,
+      "num_tokens": 168609552.0,
+      "reward": 3.480891227722168,
+      "reward_std": 0.0965752899646759,
+      "rewards/ngram_repetition2/mean": 0.9636451005935669,
+      "rewards/ngram_repetition2/std": 0.009043324738740921,
+      "rewards/ngram_repetition3/mean": 0.9984027147293091,
+      "rewards/ngram_repetition3/std": 0.003917085938155651,
+      "rewards/symbolic_reward_accuracy/mean": 0.76318359375,
+      "rewards/symbolic_reward_accuracy/std": 0.42523249983787537,
+      "rewards/symbolic_reward_partial_score/mean": 0.926025390625,
+      "rewards/symbolic_reward_partial_score/std": 0.15876662731170654,
+      "rewards/tag_count_reward/mean": 0.9990234375,
+      "rewards/tag_count_reward/std": 0.022080888971686363,
+      "rewards/thinking_answer_ratio_reward/mean": 0.9854665398597717,
+      "rewards/thinking_answer_ratio_reward/std": 0.04371679201722145,
+      "sampling/importance_sampling_ratio/max": 2.0,
+      "sampling/importance_sampling_ratio/mean": 1.607491374015808,
+      "sampling/importance_sampling_ratio/min": 1.9882392621184408e-07,
+      "sampling/sampling_logp_difference/max": 15.430846214294434,
+      "sampling/sampling_logp_difference/mean": 1.2338640689849854,
+      "step": 184
+    },
+    {
+      "epoch": 2.0,
+      "eval_clip_ratio/high_max": 0.0,
+      "eval_clip_ratio/high_mean": 0.0,
+      "eval_clip_ratio/low_mean": 0.0,
+      "eval_clip_ratio/low_min": 0.0,
+      "eval_clip_ratio/region_mean": 0.0,
+      "eval_completions/clipped_ratio": 0.0,
+      "eval_completions/max_length": 2041.3684210526317,
+      "eval_completions/max_terminated_length": 2041.3684210526317,
+      "eval_completions/mean_length": 1445.561677631579,
+      "eval_completions/mean_terminated_length": 1445.561677631579,
+      "eval_completions/min_length": 811.421052631579,
+      "eval_completions/min_terminated_length": 811.421052631579,
+      "eval_entropy": 4.193256829914294,
+      "eval_frac_reward_zero_std": 0.0,
+      "eval_loss": 0.003358551999554038,
+      "eval_num_tokens": 168609552.0,
+      "eval_reward": 3.548053026199341,
+      "eval_reward_std": 0.11297402592754224,
+      "eval_rewards/ngram_repetition2/mean": 0.9634329080581665,
+      "eval_rewards/ngram_repetition2/std": 0.008954386226832867,
+      "eval_rewards/ngram_repetition3/mean": 0.9986425355861062,
+      "eval_rewards/ngram_repetition3/std": 0.0030909376218914986,
+      "eval_rewards/symbolic_reward_accuracy/mean": 0.790296052631579,
+      "eval_rewards/symbolic_reward_accuracy/std": 0.33185332699825887,
+      "eval_rewards/symbolic_reward_partial_score/mean": 0.9396244507086905,
+      "eval_rewards/symbolic_reward_partial_score/std": 0.10662586594882764,
+      "eval_rewards/tag_count_reward/mean": 0.9983552631578947,
+      "eval_rewards/tag_count_reward/std": 0.015857031078715073,
+      "eval_rewards/thinking_answer_ratio_reward/mean": 0.986048914884266,
+      "eval_rewards/thinking_answer_ratio_reward/std": 0.025907071926140862,
+      "eval_runtime": 885.1922,
+      "eval_samples_per_second": 0.169,
+      "eval_sampling/importance_sampling_ratio/max": 2.0,
+      "eval_sampling/importance_sampling_ratio/mean": 1.6087917842363055,
+      "eval_sampling/importance_sampling_ratio/min": 0.0033183323022584724,
+      "eval_sampling/sampling_logp_difference/max": 6.073739528656006,
+      "eval_sampling/sampling_logp_difference/mean": 1.240292894212823,
+      "eval_steps_per_second": 0.002,
+      "step": 184
+    },
+    {
+      "epoch": 2.0,
+      "step": 184,
+      "total_flos": 0.0,
+      "train_loss": 0.0,
+      "train_runtime": 2.3434,
+      "train_samples_per_second": 2560.359,
+      "train_steps_per_second": 78.518
+    }
+  ],
+  "logging_steps": 4,
+  "max_steps": 184,
+  "num_input_tokens_seen": 168609552,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71f5640fe4ef5d58c11a6b709b1604b2ee41366c015db1bbc7c9a5c47f6ce303
+size 11665