Add files using upload-large-folder tool

README.md

@@ -0,0 +1,68 @@
+---
+library_name: transformers
+model_name: app/checkpoints/1/environment_test
+tags:
+- generated_from_trainer
+- trl
+- grpo
+- axolotl
+licence: license
+---
+
+# Model Card for app/checkpoints/1/environment_test
+
+This model is a fine-tuned version of [None](https://huggingface.co/None).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ductrung27-dt-none/Gradients-Rollout/runs/0vosajol) 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.27.0.dev0
+- Transformers: 4.57.6
+- Pytorch: 2.9.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{shao2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-20/README.md

@@ -0,0 +1,210 @@
+---
+base_model: /cache/models/Qwen--Qwen2.5-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-20/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-20/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-20/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-20/ref/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-20/special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-20/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-20/trainer_state.json

@@ -0,0 +1,794 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.16,
+  "eval_steps": 500,
+  "global_step": 20,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1707.0,
+      "completions/max_terminated_length": 915.0,
+      "completions/mean_length": 812.703125,
+      "completions/mean_terminated_length": 386.1499938964844,
+      "completions/min_length": 88.0,
+      "completions/min_terminated_length": 88.0,
+      "entropy": 0.319809939712286,
+      "epoch": 0.008,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.013934657908976078,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.0003,
+      "num_tokens": 52013.0,
+      "reward": 0.26906251907348633,
+      "reward_std": 0.43214160203933716,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26906251907348633,
+      "rewards/alfworld_rollout_reward_func/std": 0.48894354701042175,
+      "rollout/action_tokens_mean": 812.703125,
+      "rollout/episode_turns_mean": 25.0625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.401656150817871,
+      "sampling/importance_sampling_ratio/mean": 0.5413336753845215,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.9913136959075928,
+      "sampling/sampling_logp_difference/mean": 0.020846465602517128,
+      "step": 1,
+      "step_time": 340.2447626780008
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1338.0,
+      "completions/max_terminated_length": 912.0,
+      "completions/mean_length": 822.90625,
+      "completions/mean_terminated_length": 481.0000305175781,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 0.33951753890141845,
+      "epoch": 0.016,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.005984712392091751,
+      "kl": 0.0,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0064,
+      "num_tokens": 104679.0,
+      "reward": 0.1837500035762787,
+      "reward_std": 0.37211528420448303,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1837500035762787,
+      "rewards/alfworld_rollout_reward_func/std": 0.44909608364105225,
+      "rollout/action_tokens_mean": 822.90625,
+      "rollout/episode_turns_mean": 26.75,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.0904664993286133,
+      "sampling/importance_sampling_ratio/mean": 0.30532318353652954,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2203328609466553,
+      "sampling/sampling_logp_difference/mean": 0.021448608487844467,
+      "step": 2,
+      "step_time": 376.3160819019986
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1253.0,
+      "completions/max_terminated_length": 680.0,
+      "completions/mean_length": 735.65625,
+      "completions/mean_terminated_length": 380.5,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
+      "entropy": 0.32836992759257555,
+      "epoch": 0.024,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0066122072748839855,
+      "kl": 0.001546632844110718,
+      "learning_rate": 5e-06,
+      "loss": 0.0026,
+      "num_tokens": 151761.0,
+      "reward": 0.21156249940395355,
+      "reward_std": 0.2755644917488098,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21156249940395355,
+      "rewards/alfworld_rollout_reward_func/std": 0.44746631383895874,
+      "rollout/action_tokens_mean": 735.65625,
+      "rollout/episode_turns_mean": 26.03125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.4706838130950928,
+      "sampling/importance_sampling_ratio/mean": 0.5484369993209839,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.5531697273254395,
+      "sampling/sampling_logp_difference/mean": 0.020412152633070946,
+      "step": 3,
+      "step_time": 359.0840177950013
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.546875,
+      "completions/max_length": 1880.0,
+      "completions/max_terminated_length": 807.0,
+      "completions/mean_length": 721.78125,
+      "completions/mean_terminated_length": 425.72412109375,
+      "completions/min_length": 112.0,
+      "completions/min_terminated_length": 112.0,
+      "entropy": 0.3564013452269137,
+      "epoch": 0.032,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00901270005851984,
+      "kl": 0.001597001251866459,
+      "learning_rate": 4.996791267927632e-06,
+      "loss": 0.0123,
+      "num_tokens": 197955.0,
+      "reward": 0.40953126549720764,
+      "reward_std": 0.48288512229919434,
+      "rewards/alfworld_rollout_reward_func/mean": 0.40953126549720764,
+      "rewards/alfworld_rollout_reward_func/std": 0.5298889875411987,
+      "rollout/action_tokens_mean": 721.78125,
+      "rollout/episode_turns_mean": 22.859375,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.546875,
+      "sampling/importance_sampling_ratio/max": 2.607266664505005,
+      "sampling/importance_sampling_ratio/mean": 0.5041748881340027,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.2973906993865967,
+      "sampling/sampling_logp_difference/mean": 0.022099550813436508,
+      "step": 4,
+      "step_time": 324.93751245399835
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1772.0,
+      "completions/max_terminated_length": 1135.0,
+      "completions/mean_length": 823.984375,
+      "completions/mean_terminated_length": 511.25,
+      "completions/min_length": 238.0,
+      "completions/min_terminated_length": 238.0,
+      "entropy": 0.37002173252403736,
+      "epoch": 0.04,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006255371496081352,
+      "kl": 0.0018659899324120488,
+      "learning_rate": 4.987173308479738e-06,
+      "loss": -0.0049,
+      "num_tokens": 250690.0,
+      "reward": 0.2629687488079071,
+      "reward_std": 0.3148724138736725,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2629687786102295,
+      "rewards/alfworld_rollout_reward_func/std": 0.48142871260643005,
+      "rollout/action_tokens_mean": 823.984375,
+      "rollout/episode_turns_mean": 25.65625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.4376842975616455,
+      "sampling/importance_sampling_ratio/mean": 0.3966376781463623,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.3790063858032227,
+      "sampling/sampling_logp_difference/mean": 0.022958241403102875,
+      "step": 5,
+      "step_time": 428.7077085350011
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.546875,
+      "completions/max_length": 1754.0,
+      "completions/max_terminated_length": 1081.0,
+      "completions/mean_length": 778.265625,
+      "completions/mean_terminated_length": 513.0689697265625,
+      "completions/min_length": 89.0,
+      "completions/min_terminated_length": 89.0,
+      "entropy": 0.33683515526354313,
+      "epoch": 0.048,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.014856722205877304,
+      "kl": 0.0018188257581641665,
+      "learning_rate": 4.971170810820279e-06,
+      "loss": 0.0284,
+      "num_tokens": 300499.0,
+      "reward": 0.4112499952316284,
+      "reward_std": 0.4292137622833252,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4112499952316284,
+      "rewards/alfworld_rollout_reward_func/std": 0.5246087312698364,
+      "rollout/action_tokens_mean": 778.265625,
+      "rollout/episode_turns_mean": 24.0,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.546875,
+      "sampling/importance_sampling_ratio/max": 2.469376802444458,
+      "sampling/importance_sampling_ratio/mean": 0.41494274139404297,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.5756630897521973,
+      "sampling/sampling_logp_difference/mean": 0.020635541528463364,
+      "step": 6,
+      "step_time": 329.3498830619992
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1750.0,
+      "completions/max_terminated_length": 882.0,
+      "completions/mean_length": 834.359375,
+      "completions/mean_terminated_length": 538.4000244140625,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
+      "entropy": 0.37804630119353533,
+      "epoch": 0.056,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.011092925444245338,
+      "kl": 0.0017349459521938115,
+      "learning_rate": 4.948824853131237e-06,
+      "loss": 0.0135,
+      "num_tokens": 353898.0,
+      "reward": 0.1875,
+      "reward_std": 0.41170334815979004,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1875,
+      "rewards/alfworld_rollout_reward_func/std": 0.4403570294380188,
+      "rollout/action_tokens_mean": 834.359375,
+      "rollout/episode_turns_mean": 27.0625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.2279603481292725,
+      "sampling/importance_sampling_ratio/mean": 0.46820664405822754,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.733177900314331,
+      "sampling/sampling_logp_difference/mean": 0.022584954276680946,
+      "step": 7,
+      "step_time": 372.10799923799914
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1682.0,
+      "completions/max_terminated_length": 697.0,
+      "completions/mean_length": 786.5625,
+      "completions/mean_terminated_length": 380.8000183105469,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
+      "entropy": 0.32334376219660044,
+      "epoch": 0.064,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010468119755387306,
+      "kl": 0.0017176041819766397,
+      "learning_rate": 4.920192797165511e-06,
+      "loss": -0.0059,
+      "num_tokens": 404238.0,
+      "reward": 0.1939062476158142,
+      "reward_std": 0.31658345460891724,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1939062476158142,
+      "rewards/alfworld_rollout_reward_func/std": 0.44521304965019226,
+      "rollout/action_tokens_mean": 786.5625,
+      "rollout/episode_turns_mean": 25.984375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.8437609672546387,
+      "sampling/importance_sampling_ratio/mean": 0.6038376092910767,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.8486413955688477,
+      "sampling/sampling_logp_difference/mean": 0.0203598253428936,
+      "step": 8,
+      "step_time": 434.73393120799847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1492.0,
+      "completions/max_terminated_length": 1027.0,
+      "completions/mean_length": 818.34375,
+      "completions/mean_terminated_length": 474.0625,
+      "completions/min_length": 192.0,
+      "completions/min_terminated_length": 192.0,
+      "entropy": 0.32139473129063845,
+      "epoch": 0.072,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006268487311899662,
+      "kl": 0.0026504553206905257,
+      "learning_rate": 4.8853481410001225e-06,
+      "loss": -0.0089,
+      "num_tokens": 456612.0,
+      "reward": 0.19500000774860382,
+      "reward_std": 0.29811495542526245,
+      "rewards/alfworld_rollout_reward_func/mean": 0.19500000774860382,
+      "rewards/alfworld_rollout_reward_func/std": 0.46079298853874207,
+      "rollout/action_tokens_mean": 818.34375,
+      "rollout/episode_turns_mean": 26.109375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.6596250534057617,
+      "sampling/importance_sampling_ratio/mean": 0.42990007996559143,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3629698753356934,
+      "sampling/sampling_logp_difference/mean": 0.019470183178782463,
+      "step": 9,
+      "step_time": 399.7356371579963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.59375,
+      "completions/max_length": 1381.0,
+      "completions/max_terminated_length": 1015.0,
+      "completions/mean_length": 710.296875,
+      "completions/mean_terminated_length": 485.5384826660156,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
+      "entropy": 0.34738565888255835,
+      "epoch": 0.08,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010129230096936226,
+      "kl": 0.0015055066687637009,
+      "learning_rate": 4.844380330367701e-06,
+      "loss": 0.0042,
+      "num_tokens": 502071.0,
+      "reward": 0.3684375286102295,
+      "reward_std": 0.3336261808872223,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3684375286102295,
+      "rewards/alfworld_rollout_reward_func/std": 0.5042713284492493,
+      "rollout/action_tokens_mean": 710.296875,
+      "rollout/episode_turns_mean": 24.859375,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.59375,
+      "sampling/importance_sampling_ratio/max": 2.386782646179199,
+      "sampling/importance_sampling_ratio/mean": 0.5169863104820251,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.2723636627197266,
+      "sampling/sampling_logp_difference/mean": 0.022041790187358856,
+      "step": 10,
+      "step_time": 394.5470729360013
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 1352.0,
+      "completions/max_terminated_length": 782.0,
+      "completions/mean_length": 881.84375,
+      "completions/mean_terminated_length": 523.4444580078125,
+      "completions/min_length": 224.0,
+      "completions/min_terminated_length": 224.0,
+      "entropy": 0.3616171991452575,
+      "epoch": 0.088,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00803722906857729,
+      "kl": 0.0017023698201228399,
+      "learning_rate": 4.797394529050577e-06,
+      "loss": 0.0111,
+      "num_tokens": 558509.0,
+      "reward": 0.08921875059604645,
+      "reward_std": 0.2777005434036255,
+      "rewards/alfworld_rollout_reward_func/mean": 0.08921875059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.36485835909843445,
+      "rollout/action_tokens_mean": 881.84375,
+      "rollout/episode_turns_mean": 28.40625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.859375,
+      "sampling/importance_sampling_ratio/max": 2.7719504833221436,
+      "sampling/importance_sampling_ratio/mean": 0.4752790629863739,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.038581132888794,
+      "sampling/sampling_logp_difference/mean": 0.02142297849059105,
+      "step": 11,
+      "step_time": 396.1487058620023
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.671875,
+      "completions/max_length": 2292.0,
+      "completions/max_terminated_length": 901.0,
+      "completions/mean_length": 860.28125,
+      "completions/mean_terminated_length": 460.76190185546875,
+      "completions/min_length": 116.0,
+      "completions/min_terminated_length": 116.0,
+      "entropy": 0.35516145126894116,
+      "epoch": 0.096,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.011235510930418968,
+      "kl": 0.0016268756971840048,
+      "learning_rate": 4.744511348926855e-06,
+      "loss": 0.0048,
+      "num_tokens": 613567.0,
+      "reward": 0.26890623569488525,
+      "reward_std": 0.41516801714897156,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26890626549720764,
+      "rewards/alfworld_rollout_reward_func/std": 0.4936746656894684,
+      "rollout/action_tokens_mean": 860.28125,
+      "rollout/episode_turns_mean": 25.484375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.671875,
+      "sampling/importance_sampling_ratio/max": 2.7596683502197266,
+      "sampling/importance_sampling_ratio/mean": 0.5103189945220947,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.6376895904541016,
+      "sampling/sampling_logp_difference/mean": 0.02156790718436241,
+      "step": 12,
+      "step_time": 421.574981908002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 1382.0,
+      "completions/max_terminated_length": 775.0,
+      "completions/mean_length": 836.53125,
+      "completions/mean_terminated_length": 586.4166870117188,
+      "completions/min_length": 402.0,
+      "completions/min_terminated_length": 402.0,
+      "entropy": 0.35410969890654087,
+      "epoch": 0.104,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010335148312151432,
+      "kl": 0.0018077169770549517,
+      "learning_rate": 4.685866540361456e-06,
+      "loss": 0.0068,
+      "num_tokens": 667105.0,
+      "reward": 0.14281249046325684,
+      "reward_std": 0.3198840916156769,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14281249046325684,
+      "rewards/alfworld_rollout_reward_func/std": 0.3984073996543884,
+      "rollout/action_tokens_mean": 836.53125,
+      "rollout/episode_turns_mean": 28.453125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.8125,
+      "sampling/importance_sampling_ratio/max": 2.6870784759521484,
+      "sampling/importance_sampling_ratio/mean": 0.6008636951446533,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.14093017578125,
+      "sampling/sampling_logp_difference/mean": 0.021617209538817406,
+      "step": 13,
+      "step_time": 369.507772967002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1584.0,
+      "completions/max_terminated_length": 1063.0,
+      "completions/mean_length": 793.703125,
+      "completions/mean_terminated_length": 525.4500122070312,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
+      "entropy": 0.34002805408090353,
+      "epoch": 0.112,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00711849657818675,
+      "kl": 0.0019163258293701801,
+      "learning_rate": 4.621610643736878e-06,
+      "loss": 0.0062,
+      "num_tokens": 717902.0,
+      "reward": 0.2626562714576721,
+      "reward_std": 0.3680095076560974,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26265624165534973,
+      "rewards/alfworld_rollout_reward_func/std": 0.489940881729126,
+      "rollout/action_tokens_mean": 793.703125,
+      "rollout/episode_turns_mean": 26.28125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.815065860748291,
+      "sampling/importance_sampling_ratio/mean": 0.45154738426208496,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.7529265880584717,
+      "sampling/sampling_logp_difference/mean": 0.0205212514847517,
+      "step": 14,
+      "step_time": 327.6827599140015
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.84375,
+      "completions/max_length": 1814.0,
+      "completions/max_terminated_length": 890.0,
+      "completions/mean_length": 909.109375,
+      "completions/mean_terminated_length": 528.7999877929688,
+      "completions/min_length": 242.0,
+      "completions/min_terminated_length": 242.0,
+      "entropy": 0.35556588135659695,
+      "epoch": 0.12,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010462122038006783,
+      "kl": 0.0016676104332873365,
+      "learning_rate": 4.551908603018191e-06,
+      "loss": -0.0076,
+      "num_tokens": 776085.0,
+      "reward": 0.10140626132488251,
+      "reward_std": 0.36953580379486084,
+      "rewards/alfworld_rollout_reward_func/mean": 0.10140625387430191,
+      "rewards/alfworld_rollout_reward_func/std": 0.38891327381134033,
+      "rollout/action_tokens_mean": 909.109375,
+      "rollout/episode_turns_mean": 28.0,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.84375,
+      "sampling/importance_sampling_ratio/max": 2.0437681674957275,
+      "sampling/importance_sampling_ratio/mean": 0.3665631115436554,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.642125606536865,
+      "sampling/sampling_logp_difference/mean": 0.021498428657650948,
+      "step": 15,
+      "step_time": 409.45475878100024
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.703125,
+      "completions/max_length": 2069.0,
+      "completions/max_terminated_length": 815.0,
+      "completions/mean_length": 858.484375,
+      "completions/mean_terminated_length": 363.0,
+      "completions/min_length": 128.0,
+      "completions/min_terminated_length": 128.0,
+      "entropy": 0.3426384939812124,
+      "epoch": 0.128,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007739651948213577,
+      "kl": 0.0015719706807431066,
+      "learning_rate": 4.476939342344246e-06,
+      "loss": -0.002,
+      "num_tokens": 831028.0,
+      "reward": 0.2515625059604645,
+      "reward_std": 0.4047408103942871,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2515625059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.48498478531837463,
+      "rollout/action_tokens_mean": 858.484375,
+      "rollout/episode_turns_mean": 25.015625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.703125,
+      "sampling/importance_sampling_ratio/max": 2.197270631790161,
+      "sampling/importance_sampling_ratio/mean": 0.3564271330833435,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.3733601570129395,
+      "sampling/sampling_logp_difference/mean": 0.021193569526076317,
+      "step": 16,
+      "step_time": 341.3529842470016
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1344.0,
+      "completions/max_terminated_length": 872.0,
+      "completions/mean_length": 800.890625,
+      "completions/mean_terminated_length": 410.0000305175781,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
+      "entropy": 0.3335589300841093,
+      "epoch": 0.136,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.4890336990356445,
+      "kl": 0.8330444995917787,
+      "learning_rate": 4.396895306731978e-06,
+      "loss": 0.0005,
+      "num_tokens": 882285.0,
+      "reward": 0.18125000596046448,
+      "reward_std": 0.24295468628406525,
+      "rewards/alfworld_rollout_reward_func/mean": 0.18124999105930328,
+      "rewards/alfworld_rollout_reward_func/std": 0.4482647478580475,
+      "rollout/action_tokens_mean": 800.890625,
+      "rollout/episode_turns_mean": 26.265625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.3350934982299805,
+      "sampling/importance_sampling_ratio/mean": 0.37133079767227173,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 10.240796089172363,
+      "sampling/sampling_logp_difference/mean": 0.020599324256181717,
+      "step": 17,
+      "step_time": 436.20165634099976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.890625,
+      "completions/max_length": 1359.0,
+      "completions/max_terminated_length": 872.0,
+      "completions/mean_length": 864.640625,
+      "completions/mean_terminated_length": 638.7142944335938,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 0.3253911091014743,
+      "epoch": 0.144,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007197697646915913,
+      "kl": 0.0016915635187615408,
+      "learning_rate": 4.3119819680728e-06,
+      "loss": -0.0099,
+      "num_tokens": 937622.0,
+      "reward": 0.06968750059604645,
+      "reward_std": 0.19933855533599854,
+      "rewards/alfworld_rollout_reward_func/mean": 0.06968750059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.3224163353443146,
+      "rollout/action_tokens_mean": 864.640625,
+      "rollout/episode_turns_mean": 28.9375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.890625,
+      "sampling/importance_sampling_ratio/max": 2.2717995643615723,
+      "sampling/importance_sampling_ratio/mean": 0.5041627287864685,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.0254077911376953,
+      "sampling/sampling_logp_difference/mean": 0.019858187064528465,
+      "step": 18,
+      "step_time": 437.1889309570033
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.703125,
+      "completions/max_length": 2201.0,
+      "completions/max_terminated_length": 1115.0,
+      "completions/mean_length": 844.5,
+      "completions/mean_terminated_length": 540.631591796875,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
+      "entropy": 0.33306339103728533,
+      "epoch": 0.152,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010224551893770695,
+      "kl": 0.001574573227117071,
+      "learning_rate": 4.222417297689217e-06,
+      "loss": 0.0213,
+      "num_tokens": 991670.0,
+      "reward": 0.24437499046325684,
+      "reward_std": 0.42425790429115295,
+      "rewards/alfworld_rollout_reward_func/mean": 0.24437500536441803,
+      "rewards/alfworld_rollout_reward_func/std": 0.48038968443870544,
+      "rollout/action_tokens_mean": 844.5,
+      "rollout/episode_turns_mean": 26.171875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.703125,
+      "sampling/importance_sampling_ratio/max": 2.5866730213165283,
+      "sampling/importance_sampling_ratio/mean": 0.5376668572425842,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.632045865058899,
+      "sampling/sampling_logp_difference/mean": 0.01941317319869995,
+      "step": 19,
+      "step_time": 348.9997736020032
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1764.0,
+      "completions/max_terminated_length": 1085.0,
+      "completions/mean_length": 937.640625,
+      "completions/mean_terminated_length": 530.6875,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
+      "entropy": 0.3477782104164362,
+      "epoch": 0.16,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010612000711262226,
+      "kl": 0.001602184345756541,
+      "learning_rate": 4.128431206805556e-06,
+      "loss": 0.0044,
+      "num_tokens": 1051679.0,
+      "reward": 0.1901562511920929,
+      "reward_std": 0.39348262548446655,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1901562511920929,
+      "rewards/alfworld_rollout_reward_func/std": 0.4530303478240967,
+      "rollout/action_tokens_mean": 937.640625,
+      "rollout/episode_turns_mean": 26.6875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.5002288818359375,
+      "sampling/importance_sampling_ratio/mean": 0.4413934350013733,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.34731388092041,
+      "sampling/sampling_logp_difference/mean": 0.0198683924973011,
+      "step": 20,
+      "step_time": 353.7044870629943
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 64,
+  "num_input_tokens_seen": 1051679,
+  "num_train_epochs": 1,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-40/README.md

@@ -0,0 +1,210 @@
+---
+base_model: /cache/models/Qwen--Qwen2.5-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-40/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-40/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-40/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-40/special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-40/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-40/trainer_state.json

@@ -0,0 +1,1554 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.32,
+  "eval_steps": 500,
+  "global_step": 40,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1707.0,
+      "completions/max_terminated_length": 915.0,
+      "completions/mean_length": 812.703125,
+      "completions/mean_terminated_length": 386.1499938964844,
+      "completions/min_length": 88.0,
+      "completions/min_terminated_length": 88.0,
+      "entropy": 0.319809939712286,
+      "epoch": 0.008,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.013934657908976078,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.0003,
+      "num_tokens": 52013.0,
+      "reward": 0.26906251907348633,
+      "reward_std": 0.43214160203933716,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26906251907348633,
+      "rewards/alfworld_rollout_reward_func/std": 0.48894354701042175,
+      "rollout/action_tokens_mean": 812.703125,
+      "rollout/episode_turns_mean": 25.0625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.401656150817871,
+      "sampling/importance_sampling_ratio/mean": 0.5413336753845215,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.9913136959075928,
+      "sampling/sampling_logp_difference/mean": 0.020846465602517128,
+      "step": 1,
+      "step_time": 340.2447626780008
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1338.0,
+      "completions/max_terminated_length": 912.0,
+      "completions/mean_length": 822.90625,
+      "completions/mean_terminated_length": 481.0000305175781,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 0.33951753890141845,
+      "epoch": 0.016,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.005984712392091751,
+      "kl": 0.0,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0064,
+      "num_tokens": 104679.0,
+      "reward": 0.1837500035762787,
+      "reward_std": 0.37211528420448303,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1837500035762787,
+      "rewards/alfworld_rollout_reward_func/std": 0.44909608364105225,
+      "rollout/action_tokens_mean": 822.90625,
+      "rollout/episode_turns_mean": 26.75,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.0904664993286133,
+      "sampling/importance_sampling_ratio/mean": 0.30532318353652954,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2203328609466553,
+      "sampling/sampling_logp_difference/mean": 0.021448608487844467,
+      "step": 2,
+      "step_time": 376.3160819019986
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1253.0,
+      "completions/max_terminated_length": 680.0,
+      "completions/mean_length": 735.65625,
+      "completions/mean_terminated_length": 380.5,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
+      "entropy": 0.32836992759257555,
+      "epoch": 0.024,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0066122072748839855,
+      "kl": 0.001546632844110718,
+      "learning_rate": 5e-06,
+      "loss": 0.0026,
+      "num_tokens": 151761.0,
+      "reward": 0.21156249940395355,
+      "reward_std": 0.2755644917488098,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21156249940395355,
+      "rewards/alfworld_rollout_reward_func/std": 0.44746631383895874,
+      "rollout/action_tokens_mean": 735.65625,
+      "rollout/episode_turns_mean": 26.03125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.4706838130950928,
+      "sampling/importance_sampling_ratio/mean": 0.5484369993209839,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.5531697273254395,
+      "sampling/sampling_logp_difference/mean": 0.020412152633070946,
+      "step": 3,
+      "step_time": 359.0840177950013
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.546875,
+      "completions/max_length": 1880.0,
+      "completions/max_terminated_length": 807.0,
+      "completions/mean_length": 721.78125,
+      "completions/mean_terminated_length": 425.72412109375,
+      "completions/min_length": 112.0,
+      "completions/min_terminated_length": 112.0,
+      "entropy": 0.3564013452269137,
+      "epoch": 0.032,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00901270005851984,
+      "kl": 0.001597001251866459,
+      "learning_rate": 4.996791267927632e-06,
+      "loss": 0.0123,
+      "num_tokens": 197955.0,
+      "reward": 0.40953126549720764,
+      "reward_std": 0.48288512229919434,
+      "rewards/alfworld_rollout_reward_func/mean": 0.40953126549720764,
+      "rewards/alfworld_rollout_reward_func/std": 0.5298889875411987,
+      "rollout/action_tokens_mean": 721.78125,
+      "rollout/episode_turns_mean": 22.859375,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.546875,
+      "sampling/importance_sampling_ratio/max": 2.607266664505005,
+      "sampling/importance_sampling_ratio/mean": 0.5041748881340027,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.2973906993865967,
+      "sampling/sampling_logp_difference/mean": 0.022099550813436508,
+      "step": 4,
+      "step_time": 324.93751245399835
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1772.0,
+      "completions/max_terminated_length": 1135.0,
+      "completions/mean_length": 823.984375,
+      "completions/mean_terminated_length": 511.25,
+      "completions/min_length": 238.0,
+      "completions/min_terminated_length": 238.0,
+      "entropy": 0.37002173252403736,
+      "epoch": 0.04,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006255371496081352,
+      "kl": 0.0018659899324120488,
+      "learning_rate": 4.987173308479738e-06,
+      "loss": -0.0049,
+      "num_tokens": 250690.0,
+      "reward": 0.2629687488079071,
+      "reward_std": 0.3148724138736725,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2629687786102295,
+      "rewards/alfworld_rollout_reward_func/std": 0.48142871260643005,
+      "rollout/action_tokens_mean": 823.984375,
+      "rollout/episode_turns_mean": 25.65625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.4376842975616455,
+      "sampling/importance_sampling_ratio/mean": 0.3966376781463623,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.3790063858032227,
+      "sampling/sampling_logp_difference/mean": 0.022958241403102875,
+      "step": 5,
+      "step_time": 428.7077085350011
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.546875,
+      "completions/max_length": 1754.0,
+      "completions/max_terminated_length": 1081.0,
+      "completions/mean_length": 778.265625,
+      "completions/mean_terminated_length": 513.0689697265625,
+      "completions/min_length": 89.0,
+      "completions/min_terminated_length": 89.0,
+      "entropy": 0.33683515526354313,
+      "epoch": 0.048,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.014856722205877304,
+      "kl": 0.0018188257581641665,
+      "learning_rate": 4.971170810820279e-06,
+      "loss": 0.0284,
+      "num_tokens": 300499.0,
+      "reward": 0.4112499952316284,
+      "reward_std": 0.4292137622833252,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4112499952316284,
+      "rewards/alfworld_rollout_reward_func/std": 0.5246087312698364,
+      "rollout/action_tokens_mean": 778.265625,
+      "rollout/episode_turns_mean": 24.0,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.546875,
+      "sampling/importance_sampling_ratio/max": 2.469376802444458,
+      "sampling/importance_sampling_ratio/mean": 0.41494274139404297,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.5756630897521973,
+      "sampling/sampling_logp_difference/mean": 0.020635541528463364,
+      "step": 6,
+      "step_time": 329.3498830619992
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1750.0,
+      "completions/max_terminated_length": 882.0,
+      "completions/mean_length": 834.359375,
+      "completions/mean_terminated_length": 538.4000244140625,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
+      "entropy": 0.37804630119353533,
+      "epoch": 0.056,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.011092925444245338,
+      "kl": 0.0017349459521938115,
+      "learning_rate": 4.948824853131237e-06,
+      "loss": 0.0135,
+      "num_tokens": 353898.0,
+      "reward": 0.1875,
+      "reward_std": 0.41170334815979004,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1875,
+      "rewards/alfworld_rollout_reward_func/std": 0.4403570294380188,
+      "rollout/action_tokens_mean": 834.359375,
+      "rollout/episode_turns_mean": 27.0625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.2279603481292725,
+      "sampling/importance_sampling_ratio/mean": 0.46820664405822754,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.733177900314331,
+      "sampling/sampling_logp_difference/mean": 0.022584954276680946,
+      "step": 7,
+      "step_time": 372.10799923799914
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1682.0,
+      "completions/max_terminated_length": 697.0,
+      "completions/mean_length": 786.5625,
+      "completions/mean_terminated_length": 380.8000183105469,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
+      "entropy": 0.32334376219660044,
+      "epoch": 0.064,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010468119755387306,
+      "kl": 0.0017176041819766397,
+      "learning_rate": 4.920192797165511e-06,
+      "loss": -0.0059,
+      "num_tokens": 404238.0,
+      "reward": 0.1939062476158142,
+      "reward_std": 0.31658345460891724,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1939062476158142,
+      "rewards/alfworld_rollout_reward_func/std": 0.44521304965019226,
+      "rollout/action_tokens_mean": 786.5625,
+      "rollout/episode_turns_mean": 25.984375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.8437609672546387,
+      "sampling/importance_sampling_ratio/mean": 0.6038376092910767,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.8486413955688477,
+      "sampling/sampling_logp_difference/mean": 0.0203598253428936,
+      "step": 8,
+      "step_time": 434.73393120799847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1492.0,
+      "completions/max_terminated_length": 1027.0,
+      "completions/mean_length": 818.34375,
+      "completions/mean_terminated_length": 474.0625,
+      "completions/min_length": 192.0,
+      "completions/min_terminated_length": 192.0,
+      "entropy": 0.32139473129063845,
+      "epoch": 0.072,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006268487311899662,
+      "kl": 0.0026504553206905257,
+      "learning_rate": 4.8853481410001225e-06,
+      "loss": -0.0089,
+      "num_tokens": 456612.0,
+      "reward": 0.19500000774860382,
+      "reward_std": 0.29811495542526245,
+      "rewards/alfworld_rollout_reward_func/mean": 0.19500000774860382,
+      "rewards/alfworld_rollout_reward_func/std": 0.46079298853874207,
+      "rollout/action_tokens_mean": 818.34375,
+      "rollout/episode_turns_mean": 26.109375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.6596250534057617,
+      "sampling/importance_sampling_ratio/mean": 0.42990007996559143,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3629698753356934,
+      "sampling/sampling_logp_difference/mean": 0.019470183178782463,
+      "step": 9,
+      "step_time": 399.7356371579963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.59375,
+      "completions/max_length": 1381.0,
+      "completions/max_terminated_length": 1015.0,
+      "completions/mean_length": 710.296875,
+      "completions/mean_terminated_length": 485.5384826660156,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
+      "entropy": 0.34738565888255835,
+      "epoch": 0.08,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010129230096936226,
+      "kl": 0.0015055066687637009,
+      "learning_rate": 4.844380330367701e-06,
+      "loss": 0.0042,
+      "num_tokens": 502071.0,
+      "reward": 0.3684375286102295,
+      "reward_std": 0.3336261808872223,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3684375286102295,
+      "rewards/alfworld_rollout_reward_func/std": 0.5042713284492493,
+      "rollout/action_tokens_mean": 710.296875,
+      "rollout/episode_turns_mean": 24.859375,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.59375,
+      "sampling/importance_sampling_ratio/max": 2.386782646179199,
+      "sampling/importance_sampling_ratio/mean": 0.5169863104820251,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.2723636627197266,
+      "sampling/sampling_logp_difference/mean": 0.022041790187358856,
+      "step": 10,
+      "step_time": 394.5470729360013
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 1352.0,
+      "completions/max_terminated_length": 782.0,
+      "completions/mean_length": 881.84375,
+      "completions/mean_terminated_length": 523.4444580078125,
+      "completions/min_length": 224.0,
+      "completions/min_terminated_length": 224.0,
+      "entropy": 0.3616171991452575,
+      "epoch": 0.088,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00803722906857729,
+      "kl": 0.0017023698201228399,
+      "learning_rate": 4.797394529050577e-06,
+      "loss": 0.0111,
+      "num_tokens": 558509.0,
+      "reward": 0.08921875059604645,
+      "reward_std": 0.2777005434036255,
+      "rewards/alfworld_rollout_reward_func/mean": 0.08921875059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.36485835909843445,
+      "rollout/action_tokens_mean": 881.84375,
+      "rollout/episode_turns_mean": 28.40625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.859375,
+      "sampling/importance_sampling_ratio/max": 2.7719504833221436,
+      "sampling/importance_sampling_ratio/mean": 0.4752790629863739,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.038581132888794,
+      "sampling/sampling_logp_difference/mean": 0.02142297849059105,
+      "step": 11,
+      "step_time": 396.1487058620023
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.671875,
+      "completions/max_length": 2292.0,
+      "completions/max_terminated_length": 901.0,
+      "completions/mean_length": 860.28125,
+      "completions/mean_terminated_length": 460.76190185546875,
+      "completions/min_length": 116.0,
+      "completions/min_terminated_length": 116.0,
+      "entropy": 0.35516145126894116,
+      "epoch": 0.096,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.011235510930418968,
+      "kl": 0.0016268756971840048,
+      "learning_rate": 4.744511348926855e-06,
+      "loss": 0.0048,
+      "num_tokens": 613567.0,
+      "reward": 0.26890623569488525,
+      "reward_std": 0.41516801714897156,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26890626549720764,
+      "rewards/alfworld_rollout_reward_func/std": 0.4936746656894684,
+      "rollout/action_tokens_mean": 860.28125,
+      "rollout/episode_turns_mean": 25.484375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.671875,
+      "sampling/importance_sampling_ratio/max": 2.7596683502197266,
+      "sampling/importance_sampling_ratio/mean": 0.5103189945220947,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.6376895904541016,
+      "sampling/sampling_logp_difference/mean": 0.02156790718436241,
+      "step": 12,
+      "step_time": 421.574981908002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 1382.0,
+      "completions/max_terminated_length": 775.0,
+      "completions/mean_length": 836.53125,
+      "completions/mean_terminated_length": 586.4166870117188,
+      "completions/min_length": 402.0,
+      "completions/min_terminated_length": 402.0,
+      "entropy": 0.35410969890654087,
+      "epoch": 0.104,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010335148312151432,
+      "kl": 0.0018077169770549517,
+      "learning_rate": 4.685866540361456e-06,
+      "loss": 0.0068,
+      "num_tokens": 667105.0,
+      "reward": 0.14281249046325684,
+      "reward_std": 0.3198840916156769,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14281249046325684,
+      "rewards/alfworld_rollout_reward_func/std": 0.3984073996543884,
+      "rollout/action_tokens_mean": 836.53125,
+      "rollout/episode_turns_mean": 28.453125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.8125,
+      "sampling/importance_sampling_ratio/max": 2.6870784759521484,
+      "sampling/importance_sampling_ratio/mean": 0.6008636951446533,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.14093017578125,
+      "sampling/sampling_logp_difference/mean": 0.021617209538817406,
+      "step": 13,
+      "step_time": 369.507772967002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1584.0,
+      "completions/max_terminated_length": 1063.0,
+      "completions/mean_length": 793.703125,
+      "completions/mean_terminated_length": 525.4500122070312,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
+      "entropy": 0.34002805408090353,
+      "epoch": 0.112,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00711849657818675,
+      "kl": 0.0019163258293701801,
+      "learning_rate": 4.621610643736878e-06,
+      "loss": 0.0062,
+      "num_tokens": 717902.0,
+      "reward": 0.2626562714576721,
+      "reward_std": 0.3680095076560974,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26265624165534973,
+      "rewards/alfworld_rollout_reward_func/std": 0.489940881729126,
+      "rollout/action_tokens_mean": 793.703125,
+      "rollout/episode_turns_mean": 26.28125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.815065860748291,
+      "sampling/importance_sampling_ratio/mean": 0.45154738426208496,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.7529265880584717,
+      "sampling/sampling_logp_difference/mean": 0.0205212514847517,
+      "step": 14,
+      "step_time": 327.6827599140015
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.84375,
+      "completions/max_length": 1814.0,
+      "completions/max_terminated_length": 890.0,
+      "completions/mean_length": 909.109375,
+      "completions/mean_terminated_length": 528.7999877929688,
+      "completions/min_length": 242.0,
+      "completions/min_terminated_length": 242.0,
+      "entropy": 0.35556588135659695,
+      "epoch": 0.12,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010462122038006783,
+      "kl": 0.0016676104332873365,
+      "learning_rate": 4.551908603018191e-06,
+      "loss": -0.0076,
+      "num_tokens": 776085.0,
+      "reward": 0.10140626132488251,
+      "reward_std": 0.36953580379486084,
+      "rewards/alfworld_rollout_reward_func/mean": 0.10140625387430191,
+      "rewards/alfworld_rollout_reward_func/std": 0.38891327381134033,
+      "rollout/action_tokens_mean": 909.109375,
+      "rollout/episode_turns_mean": 28.0,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.84375,
+      "sampling/importance_sampling_ratio/max": 2.0437681674957275,
+      "sampling/importance_sampling_ratio/mean": 0.3665631115436554,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.642125606536865,
+      "sampling/sampling_logp_difference/mean": 0.021498428657650948,
+      "step": 15,
+      "step_time": 409.45475878100024
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.703125,
+      "completions/max_length": 2069.0,
+      "completions/max_terminated_length": 815.0,
+      "completions/mean_length": 858.484375,
+      "completions/mean_terminated_length": 363.0,
+      "completions/min_length": 128.0,
+      "completions/min_terminated_length": 128.0,
+      "entropy": 0.3426384939812124,
+      "epoch": 0.128,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007739651948213577,
+      "kl": 0.0015719706807431066,
+      "learning_rate": 4.476939342344246e-06,
+      "loss": -0.002,
+      "num_tokens": 831028.0,
+      "reward": 0.2515625059604645,
+      "reward_std": 0.4047408103942871,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2515625059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.48498478531837463,
+      "rollout/action_tokens_mean": 858.484375,
+      "rollout/episode_turns_mean": 25.015625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.703125,
+      "sampling/importance_sampling_ratio/max": 2.197270631790161,
+      "sampling/importance_sampling_ratio/mean": 0.3564271330833435,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.3733601570129395,
+      "sampling/sampling_logp_difference/mean": 0.021193569526076317,
+      "step": 16,
+      "step_time": 341.3529842470016
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1344.0,
+      "completions/max_terminated_length": 872.0,
+      "completions/mean_length": 800.890625,
+      "completions/mean_terminated_length": 410.0000305175781,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
+      "entropy": 0.3335589300841093,
+      "epoch": 0.136,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.4890336990356445,
+      "kl": 0.8330444995917787,
+      "learning_rate": 4.396895306731978e-06,
+      "loss": 0.0005,
+      "num_tokens": 882285.0,
+      "reward": 0.18125000596046448,
+      "reward_std": 0.24295468628406525,
+      "rewards/alfworld_rollout_reward_func/mean": 0.18124999105930328,
+      "rewards/alfworld_rollout_reward_func/std": 0.4482647478580475,
+      "rollout/action_tokens_mean": 800.890625,
+      "rollout/episode_turns_mean": 26.265625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.3350934982299805,
+      "sampling/importance_sampling_ratio/mean": 0.37133079767227173,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 10.240796089172363,
+      "sampling/sampling_logp_difference/mean": 0.020599324256181717,
+      "step": 17,
+      "step_time": 436.20165634099976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.890625,
+      "completions/max_length": 1359.0,
+      "completions/max_terminated_length": 872.0,
+      "completions/mean_length": 864.640625,
+      "completions/mean_terminated_length": 638.7142944335938,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 0.3253911091014743,
+      "epoch": 0.144,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007197697646915913,
+      "kl": 0.0016915635187615408,
+      "learning_rate": 4.3119819680728e-06,
+      "loss": -0.0099,
+      "num_tokens": 937622.0,
+      "reward": 0.06968750059604645,
+      "reward_std": 0.19933855533599854,
+      "rewards/alfworld_rollout_reward_func/mean": 0.06968750059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.3224163353443146,
+      "rollout/action_tokens_mean": 864.640625,
+      "rollout/episode_turns_mean": 28.9375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.890625,
+      "sampling/importance_sampling_ratio/max": 2.2717995643615723,
+      "sampling/importance_sampling_ratio/mean": 0.5041627287864685,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.0254077911376953,
+      "sampling/sampling_logp_difference/mean": 0.019858187064528465,
+      "step": 18,
+      "step_time": 437.1889309570033
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.703125,
+      "completions/max_length": 2201.0,
+      "completions/max_terminated_length": 1115.0,
+      "completions/mean_length": 844.5,
+      "completions/mean_terminated_length": 540.631591796875,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
+      "entropy": 0.33306339103728533,
+      "epoch": 0.152,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010224551893770695,
+      "kl": 0.001574573227117071,
+      "learning_rate": 4.222417297689217e-06,
+      "loss": 0.0213,
+      "num_tokens": 991670.0,
+      "reward": 0.24437499046325684,
+      "reward_std": 0.42425790429115295,
+      "rewards/alfworld_rollout_reward_func/mean": 0.24437500536441803,
+      "rewards/alfworld_rollout_reward_func/std": 0.48038968443870544,
+      "rollout/action_tokens_mean": 844.5,
+      "rollout/episode_turns_mean": 26.171875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.703125,
+      "sampling/importance_sampling_ratio/max": 2.5866730213165283,
+      "sampling/importance_sampling_ratio/mean": 0.5376668572425842,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.632045865058899,
+      "sampling/sampling_logp_difference/mean": 0.01941317319869995,
+      "step": 19,
+      "step_time": 348.9997736020032
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1764.0,
+      "completions/max_terminated_length": 1085.0,
+      "completions/mean_length": 937.640625,
+      "completions/mean_terminated_length": 530.6875,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
+      "entropy": 0.3477782104164362,
+      "epoch": 0.16,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010612000711262226,
+      "kl": 0.001602184345756541,
+      "learning_rate": 4.128431206805556e-06,
+      "loss": 0.0044,
+      "num_tokens": 1051679.0,
+      "reward": 0.1901562511920929,
+      "reward_std": 0.39348262548446655,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1901562511920929,
+      "rewards/alfworld_rollout_reward_func/std": 0.4530303478240967,
+      "rollout/action_tokens_mean": 937.640625,
+      "rollout/episode_turns_mean": 26.6875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.5002288818359375,
+      "sampling/importance_sampling_ratio/mean": 0.4413934350013733,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.34731388092041,
+      "sampling/sampling_logp_difference/mean": 0.0198683924973011,
+      "step": 20,
+      "step_time": 353.7044870629943
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.65625,
+      "completions/max_length": 1475.0,
+      "completions/max_terminated_length": 939.0,
+      "completions/mean_length": 800.453125,
+      "completions/mean_terminated_length": 418.0909118652344,
+      "completions/min_length": 99.0,
+      "completions/min_terminated_length": 99.0,
+      "entropy": 0.36278699059039354,
+      "epoch": 0.168,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010538731701672077,
+      "kl": 0.001873640143458033,
+      "learning_rate": 4.030264956369158e-06,
+      "loss": -0.0018,
+      "num_tokens": 1102908.0,
+      "reward": 0.2993749976158142,
+      "reward_std": 0.40504124760627747,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2993749976158142,
+      "rewards/alfworld_rollout_reward_func/std": 0.4982314109802246,
+      "rollout/action_tokens_mean": 800.453125,
+      "rollout/episode_turns_mean": 24.4375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.65625,
+      "sampling/importance_sampling_ratio/max": 2.447692632675171,
+      "sampling/importance_sampling_ratio/mean": 0.448123574256897,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2475838661193848,
+      "sampling/sampling_logp_difference/mean": 0.022293493151664734,
+      "step": 21,
+      "step_time": 452.2219315199982
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.625,
+      "completions/max_length": 1566.0,
+      "completions/max_terminated_length": 929.0,
+      "completions/mean_length": 743.703125,
+      "completions/mean_terminated_length": 460.91668701171875,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
+      "entropy": 0.3329707160592079,
+      "epoch": 0.176,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007643942255526781,
+      "kl": 0.0019213874111301266,
+      "learning_rate": 3.9281705377369814e-06,
+      "loss": 0.0059,
+      "num_tokens": 1150505.0,
+      "reward": 0.32625001668930054,
+      "reward_std": 0.4726526141166687,
+      "rewards/alfworld_rollout_reward_func/mean": 0.32624998688697815,
+      "rewards/alfworld_rollout_reward_func/std": 0.5121089816093445,
+      "rollout/action_tokens_mean": 743.703125,
+      "rollout/episode_turns_mean": 24.796875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.625,
+      "sampling/importance_sampling_ratio/max": 2.206357479095459,
+      "sampling/importance_sampling_ratio/mean": 0.4393410086631775,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.366598129272461,
+      "sampling/sampling_logp_difference/mean": 0.020861441269516945,
+      "step": 22,
+      "step_time": 350.210106904995
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.65625,
+      "completions/max_length": 1494.0,
+      "completions/max_terminated_length": 1159.0,
+      "completions/mean_length": 797.828125,
+      "completions/mean_terminated_length": 504.04547119140625,
+      "completions/min_length": 87.0,
+      "completions/min_terminated_length": 87.0,
+      "entropy": 0.35823060059919953,
+      "epoch": 0.184,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010937543585896492,
+      "kl": 0.00202080328745069,
+      "learning_rate": 3.8224100258174066e-06,
+      "loss": 0.0064,
+      "num_tokens": 1201566.0,
+      "reward": 0.3057812452316284,
+      "reward_std": 0.36121153831481934,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3057812452316284,
+      "rewards/alfworld_rollout_reward_func/std": 0.49343568086624146,
+      "rollout/action_tokens_mean": 797.828125,
+      "rollout/episode_turns_mean": 25.328125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.65625,
+      "sampling/importance_sampling_ratio/max": 2.8112552165985107,
+      "sampling/importance_sampling_ratio/mean": 0.43328696489334106,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2014861106872559,
+      "sampling/sampling_logp_difference/mean": 0.022007007151842117,
+      "step": 23,
+      "step_time": 299.4183257309978
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1409.0,
+      "completions/max_terminated_length": 962.0,
+      "completions/mean_length": 749.84375,
+      "completions/mean_terminated_length": 406.0,
+      "completions/min_length": 135.0,
+      "completions/min_terminated_length": 135.0,
+      "entropy": 0.314591265283525,
+      "epoch": 0.192,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.008087356574833393,
+      "kl": 0.0017565299822308589,
+      "learning_rate": 3.7132549063277033e-06,
+      "loss": -0.0022,
+      "num_tokens": 1249556.0,
+      "reward": 0.20937500894069672,
+      "reward_std": 0.2735254764556885,
+      "rewards/alfworld_rollout_reward_func/mean": 0.20937500894069672,
+      "rewards/alfworld_rollout_reward_func/std": 0.45049840211868286,
+      "rollout/action_tokens_mean": 749.84375,
+      "rollout/episode_turns_mean": 25.53125,
+      "rollout/mismatch_mean": 0.03125,
+      "rollout/mismatch_ratio": 0.03125,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.822704792022705,
+      "sampling/importance_sampling_ratio/mean": 0.4837469756603241,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2067627906799316,
+      "sampling/sampling_logp_difference/mean": 0.021238919347524643,
+      "step": 24,
+      "step_time": 423.74345705999986
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.828125,
+      "completions/max_length": 1469.0,
+      "completions/max_terminated_length": 854.0,
+      "completions/mean_length": 757.296875,
+      "completions/mean_terminated_length": 445.2727355957031,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
+      "entropy": 0.3146928045898676,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.005950183607637882,
+      "kl": 0.0024588965916336747,
+      "learning_rate": 3.600985378894086e-06,
+      "loss": -0.0029,
+      "num_tokens": 1298023.0,
+      "reward": 0.12843748927116394,
+      "reward_std": 0.3103031814098358,
+      "rewards/alfworld_rollout_reward_func/mean": 0.12843750417232513,
+      "rewards/alfworld_rollout_reward_func/std": 0.39419689774513245,
+      "rollout/action_tokens_mean": 757.296875,
+      "rollout/episode_turns_mean": 27.515625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.828125,
+      "sampling/importance_sampling_ratio/max": 1.9042965173721313,
+      "sampling/importance_sampling_ratio/mean": 0.30797556042671204,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.0986688137054443,
+      "sampling/sampling_logp_difference/mean": 0.0199856199324131,
+      "step": 25,
+      "step_time": 379.6568018289963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.796875,
+      "completions/max_length": 1607.0,
+      "completions/max_terminated_length": 975.0,
+      "completions/mean_length": 820.5,
+      "completions/mean_terminated_length": 396.923095703125,
+      "completions/min_length": 174.0,
+      "completions/min_terminated_length": 174.0,
+      "entropy": 0.301429130602628,
+      "epoch": 0.208,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006396389566361904,
+      "kl": 0.001548040645502624,
+      "learning_rate": 3.4858896377832966e-06,
+      "loss": 0.0009,
+      "num_tokens": 1350535.0,
+      "reward": 0.1639062464237213,
+      "reward_std": 0.32607388496398926,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1639062464237213,
+      "rewards/alfworld_rollout_reward_func/std": 0.418832927942276,
+      "rollout/action_tokens_mean": 820.5,
+      "rollout/episode_turns_mean": 26.59375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.796875,
+      "sampling/importance_sampling_ratio/max": 2.7253878116607666,
+      "sampling/importance_sampling_ratio/mean": 0.46614933013916016,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.6588082313537598,
+      "sampling/sampling_logp_difference/mean": 0.018608851358294487,
+      "step": 26,
+      "step_time": 458.5120370709956
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 1363.0,
+      "completions/max_terminated_length": 851.0,
+      "completions/mean_length": 843.46875,
+      "completions/mean_terminated_length": 518.0833740234375,
+      "completions/min_length": 123.0,
+      "completions/min_terminated_length": 123.0,
+      "entropy": 0.3490098095498979,
+      "epoch": 0.216,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00987387914210558,
+      "kl": 0.001589072278875392,
+      "learning_rate": 3.3682631321120507e-06,
+      "loss": -0.0109,
+      "num_tokens": 1404517.0,
+      "reward": 0.14921875298023224,
+      "reward_std": 0.3126556873321533,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14921875298023224,
+      "rewards/alfworld_rollout_reward_func/std": 0.40770089626312256,
+      "rollout/action_tokens_mean": 843.46875,
+      "rollout/episode_turns_mean": 27.703125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.8125,
+      "sampling/importance_sampling_ratio/max": 2.323033332824707,
+      "sampling/importance_sampling_ratio/mean": 0.4162726402282715,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.8558279275894165,
+      "sampling/sampling_logp_difference/mean": 0.020594514906406403,
+      "step": 27,
+      "step_time": 366.20077905700964
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5625,
+      "completions/max_length": 2369.0,
+      "completions/max_terminated_length": 1151.0,
+      "completions/mean_length": 727.859375,
+      "completions/mean_terminated_length": 404.3571472167969,
+      "completions/min_length": 97.0,
+      "completions/min_terminated_length": 97.0,
+      "entropy": 0.30788369150832295,
+      "epoch": 0.224,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01545465737581253,
+      "kl": 0.0016130298517964548,
+      "learning_rate": 3.248407807433396e-06,
+      "loss": 0.012,
+      "num_tokens": 1451100.0,
+      "reward": 0.3981249928474426,
+      "reward_std": 0.5213440656661987,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3981249928474426,
+      "rewards/alfworld_rollout_reward_func/std": 0.5216788649559021,
+      "rollout/action_tokens_mean": 727.859375,
+      "rollout/episode_turns_mean": 22.984375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.5625,
+      "sampling/importance_sampling_ratio/max": 2.7356457710266113,
+      "sampling/importance_sampling_ratio/mean": 0.6839978694915771,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.65842604637146,
+      "sampling/sampling_logp_difference/mean": 0.019816581159830093,
+      "step": 28,
+      "step_time": 317.7198553449962
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1781.0,
+      "completions/max_terminated_length": 980.0,
+      "completions/mean_length": 844.03125,
+      "completions/mean_terminated_length": 524.75,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
+      "entropy": 0.3514281096868217,
+      "epoch": 0.232,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010329142212867737,
+      "kl": 0.0034811663499567658,
+      "learning_rate": 3.1266313306468018e-06,
+      "loss": -0.0153,
+      "num_tokens": 1505118.0,
+      "reward": 0.20656250417232513,
+      "reward_std": 0.29942071437835693,
+      "rewards/alfworld_rollout_reward_func/mean": 0.20656250417232513,
+      "rewards/alfworld_rollout_reward_func/std": 0.4480122923851013,
+      "rollout/action_tokens_mean": 844.03125,
+      "rollout/episode_turns_mean": 26.890625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.6475582122802734,
+      "sampling/importance_sampling_ratio/mean": 0.39373910427093506,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 5.319100856781006,
+      "sampling/sampling_logp_difference/mean": 0.022610869258642197,
+      "step": 29,
+      "step_time": 372.01922321400525
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1465.0,
+      "completions/max_terminated_length": 804.0,
+      "completions/mean_length": 781.265625,
+      "completions/mean_terminated_length": 451.8125,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
+      "entropy": 0.2927012937143445,
+      "epoch": 0.24,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006770053878426552,
+      "kl": 0.0015057786895340541,
+      "learning_rate": 3.0032463002216504e-06,
+      "loss": 0.0038,
+      "num_tokens": 1555119.0,
+      "reward": 0.21671874821186066,
+      "reward_std": 0.24240338802337646,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21671876311302185,
+      "rewards/alfworld_rollout_reward_func/std": 0.44675213098526,
+      "rollout/action_tokens_mean": 781.265625,
+      "rollout/episode_turns_mean": 26.21875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.8479506969451904,
+      "sampling/importance_sampling_ratio/mean": 0.49506521224975586,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.7825241088867188,
+      "sampling/sampling_logp_difference/mean": 0.01828092336654663,
+      "step": 30,
+      "step_time": 382.84014304800075
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 1591.0,
+      "completions/max_terminated_length": 899.0,
+      "completions/mean_length": 807.0625,
+      "completions/mean_terminated_length": 503.8333435058594,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
+      "entropy": 0.33275228925049305,
+      "epoch": 0.248,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.009322401136159897,
+      "kl": 0.0018803979564836482,
+      "learning_rate": 2.878569443761442e-06,
+      "loss": -0.0078,
+      "num_tokens": 1606771.0,
+      "reward": 0.14656250178813934,
+      "reward_std": 0.337169349193573,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14656248688697815,
+      "rewards/alfworld_rollout_reward_func/std": 0.4100453853607178,
+      "rollout/action_tokens_mean": 807.0625,
+      "rollout/episode_turns_mean": 27.546875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.8125,
+      "sampling/importance_sampling_ratio/max": 2.9021573066711426,
+      "sampling/importance_sampling_ratio/mean": 0.42428821325302124,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.855207920074463,
+      "sampling/sampling_logp_difference/mean": 0.021321609616279602,
+      "step": 31,
+      "step_time": 494.6054373920015
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.796875,
+      "completions/max_length": 1383.0,
+      "completions/max_terminated_length": 1038.0,
+      "completions/mean_length": 791.734375,
+      "completions/mean_terminated_length": 412.923095703125,
+      "completions/min_length": 125.0,
+      "completions/min_terminated_length": 125.0,
+      "entropy": 0.38143961410969496,
+      "epoch": 0.256,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010985196568071842,
+      "kl": 0.0017831376208050642,
+      "learning_rate": 2.752920804968581e-06,
+      "loss": -0.0093,
+      "num_tokens": 1657442.0,
+      "reward": 0.14656248688697815,
+      "reward_std": 0.27167972922325134,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14656250178813934,
+      "rewards/alfworld_rollout_reward_func/std": 0.4263809323310852,
+      "rollout/action_tokens_mean": 791.734375,
+      "rollout/episode_turns_mean": 26.5,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.796875,
+      "sampling/importance_sampling_ratio/max": 2.170595169067383,
+      "sampling/importance_sampling_ratio/mean": 0.39966434240341187,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.398648977279663,
+      "sampling/sampling_logp_difference/mean": 0.022872522473335266,
+      "step": 32,
+      "step_time": 305.8924537160019
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.734375,
+      "completions/max_length": 1564.0,
+      "completions/max_terminated_length": 1151.0,
+      "completions/mean_length": 914.953125,
+      "completions/mean_terminated_length": 616.4705810546875,
+      "completions/min_length": 291.0,
+      "completions/min_terminated_length": 291.0,
+      "entropy": 0.39743291586637497,
+      "epoch": 0.264,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010168999433517456,
+      "kl": 0.0017962153833650518,
+      "learning_rate": 2.626622922096782e-06,
+      "loss": -0.0045,
+      "num_tokens": 1715999.0,
+      "reward": 0.21843750774860382,
+      "reward_std": 0.4097728729248047,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21843750774860382,
+      "rewards/alfworld_rollout_reward_func/std": 0.46011462807655334,
+      "rollout/action_tokens_mean": 914.953125,
+      "rollout/episode_turns_mean": 27.140625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.734375,
+      "sampling/importance_sampling_ratio/max": 2.6107707023620605,
+      "sampling/importance_sampling_ratio/mean": 0.386021226644516,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2772090435028076,
+      "sampling/sampling_logp_difference/mean": 0.02191011980175972,
+      "step": 33,
+      "step_time": 360.4347860389935
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.796875,
+      "completions/max_length": 1527.0,
+      "completions/max_terminated_length": 743.0,
+      "completions/mean_length": 862.9375,
+      "completions/mean_terminated_length": 506.3077087402344,
+      "completions/min_length": 254.0,
+      "completions/min_terminated_length": 254.0,
+      "entropy": 0.35848523303866386,
+      "epoch": 0.272,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01134358998388052,
+      "kl": 0.001760420509526739,
+      "learning_rate": 2.5e-06,
+      "loss": 0.016,
+      "num_tokens": 1771227.0,
+      "reward": 0.15781250596046448,
+      "reward_std": 0.29862138628959656,
+      "rewards/alfworld_rollout_reward_func/mean": 0.15781249105930328,
+      "rewards/alfworld_rollout_reward_func/std": 0.4202209413051605,
+      "rollout/action_tokens_mean": 862.9375,
+      "rollout/episode_turns_mean": 27.515625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.796875,
+      "sampling/importance_sampling_ratio/max": 2.3946166038513184,
+      "sampling/importance_sampling_ratio/mean": 0.41097065806388855,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.3659703731536865,
+      "sampling/sampling_logp_difference/mean": 0.020839743316173553,
+      "step": 34,
+      "step_time": 255.86244797499785
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.640625,
+      "completions/max_length": 1503.0,
+      "completions/max_terminated_length": 1127.0,
+      "completions/mean_length": 801.078125,
+      "completions/mean_terminated_length": 446.0434875488281,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
+      "entropy": 0.33974334690719843,
+      "epoch": 0.28,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007748698350042105,
+      "kl": 0.0017098757416533772,
+      "learning_rate": 2.3733770779032185e-06,
+      "loss": 0.0012,
+      "num_tokens": 1822496.0,
+      "reward": 0.3109375238418579,
+      "reward_std": 0.34154438972473145,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3109375238418579,
+      "rewards/alfworld_rollout_reward_func/std": 0.5012863874435425,
+      "rollout/action_tokens_mean": 801.078125,
+      "rollout/episode_turns_mean": 24.203125,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.640625,
+      "sampling/importance_sampling_ratio/max": 2.6798601150512695,
+      "sampling/importance_sampling_ratio/mean": 0.4648974537849426,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.4488577842712402,
+      "sampling/sampling_logp_difference/mean": 0.020977256819605827,
+      "step": 35,
+      "step_time": 369.7442883429976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5625,
+      "completions/max_length": 1817.0,
+      "completions/max_terminated_length": 1098.0,
+      "completions/mean_length": 763.078125,
+      "completions/mean_terminated_length": 383.21429443359375,
+      "completions/min_length": 90.0,
+      "completions/min_terminated_length": 90.0,
+      "entropy": 0.37786182574927807,
+      "epoch": 0.288,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.012169571593403816,
+      "kl": 0.0024242461659014225,
+      "learning_rate": 2.24707919503142e-06,
+      "loss": 0.0067,
+      "num_tokens": 1871333.0,
+      "reward": 0.3814062476158142,
+      "reward_std": 0.325703501701355,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3814062476158142,
+      "rewards/alfworld_rollout_reward_func/std": 0.5243707299232483,
+      "rollout/action_tokens_mean": 763.078125,
+      "rollout/episode_turns_mean": 22.328125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.5625,
+      "sampling/importance_sampling_ratio/max": 2.630308151245117,
+      "sampling/importance_sampling_ratio/mean": 0.4907624125480652,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.9331345558166504,
+      "sampling/sampling_logp_difference/mean": 0.022463176399469376,
+      "step": 36,
+      "step_time": 452.8674500970046
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.453125,
+      "completions/max_length": 1560.0,
+      "completions/max_terminated_length": 1173.0,
+      "completions/mean_length": 592.9375,
+      "completions/mean_terminated_length": 322.4285583496094,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
+      "entropy": 0.2960165496915579,
+      "epoch": 0.296,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.009033096954226494,
+      "kl": 0.0017318885384156602,
+      "learning_rate": 2.1214305562385592e-06,
+      "loss": 0.0028,
+      "num_tokens": 1909281.0,
+      "reward": 0.5143750309944153,
+      "reward_std": 0.3860568404197693,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5143750309944153,
+      "rewards/alfworld_rollout_reward_func/std": 0.5290928483009338,
+      "rollout/action_tokens_mean": 592.9375,
+      "rollout/episode_turns_mean": 19.015625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.453125,
+      "sampling/importance_sampling_ratio/max": 2.8567276000976562,
+      "sampling/importance_sampling_ratio/mean": 0.5531047582626343,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.304647445678711,
+      "sampling/sampling_logp_difference/mean": 0.01983683556318283,
+      "step": 37,
+      "step_time": 388.3879080370025
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.53125,
+      "completions/max_length": 1343.0,
+      "completions/max_terminated_length": 792.0,
+      "completions/mean_length": 662.140625,
+      "completions/mean_terminated_length": 343.8333435058594,
+      "completions/min_length": 77.0,
+      "completions/min_terminated_length": 77.0,
+      "entropy": 0.32609579246491194,
+      "epoch": 0.304,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.014773277565836906,
+      "kl": 0.0018760439170364407,
+      "learning_rate": 1.9967536997783495e-06,
+      "loss": 0.0014,
+      "num_tokens": 1951658.0,
+      "reward": 0.4350000023841858,
+      "reward_std": 0.3387959599494934,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4350000023841858,
+      "rewards/alfworld_rollout_reward_func/std": 0.5278016924858093,
+      "rollout/action_tokens_mean": 662.140625,
+      "rollout/episode_turns_mean": 21.40625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.53125,
+      "sampling/importance_sampling_ratio/max": 2.9260754585266113,
+      "sampling/importance_sampling_ratio/mean": 0.563366711139679,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.768939971923828,
+      "sampling/sampling_logp_difference/mean": 0.022423237562179565,
+      "step": 38,
+      "step_time": 436.1079989810023
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.734375,
+      "completions/max_length": 1869.0,
+      "completions/max_terminated_length": 1018.0,
+      "completions/mean_length": 943.40625,
+      "completions/mean_terminated_length": 584.4117431640625,
+      "completions/min_length": 237.0,
+      "completions/min_terminated_length": 237.0,
+      "entropy": 0.3758079996332526,
+      "epoch": 0.312,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01313227228820324,
+      "kl": 0.0022639515991613735,
+      "learning_rate": 1.8733686693531986e-06,
+      "loss": -0.0025,
+      "num_tokens": 2012036.0,
+      "reward": 0.21578124165534973,
+      "reward_std": 0.4323212504386902,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21578124165534973,
+      "rewards/alfworld_rollout_reward_func/std": 0.46428921818733215,
+      "rollout/action_tokens_mean": 943.40625,
+      "rollout/episode_turns_mean": 26.90625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.734375,
+      "sampling/importance_sampling_ratio/max": 2.5490753650665283,
+      "sampling/importance_sampling_ratio/mean": 0.3670840859413147,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.389880657196045,
+      "sampling/sampling_logp_difference/mean": 0.021743088960647583,
+      "step": 39,
+      "step_time": 354.37862886100265
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.671875,
+      "completions/max_length": 1334.0,
+      "completions/max_terminated_length": 1062.0,
+      "completions/mean_length": 701.15625,
+      "completions/mean_terminated_length": 308.1428527832031,
+      "completions/min_length": 155.0,
+      "completions/min_terminated_length": 155.0,
+      "entropy": 0.2994346688501537,
+      "epoch": 0.32,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.009722444228827953,
+      "kl": 0.0016241020402958384,
+      "learning_rate": 1.7515921925666053e-06,
+      "loss": -0.0206,
+      "num_tokens": 2056910.0,
+      "reward": 0.29640626907348633,
+      "reward_std": 0.23870769143104553,
+      "rewards/alfworld_rollout_reward_func/mean": 0.29640626907348633,
+      "rewards/alfworld_rollout_reward_func/std": 0.4918813109397888,
+      "rollout/action_tokens_mean": 701.15625,
+      "rollout/episode_turns_mean": 23.625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.671875,
+      "sampling/importance_sampling_ratio/max": 2.825843572616577,
+      "sampling/importance_sampling_ratio/mean": 0.5806776881217957,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3793482780456543,
+      "sampling/sampling_logp_difference/mean": 0.01931983232498169,
+      "step": 40,
+      "step_time": 355.1650720970065
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 64,
+  "num_input_tokens_seen": 2056910,
+  "num_train_epochs": 1,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-60/README.md

@@ -0,0 +1,210 @@
+---
+base_model: /cache/models/Qwen--Qwen2.5-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-60/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-60/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-60/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-60/ref/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-60/special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-60/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-60/trainer_state.json

@@ -0,0 +1,2314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.48,
+  "eval_steps": 500,
+  "global_step": 60,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1707.0,
+      "completions/max_terminated_length": 915.0,
+      "completions/mean_length": 812.703125,
+      "completions/mean_terminated_length": 386.1499938964844,
+      "completions/min_length": 88.0,
+      "completions/min_terminated_length": 88.0,
+      "entropy": 0.319809939712286,
+      "epoch": 0.008,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.013934657908976078,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.0003,
+      "num_tokens": 52013.0,
+      "reward": 0.26906251907348633,
+      "reward_std": 0.43214160203933716,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26906251907348633,
+      "rewards/alfworld_rollout_reward_func/std": 0.48894354701042175,
+      "rollout/action_tokens_mean": 812.703125,
+      "rollout/episode_turns_mean": 25.0625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.401656150817871,
+      "sampling/importance_sampling_ratio/mean": 0.5413336753845215,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.9913136959075928,
+      "sampling/sampling_logp_difference/mean": 0.020846465602517128,
+      "step": 1,
+      "step_time": 340.2447626780008
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1338.0,
+      "completions/max_terminated_length": 912.0,
+      "completions/mean_length": 822.90625,
+      "completions/mean_terminated_length": 481.0000305175781,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 0.33951753890141845,
+      "epoch": 0.016,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.005984712392091751,
+      "kl": 0.0,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0064,
+      "num_tokens": 104679.0,
+      "reward": 0.1837500035762787,
+      "reward_std": 0.37211528420448303,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1837500035762787,
+      "rewards/alfworld_rollout_reward_func/std": 0.44909608364105225,
+      "rollout/action_tokens_mean": 822.90625,
+      "rollout/episode_turns_mean": 26.75,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.0904664993286133,
+      "sampling/importance_sampling_ratio/mean": 0.30532318353652954,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2203328609466553,
+      "sampling/sampling_logp_difference/mean": 0.021448608487844467,
+      "step": 2,
+      "step_time": 376.3160819019986
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1253.0,
+      "completions/max_terminated_length": 680.0,
+      "completions/mean_length": 735.65625,
+      "completions/mean_terminated_length": 380.5,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
+      "entropy": 0.32836992759257555,
+      "epoch": 0.024,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0066122072748839855,
+      "kl": 0.001546632844110718,
+      "learning_rate": 5e-06,
+      "loss": 0.0026,
+      "num_tokens": 151761.0,
+      "reward": 0.21156249940395355,
+      "reward_std": 0.2755644917488098,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21156249940395355,
+      "rewards/alfworld_rollout_reward_func/std": 0.44746631383895874,
+      "rollout/action_tokens_mean": 735.65625,
+      "rollout/episode_turns_mean": 26.03125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.4706838130950928,
+      "sampling/importance_sampling_ratio/mean": 0.5484369993209839,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.5531697273254395,
+      "sampling/sampling_logp_difference/mean": 0.020412152633070946,
+      "step": 3,
+      "step_time": 359.0840177950013
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.546875,
+      "completions/max_length": 1880.0,
+      "completions/max_terminated_length": 807.0,
+      "completions/mean_length": 721.78125,
+      "completions/mean_terminated_length": 425.72412109375,
+      "completions/min_length": 112.0,
+      "completions/min_terminated_length": 112.0,
+      "entropy": 0.3564013452269137,
+      "epoch": 0.032,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00901270005851984,
+      "kl": 0.001597001251866459,
+      "learning_rate": 4.996791267927632e-06,
+      "loss": 0.0123,
+      "num_tokens": 197955.0,
+      "reward": 0.40953126549720764,
+      "reward_std": 0.48288512229919434,
+      "rewards/alfworld_rollout_reward_func/mean": 0.40953126549720764,
+      "rewards/alfworld_rollout_reward_func/std": 0.5298889875411987,
+      "rollout/action_tokens_mean": 721.78125,
+      "rollout/episode_turns_mean": 22.859375,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.546875,
+      "sampling/importance_sampling_ratio/max": 2.607266664505005,
+      "sampling/importance_sampling_ratio/mean": 0.5041748881340027,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.2973906993865967,
+      "sampling/sampling_logp_difference/mean": 0.022099550813436508,
+      "step": 4,
+      "step_time": 324.93751245399835
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1772.0,
+      "completions/max_terminated_length": 1135.0,
+      "completions/mean_length": 823.984375,
+      "completions/mean_terminated_length": 511.25,
+      "completions/min_length": 238.0,
+      "completions/min_terminated_length": 238.0,
+      "entropy": 0.37002173252403736,
+      "epoch": 0.04,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006255371496081352,
+      "kl": 0.0018659899324120488,
+      "learning_rate": 4.987173308479738e-06,
+      "loss": -0.0049,
+      "num_tokens": 250690.0,
+      "reward": 0.2629687488079071,
+      "reward_std": 0.3148724138736725,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2629687786102295,
+      "rewards/alfworld_rollout_reward_func/std": 0.48142871260643005,
+      "rollout/action_tokens_mean": 823.984375,
+      "rollout/episode_turns_mean": 25.65625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.4376842975616455,
+      "sampling/importance_sampling_ratio/mean": 0.3966376781463623,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.3790063858032227,
+      "sampling/sampling_logp_difference/mean": 0.022958241403102875,
+      "step": 5,
+      "step_time": 428.7077085350011
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.546875,
+      "completions/max_length": 1754.0,
+      "completions/max_terminated_length": 1081.0,
+      "completions/mean_length": 778.265625,
+      "completions/mean_terminated_length": 513.0689697265625,
+      "completions/min_length": 89.0,
+      "completions/min_terminated_length": 89.0,
+      "entropy": 0.33683515526354313,
+      "epoch": 0.048,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.014856722205877304,
+      "kl": 0.0018188257581641665,
+      "learning_rate": 4.971170810820279e-06,
+      "loss": 0.0284,
+      "num_tokens": 300499.0,
+      "reward": 0.4112499952316284,
+      "reward_std": 0.4292137622833252,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4112499952316284,
+      "rewards/alfworld_rollout_reward_func/std": 0.5246087312698364,
+      "rollout/action_tokens_mean": 778.265625,
+      "rollout/episode_turns_mean": 24.0,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.546875,
+      "sampling/importance_sampling_ratio/max": 2.469376802444458,
+      "sampling/importance_sampling_ratio/mean": 0.41494274139404297,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.5756630897521973,
+      "sampling/sampling_logp_difference/mean": 0.020635541528463364,
+      "step": 6,
+      "step_time": 329.3498830619992
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1750.0,
+      "completions/max_terminated_length": 882.0,
+      "completions/mean_length": 834.359375,
+      "completions/mean_terminated_length": 538.4000244140625,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
+      "entropy": 0.37804630119353533,
+      "epoch": 0.056,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.011092925444245338,
+      "kl": 0.0017349459521938115,
+      "learning_rate": 4.948824853131237e-06,
+      "loss": 0.0135,
+      "num_tokens": 353898.0,
+      "reward": 0.1875,
+      "reward_std": 0.41170334815979004,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1875,
+      "rewards/alfworld_rollout_reward_func/std": 0.4403570294380188,
+      "rollout/action_tokens_mean": 834.359375,
+      "rollout/episode_turns_mean": 27.0625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.2279603481292725,
+      "sampling/importance_sampling_ratio/mean": 0.46820664405822754,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.733177900314331,
+      "sampling/sampling_logp_difference/mean": 0.022584954276680946,
+      "step": 7,
+      "step_time": 372.10799923799914
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1682.0,
+      "completions/max_terminated_length": 697.0,
+      "completions/mean_length": 786.5625,
+      "completions/mean_terminated_length": 380.8000183105469,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
+      "entropy": 0.32334376219660044,
+      "epoch": 0.064,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010468119755387306,
+      "kl": 0.0017176041819766397,
+      "learning_rate": 4.920192797165511e-06,
+      "loss": -0.0059,
+      "num_tokens": 404238.0,
+      "reward": 0.1939062476158142,
+      "reward_std": 0.31658345460891724,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1939062476158142,
+      "rewards/alfworld_rollout_reward_func/std": 0.44521304965019226,
+      "rollout/action_tokens_mean": 786.5625,
+      "rollout/episode_turns_mean": 25.984375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.8437609672546387,
+      "sampling/importance_sampling_ratio/mean": 0.6038376092910767,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.8486413955688477,
+      "sampling/sampling_logp_difference/mean": 0.0203598253428936,
+      "step": 8,
+      "step_time": 434.73393120799847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1492.0,
+      "completions/max_terminated_length": 1027.0,
+      "completions/mean_length": 818.34375,
+      "completions/mean_terminated_length": 474.0625,
+      "completions/min_length": 192.0,
+      "completions/min_terminated_length": 192.0,
+      "entropy": 0.32139473129063845,
+      "epoch": 0.072,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006268487311899662,
+      "kl": 0.0026504553206905257,
+      "learning_rate": 4.8853481410001225e-06,
+      "loss": -0.0089,
+      "num_tokens": 456612.0,
+      "reward": 0.19500000774860382,
+      "reward_std": 0.29811495542526245,
+      "rewards/alfworld_rollout_reward_func/mean": 0.19500000774860382,
+      "rewards/alfworld_rollout_reward_func/std": 0.46079298853874207,
+      "rollout/action_tokens_mean": 818.34375,
+      "rollout/episode_turns_mean": 26.109375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.6596250534057617,
+      "sampling/importance_sampling_ratio/mean": 0.42990007996559143,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3629698753356934,
+      "sampling/sampling_logp_difference/mean": 0.019470183178782463,
+      "step": 9,
+      "step_time": 399.7356371579963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.59375,
+      "completions/max_length": 1381.0,
+      "completions/max_terminated_length": 1015.0,
+      "completions/mean_length": 710.296875,
+      "completions/mean_terminated_length": 485.5384826660156,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
+      "entropy": 0.34738565888255835,
+      "epoch": 0.08,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010129230096936226,
+      "kl": 0.0015055066687637009,
+      "learning_rate": 4.844380330367701e-06,
+      "loss": 0.0042,
+      "num_tokens": 502071.0,
+      "reward": 0.3684375286102295,
+      "reward_std": 0.3336261808872223,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3684375286102295,
+      "rewards/alfworld_rollout_reward_func/std": 0.5042713284492493,
+      "rollout/action_tokens_mean": 710.296875,
+      "rollout/episode_turns_mean": 24.859375,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.59375,
+      "sampling/importance_sampling_ratio/max": 2.386782646179199,
+      "sampling/importance_sampling_ratio/mean": 0.5169863104820251,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.2723636627197266,
+      "sampling/sampling_logp_difference/mean": 0.022041790187358856,
+      "step": 10,
+      "step_time": 394.5470729360013
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 1352.0,
+      "completions/max_terminated_length": 782.0,
+      "completions/mean_length": 881.84375,
+      "completions/mean_terminated_length": 523.4444580078125,
+      "completions/min_length": 224.0,
+      "completions/min_terminated_length": 224.0,
+      "entropy": 0.3616171991452575,
+      "epoch": 0.088,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00803722906857729,
+      "kl": 0.0017023698201228399,
+      "learning_rate": 4.797394529050577e-06,
+      "loss": 0.0111,
+      "num_tokens": 558509.0,
+      "reward": 0.08921875059604645,
+      "reward_std": 0.2777005434036255,
+      "rewards/alfworld_rollout_reward_func/mean": 0.08921875059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.36485835909843445,
+      "rollout/action_tokens_mean": 881.84375,
+      "rollout/episode_turns_mean": 28.40625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.859375,
+      "sampling/importance_sampling_ratio/max": 2.7719504833221436,
+      "sampling/importance_sampling_ratio/mean": 0.4752790629863739,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.038581132888794,
+      "sampling/sampling_logp_difference/mean": 0.02142297849059105,
+      "step": 11,
+      "step_time": 396.1487058620023
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.671875,
+      "completions/max_length": 2292.0,
+      "completions/max_terminated_length": 901.0,
+      "completions/mean_length": 860.28125,
+      "completions/mean_terminated_length": 460.76190185546875,
+      "completions/min_length": 116.0,
+      "completions/min_terminated_length": 116.0,
+      "entropy": 0.35516145126894116,
+      "epoch": 0.096,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.011235510930418968,
+      "kl": 0.0016268756971840048,
+      "learning_rate": 4.744511348926855e-06,
+      "loss": 0.0048,
+      "num_tokens": 613567.0,
+      "reward": 0.26890623569488525,
+      "reward_std": 0.41516801714897156,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26890626549720764,
+      "rewards/alfworld_rollout_reward_func/std": 0.4936746656894684,
+      "rollout/action_tokens_mean": 860.28125,
+      "rollout/episode_turns_mean": 25.484375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.671875,
+      "sampling/importance_sampling_ratio/max": 2.7596683502197266,
+      "sampling/importance_sampling_ratio/mean": 0.5103189945220947,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.6376895904541016,
+      "sampling/sampling_logp_difference/mean": 0.02156790718436241,
+      "step": 12,
+      "step_time": 421.574981908002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 1382.0,
+      "completions/max_terminated_length": 775.0,
+      "completions/mean_length": 836.53125,
+      "completions/mean_terminated_length": 586.4166870117188,
+      "completions/min_length": 402.0,
+      "completions/min_terminated_length": 402.0,
+      "entropy": 0.35410969890654087,
+      "epoch": 0.104,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010335148312151432,
+      "kl": 0.0018077169770549517,
+      "learning_rate": 4.685866540361456e-06,
+      "loss": 0.0068,
+      "num_tokens": 667105.0,
+      "reward": 0.14281249046325684,
+      "reward_std": 0.3198840916156769,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14281249046325684,
+      "rewards/alfworld_rollout_reward_func/std": 0.3984073996543884,
+      "rollout/action_tokens_mean": 836.53125,
+      "rollout/episode_turns_mean": 28.453125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.8125,
+      "sampling/importance_sampling_ratio/max": 2.6870784759521484,
+      "sampling/importance_sampling_ratio/mean": 0.6008636951446533,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.14093017578125,
+      "sampling/sampling_logp_difference/mean": 0.021617209538817406,
+      "step": 13,
+      "step_time": 369.507772967002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6875,
+      "completions/max_length": 1584.0,
+      "completions/max_terminated_length": 1063.0,
+      "completions/mean_length": 793.703125,
+      "completions/mean_terminated_length": 525.4500122070312,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
+      "entropy": 0.34002805408090353,
+      "epoch": 0.112,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00711849657818675,
+      "kl": 0.0019163258293701801,
+      "learning_rate": 4.621610643736878e-06,
+      "loss": 0.0062,
+      "num_tokens": 717902.0,
+      "reward": 0.2626562714576721,
+      "reward_std": 0.3680095076560974,
+      "rewards/alfworld_rollout_reward_func/mean": 0.26265624165534973,
+      "rewards/alfworld_rollout_reward_func/std": 0.489940881729126,
+      "rollout/action_tokens_mean": 793.703125,
+      "rollout/episode_turns_mean": 26.28125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.6875,
+      "sampling/importance_sampling_ratio/max": 2.815065860748291,
+      "sampling/importance_sampling_ratio/mean": 0.45154738426208496,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.7529265880584717,
+      "sampling/sampling_logp_difference/mean": 0.0205212514847517,
+      "step": 14,
+      "step_time": 327.6827599140015
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.84375,
+      "completions/max_length": 1814.0,
+      "completions/max_terminated_length": 890.0,
+      "completions/mean_length": 909.109375,
+      "completions/mean_terminated_length": 528.7999877929688,
+      "completions/min_length": 242.0,
+      "completions/min_terminated_length": 242.0,
+      "entropy": 0.35556588135659695,
+      "epoch": 0.12,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010462122038006783,
+      "kl": 0.0016676104332873365,
+      "learning_rate": 4.551908603018191e-06,
+      "loss": -0.0076,
+      "num_tokens": 776085.0,
+      "reward": 0.10140626132488251,
+      "reward_std": 0.36953580379486084,
+      "rewards/alfworld_rollout_reward_func/mean": 0.10140625387430191,
+      "rewards/alfworld_rollout_reward_func/std": 0.38891327381134033,
+      "rollout/action_tokens_mean": 909.109375,
+      "rollout/episode_turns_mean": 28.0,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.84375,
+      "sampling/importance_sampling_ratio/max": 2.0437681674957275,
+      "sampling/importance_sampling_ratio/mean": 0.3665631115436554,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.642125606536865,
+      "sampling/sampling_logp_difference/mean": 0.021498428657650948,
+      "step": 15,
+      "step_time": 409.45475878100024
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.703125,
+      "completions/max_length": 2069.0,
+      "completions/max_terminated_length": 815.0,
+      "completions/mean_length": 858.484375,
+      "completions/mean_terminated_length": 363.0,
+      "completions/min_length": 128.0,
+      "completions/min_terminated_length": 128.0,
+      "entropy": 0.3426384939812124,
+      "epoch": 0.128,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007739651948213577,
+      "kl": 0.0015719706807431066,
+      "learning_rate": 4.476939342344246e-06,
+      "loss": -0.002,
+      "num_tokens": 831028.0,
+      "reward": 0.2515625059604645,
+      "reward_std": 0.4047408103942871,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2515625059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.48498478531837463,
+      "rollout/action_tokens_mean": 858.484375,
+      "rollout/episode_turns_mean": 25.015625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.703125,
+      "sampling/importance_sampling_ratio/max": 2.197270631790161,
+      "sampling/importance_sampling_ratio/mean": 0.3564271330833435,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.3733601570129395,
+      "sampling/sampling_logp_difference/mean": 0.021193569526076317,
+      "step": 16,
+      "step_time": 341.3529842470016
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1344.0,
+      "completions/max_terminated_length": 872.0,
+      "completions/mean_length": 800.890625,
+      "completions/mean_terminated_length": 410.0000305175781,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
+      "entropy": 0.3335589300841093,
+      "epoch": 0.136,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.4890336990356445,
+      "kl": 0.8330444995917787,
+      "learning_rate": 4.396895306731978e-06,
+      "loss": 0.0005,
+      "num_tokens": 882285.0,
+      "reward": 0.18125000596046448,
+      "reward_std": 0.24295468628406525,
+      "rewards/alfworld_rollout_reward_func/mean": 0.18124999105930328,
+      "rewards/alfworld_rollout_reward_func/std": 0.4482647478580475,
+      "rollout/action_tokens_mean": 800.890625,
+      "rollout/episode_turns_mean": 26.265625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.3350934982299805,
+      "sampling/importance_sampling_ratio/mean": 0.37133079767227173,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 10.240796089172363,
+      "sampling/sampling_logp_difference/mean": 0.020599324256181717,
+      "step": 17,
+      "step_time": 436.20165634099976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.890625,
+      "completions/max_length": 1359.0,
+      "completions/max_terminated_length": 872.0,
+      "completions/mean_length": 864.640625,
+      "completions/mean_terminated_length": 638.7142944335938,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 0.3253911091014743,
+      "epoch": 0.144,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007197697646915913,
+      "kl": 0.0016915635187615408,
+      "learning_rate": 4.3119819680728e-06,
+      "loss": -0.0099,
+      "num_tokens": 937622.0,
+      "reward": 0.06968750059604645,
+      "reward_std": 0.19933855533599854,
+      "rewards/alfworld_rollout_reward_func/mean": 0.06968750059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.3224163353443146,
+      "rollout/action_tokens_mean": 864.640625,
+      "rollout/episode_turns_mean": 28.9375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.890625,
+      "sampling/importance_sampling_ratio/max": 2.2717995643615723,
+      "sampling/importance_sampling_ratio/mean": 0.5041627287864685,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.0254077911376953,
+      "sampling/sampling_logp_difference/mean": 0.019858187064528465,
+      "step": 18,
+      "step_time": 437.1889309570033
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.703125,
+      "completions/max_length": 2201.0,
+      "completions/max_terminated_length": 1115.0,
+      "completions/mean_length": 844.5,
+      "completions/mean_terminated_length": 540.631591796875,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
+      "entropy": 0.33306339103728533,
+      "epoch": 0.152,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010224551893770695,
+      "kl": 0.001574573227117071,
+      "learning_rate": 4.222417297689217e-06,
+      "loss": 0.0213,
+      "num_tokens": 991670.0,
+      "reward": 0.24437499046325684,
+      "reward_std": 0.42425790429115295,
+      "rewards/alfworld_rollout_reward_func/mean": 0.24437500536441803,
+      "rewards/alfworld_rollout_reward_func/std": 0.48038968443870544,
+      "rollout/action_tokens_mean": 844.5,
+      "rollout/episode_turns_mean": 26.171875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.703125,
+      "sampling/importance_sampling_ratio/max": 2.5866730213165283,
+      "sampling/importance_sampling_ratio/mean": 0.5376668572425842,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.632045865058899,
+      "sampling/sampling_logp_difference/mean": 0.01941317319869995,
+      "step": 19,
+      "step_time": 348.9997736020032
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1764.0,
+      "completions/max_terminated_length": 1085.0,
+      "completions/mean_length": 937.640625,
+      "completions/mean_terminated_length": 530.6875,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
+      "entropy": 0.3477782104164362,
+      "epoch": 0.16,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010612000711262226,
+      "kl": 0.001602184345756541,
+      "learning_rate": 4.128431206805556e-06,
+      "loss": 0.0044,
+      "num_tokens": 1051679.0,
+      "reward": 0.1901562511920929,
+      "reward_std": 0.39348262548446655,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1901562511920929,
+      "rewards/alfworld_rollout_reward_func/std": 0.4530303478240967,
+      "rollout/action_tokens_mean": 937.640625,
+      "rollout/episode_turns_mean": 26.6875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.5002288818359375,
+      "sampling/importance_sampling_ratio/mean": 0.4413934350013733,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.34731388092041,
+      "sampling/sampling_logp_difference/mean": 0.0198683924973011,
+      "step": 20,
+      "step_time": 353.7044870629943
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.65625,
+      "completions/max_length": 1475.0,
+      "completions/max_terminated_length": 939.0,
+      "completions/mean_length": 800.453125,
+      "completions/mean_terminated_length": 418.0909118652344,
+      "completions/min_length": 99.0,
+      "completions/min_terminated_length": 99.0,
+      "entropy": 0.36278699059039354,
+      "epoch": 0.168,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010538731701672077,
+      "kl": 0.001873640143458033,
+      "learning_rate": 4.030264956369158e-06,
+      "loss": -0.0018,
+      "num_tokens": 1102908.0,
+      "reward": 0.2993749976158142,
+      "reward_std": 0.40504124760627747,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2993749976158142,
+      "rewards/alfworld_rollout_reward_func/std": 0.4982314109802246,
+      "rollout/action_tokens_mean": 800.453125,
+      "rollout/episode_turns_mean": 24.4375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.65625,
+      "sampling/importance_sampling_ratio/max": 2.447692632675171,
+      "sampling/importance_sampling_ratio/mean": 0.448123574256897,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2475838661193848,
+      "sampling/sampling_logp_difference/mean": 0.022293493151664734,
+      "step": 21,
+      "step_time": 452.2219315199982
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.625,
+      "completions/max_length": 1566.0,
+      "completions/max_terminated_length": 929.0,
+      "completions/mean_length": 743.703125,
+      "completions/mean_terminated_length": 460.91668701171875,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
+      "entropy": 0.3329707160592079,
+      "epoch": 0.176,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007643942255526781,
+      "kl": 0.0019213874111301266,
+      "learning_rate": 3.9281705377369814e-06,
+      "loss": 0.0059,
+      "num_tokens": 1150505.0,
+      "reward": 0.32625001668930054,
+      "reward_std": 0.4726526141166687,
+      "rewards/alfworld_rollout_reward_func/mean": 0.32624998688697815,
+      "rewards/alfworld_rollout_reward_func/std": 0.5121089816093445,
+      "rollout/action_tokens_mean": 743.703125,
+      "rollout/episode_turns_mean": 24.796875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.625,
+      "sampling/importance_sampling_ratio/max": 2.206357479095459,
+      "sampling/importance_sampling_ratio/mean": 0.4393410086631775,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.366598129272461,
+      "sampling/sampling_logp_difference/mean": 0.020861441269516945,
+      "step": 22,
+      "step_time": 350.210106904995
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.65625,
+      "completions/max_length": 1494.0,
+      "completions/max_terminated_length": 1159.0,
+      "completions/mean_length": 797.828125,
+      "completions/mean_terminated_length": 504.04547119140625,
+      "completions/min_length": 87.0,
+      "completions/min_terminated_length": 87.0,
+      "entropy": 0.35823060059919953,
+      "epoch": 0.184,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010937543585896492,
+      "kl": 0.00202080328745069,
+      "learning_rate": 3.8224100258174066e-06,
+      "loss": 0.0064,
+      "num_tokens": 1201566.0,
+      "reward": 0.3057812452316284,
+      "reward_std": 0.36121153831481934,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3057812452316284,
+      "rewards/alfworld_rollout_reward_func/std": 0.49343568086624146,
+      "rollout/action_tokens_mean": 797.828125,
+      "rollout/episode_turns_mean": 25.328125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.65625,
+      "sampling/importance_sampling_ratio/max": 2.8112552165985107,
+      "sampling/importance_sampling_ratio/mean": 0.43328696489334106,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2014861106872559,
+      "sampling/sampling_logp_difference/mean": 0.022007007151842117,
+      "step": 23,
+      "step_time": 299.4183257309978
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1409.0,
+      "completions/max_terminated_length": 962.0,
+      "completions/mean_length": 749.84375,
+      "completions/mean_terminated_length": 406.0,
+      "completions/min_length": 135.0,
+      "completions/min_terminated_length": 135.0,
+      "entropy": 0.314591265283525,
+      "epoch": 0.192,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.008087356574833393,
+      "kl": 0.0017565299822308589,
+      "learning_rate": 3.7132549063277033e-06,
+      "loss": -0.0022,
+      "num_tokens": 1249556.0,
+      "reward": 0.20937500894069672,
+      "reward_std": 0.2735254764556885,
+      "rewards/alfworld_rollout_reward_func/mean": 0.20937500894069672,
+      "rewards/alfworld_rollout_reward_func/std": 0.45049840211868286,
+      "rollout/action_tokens_mean": 749.84375,
+      "rollout/episode_turns_mean": 25.53125,
+      "rollout/mismatch_mean": 0.03125,
+      "rollout/mismatch_ratio": 0.03125,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.822704792022705,
+      "sampling/importance_sampling_ratio/mean": 0.4837469756603241,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2067627906799316,
+      "sampling/sampling_logp_difference/mean": 0.021238919347524643,
+      "step": 24,
+      "step_time": 423.74345705999986
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.828125,
+      "completions/max_length": 1469.0,
+      "completions/max_terminated_length": 854.0,
+      "completions/mean_length": 757.296875,
+      "completions/mean_terminated_length": 445.2727355957031,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
+      "entropy": 0.3146928045898676,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.005950183607637882,
+      "kl": 0.0024588965916336747,
+      "learning_rate": 3.600985378894086e-06,
+      "loss": -0.0029,
+      "num_tokens": 1298023.0,
+      "reward": 0.12843748927116394,
+      "reward_std": 0.3103031814098358,
+      "rewards/alfworld_rollout_reward_func/mean": 0.12843750417232513,
+      "rewards/alfworld_rollout_reward_func/std": 0.39419689774513245,
+      "rollout/action_tokens_mean": 757.296875,
+      "rollout/episode_turns_mean": 27.515625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.828125,
+      "sampling/importance_sampling_ratio/max": 1.9042965173721313,
+      "sampling/importance_sampling_ratio/mean": 0.30797556042671204,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.0986688137054443,
+      "sampling/sampling_logp_difference/mean": 0.0199856199324131,
+      "step": 25,
+      "step_time": 379.6568018289963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.796875,
+      "completions/max_length": 1607.0,
+      "completions/max_terminated_length": 975.0,
+      "completions/mean_length": 820.5,
+      "completions/mean_terminated_length": 396.923095703125,
+      "completions/min_length": 174.0,
+      "completions/min_terminated_length": 174.0,
+      "entropy": 0.301429130602628,
+      "epoch": 0.208,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006396389566361904,
+      "kl": 0.001548040645502624,
+      "learning_rate": 3.4858896377832966e-06,
+      "loss": 0.0009,
+      "num_tokens": 1350535.0,
+      "reward": 0.1639062464237213,
+      "reward_std": 0.32607388496398926,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1639062464237213,
+      "rewards/alfworld_rollout_reward_func/std": 0.418832927942276,
+      "rollout/action_tokens_mean": 820.5,
+      "rollout/episode_turns_mean": 26.59375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.796875,
+      "sampling/importance_sampling_ratio/max": 2.7253878116607666,
+      "sampling/importance_sampling_ratio/mean": 0.46614933013916016,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.6588082313537598,
+      "sampling/sampling_logp_difference/mean": 0.018608851358294487,
+      "step": 26,
+      "step_time": 458.5120370709956
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 1363.0,
+      "completions/max_terminated_length": 851.0,
+      "completions/mean_length": 843.46875,
+      "completions/mean_terminated_length": 518.0833740234375,
+      "completions/min_length": 123.0,
+      "completions/min_terminated_length": 123.0,
+      "entropy": 0.3490098095498979,
+      "epoch": 0.216,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00987387914210558,
+      "kl": 0.001589072278875392,
+      "learning_rate": 3.3682631321120507e-06,
+      "loss": -0.0109,
+      "num_tokens": 1404517.0,
+      "reward": 0.14921875298023224,
+      "reward_std": 0.3126556873321533,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14921875298023224,
+      "rewards/alfworld_rollout_reward_func/std": 0.40770089626312256,
+      "rollout/action_tokens_mean": 843.46875,
+      "rollout/episode_turns_mean": 27.703125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.8125,
+      "sampling/importance_sampling_ratio/max": 2.323033332824707,
+      "sampling/importance_sampling_ratio/mean": 0.4162726402282715,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.8558279275894165,
+      "sampling/sampling_logp_difference/mean": 0.020594514906406403,
+      "step": 27,
+      "step_time": 366.20077905700964
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5625,
+      "completions/max_length": 2369.0,
+      "completions/max_terminated_length": 1151.0,
+      "completions/mean_length": 727.859375,
+      "completions/mean_terminated_length": 404.3571472167969,
+      "completions/min_length": 97.0,
+      "completions/min_terminated_length": 97.0,
+      "entropy": 0.30788369150832295,
+      "epoch": 0.224,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01545465737581253,
+      "kl": 0.0016130298517964548,
+      "learning_rate": 3.248407807433396e-06,
+      "loss": 0.012,
+      "num_tokens": 1451100.0,
+      "reward": 0.3981249928474426,
+      "reward_std": 0.5213440656661987,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3981249928474426,
+      "rewards/alfworld_rollout_reward_func/std": 0.5216788649559021,
+      "rollout/action_tokens_mean": 727.859375,
+      "rollout/episode_turns_mean": 22.984375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.5625,
+      "sampling/importance_sampling_ratio/max": 2.7356457710266113,
+      "sampling/importance_sampling_ratio/mean": 0.6839978694915771,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.65842604637146,
+      "sampling/sampling_logp_difference/mean": 0.019816581159830093,
+      "step": 28,
+      "step_time": 317.7198553449962
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1781.0,
+      "completions/max_terminated_length": 980.0,
+      "completions/mean_length": 844.03125,
+      "completions/mean_terminated_length": 524.75,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
+      "entropy": 0.3514281096868217,
+      "epoch": 0.232,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010329142212867737,
+      "kl": 0.0034811663499567658,
+      "learning_rate": 3.1266313306468018e-06,
+      "loss": -0.0153,
+      "num_tokens": 1505118.0,
+      "reward": 0.20656250417232513,
+      "reward_std": 0.29942071437835693,
+      "rewards/alfworld_rollout_reward_func/mean": 0.20656250417232513,
+      "rewards/alfworld_rollout_reward_func/std": 0.4480122923851013,
+      "rollout/action_tokens_mean": 844.03125,
+      "rollout/episode_turns_mean": 26.890625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.6475582122802734,
+      "sampling/importance_sampling_ratio/mean": 0.39373910427093506,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 5.319100856781006,
+      "sampling/sampling_logp_difference/mean": 0.022610869258642197,
+      "step": 29,
+      "step_time": 372.01922321400525
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1465.0,
+      "completions/max_terminated_length": 804.0,
+      "completions/mean_length": 781.265625,
+      "completions/mean_terminated_length": 451.8125,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
+      "entropy": 0.2927012937143445,
+      "epoch": 0.24,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006770053878426552,
+      "kl": 0.0015057786895340541,
+      "learning_rate": 3.0032463002216504e-06,
+      "loss": 0.0038,
+      "num_tokens": 1555119.0,
+      "reward": 0.21671874821186066,
+      "reward_std": 0.24240338802337646,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21671876311302185,
+      "rewards/alfworld_rollout_reward_func/std": 0.44675213098526,
+      "rollout/action_tokens_mean": 781.265625,
+      "rollout/episode_turns_mean": 26.21875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.8479506969451904,
+      "sampling/importance_sampling_ratio/mean": 0.49506521224975586,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.7825241088867188,
+      "sampling/sampling_logp_difference/mean": 0.01828092336654663,
+      "step": 30,
+      "step_time": 382.84014304800075
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 1591.0,
+      "completions/max_terminated_length": 899.0,
+      "completions/mean_length": 807.0625,
+      "completions/mean_terminated_length": 503.8333435058594,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
+      "entropy": 0.33275228925049305,
+      "epoch": 0.248,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.009322401136159897,
+      "kl": 0.0018803979564836482,
+      "learning_rate": 2.878569443761442e-06,
+      "loss": -0.0078,
+      "num_tokens": 1606771.0,
+      "reward": 0.14656250178813934,
+      "reward_std": 0.337169349193573,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14656248688697815,
+      "rewards/alfworld_rollout_reward_func/std": 0.4100453853607178,
+      "rollout/action_tokens_mean": 807.0625,
+      "rollout/episode_turns_mean": 27.546875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.8125,
+      "sampling/importance_sampling_ratio/max": 2.9021573066711426,
+      "sampling/importance_sampling_ratio/mean": 0.42428821325302124,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.855207920074463,
+      "sampling/sampling_logp_difference/mean": 0.021321609616279602,
+      "step": 31,
+      "step_time": 494.6054373920015
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.796875,
+      "completions/max_length": 1383.0,
+      "completions/max_terminated_length": 1038.0,
+      "completions/mean_length": 791.734375,
+      "completions/mean_terminated_length": 412.923095703125,
+      "completions/min_length": 125.0,
+      "completions/min_terminated_length": 125.0,
+      "entropy": 0.38143961410969496,
+      "epoch": 0.256,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010985196568071842,
+      "kl": 0.0017831376208050642,
+      "learning_rate": 2.752920804968581e-06,
+      "loss": -0.0093,
+      "num_tokens": 1657442.0,
+      "reward": 0.14656248688697815,
+      "reward_std": 0.27167972922325134,
+      "rewards/alfworld_rollout_reward_func/mean": 0.14656250178813934,
+      "rewards/alfworld_rollout_reward_func/std": 0.4263809323310852,
+      "rollout/action_tokens_mean": 791.734375,
+      "rollout/episode_turns_mean": 26.5,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.796875,
+      "sampling/importance_sampling_ratio/max": 2.170595169067383,
+      "sampling/importance_sampling_ratio/mean": 0.39966434240341187,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.398648977279663,
+      "sampling/sampling_logp_difference/mean": 0.022872522473335266,
+      "step": 32,
+      "step_time": 305.8924537160019
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.734375,
+      "completions/max_length": 1564.0,
+      "completions/max_terminated_length": 1151.0,
+      "completions/mean_length": 914.953125,
+      "completions/mean_terminated_length": 616.4705810546875,
+      "completions/min_length": 291.0,
+      "completions/min_terminated_length": 291.0,
+      "entropy": 0.39743291586637497,
+      "epoch": 0.264,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010168999433517456,
+      "kl": 0.0017962153833650518,
+      "learning_rate": 2.626622922096782e-06,
+      "loss": -0.0045,
+      "num_tokens": 1715999.0,
+      "reward": 0.21843750774860382,
+      "reward_std": 0.4097728729248047,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21843750774860382,
+      "rewards/alfworld_rollout_reward_func/std": 0.46011462807655334,
+      "rollout/action_tokens_mean": 914.953125,
+      "rollout/episode_turns_mean": 27.140625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.734375,
+      "sampling/importance_sampling_ratio/max": 2.6107707023620605,
+      "sampling/importance_sampling_ratio/mean": 0.386021226644516,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2772090435028076,
+      "sampling/sampling_logp_difference/mean": 0.02191011980175972,
+      "step": 33,
+      "step_time": 360.4347860389935
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.796875,
+      "completions/max_length": 1527.0,
+      "completions/max_terminated_length": 743.0,
+      "completions/mean_length": 862.9375,
+      "completions/mean_terminated_length": 506.3077087402344,
+      "completions/min_length": 254.0,
+      "completions/min_terminated_length": 254.0,
+      "entropy": 0.35848523303866386,
+      "epoch": 0.272,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01134358998388052,
+      "kl": 0.001760420509526739,
+      "learning_rate": 2.5e-06,
+      "loss": 0.016,
+      "num_tokens": 1771227.0,
+      "reward": 0.15781250596046448,
+      "reward_std": 0.29862138628959656,
+      "rewards/alfworld_rollout_reward_func/mean": 0.15781249105930328,
+      "rewards/alfworld_rollout_reward_func/std": 0.4202209413051605,
+      "rollout/action_tokens_mean": 862.9375,
+      "rollout/episode_turns_mean": 27.515625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.796875,
+      "sampling/importance_sampling_ratio/max": 2.3946166038513184,
+      "sampling/importance_sampling_ratio/mean": 0.41097065806388855,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.3659703731536865,
+      "sampling/sampling_logp_difference/mean": 0.020839743316173553,
+      "step": 34,
+      "step_time": 255.86244797499785
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.640625,
+      "completions/max_length": 1503.0,
+      "completions/max_terminated_length": 1127.0,
+      "completions/mean_length": 801.078125,
+      "completions/mean_terminated_length": 446.0434875488281,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
+      "entropy": 0.33974334690719843,
+      "epoch": 0.28,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.007748698350042105,
+      "kl": 0.0017098757416533772,
+      "learning_rate": 2.3733770779032185e-06,
+      "loss": 0.0012,
+      "num_tokens": 1822496.0,
+      "reward": 0.3109375238418579,
+      "reward_std": 0.34154438972473145,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3109375238418579,
+      "rewards/alfworld_rollout_reward_func/std": 0.5012863874435425,
+      "rollout/action_tokens_mean": 801.078125,
+      "rollout/episode_turns_mean": 24.203125,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.640625,
+      "sampling/importance_sampling_ratio/max": 2.6798601150512695,
+      "sampling/importance_sampling_ratio/mean": 0.4648974537849426,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.4488577842712402,
+      "sampling/sampling_logp_difference/mean": 0.020977256819605827,
+      "step": 35,
+      "step_time": 369.7442883429976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5625,
+      "completions/max_length": 1817.0,
+      "completions/max_terminated_length": 1098.0,
+      "completions/mean_length": 763.078125,
+      "completions/mean_terminated_length": 383.21429443359375,
+      "completions/min_length": 90.0,
+      "completions/min_terminated_length": 90.0,
+      "entropy": 0.37786182574927807,
+      "epoch": 0.288,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.012169571593403816,
+      "kl": 0.0024242461659014225,
+      "learning_rate": 2.24707919503142e-06,
+      "loss": 0.0067,
+      "num_tokens": 1871333.0,
+      "reward": 0.3814062476158142,
+      "reward_std": 0.325703501701355,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3814062476158142,
+      "rewards/alfworld_rollout_reward_func/std": 0.5243707299232483,
+      "rollout/action_tokens_mean": 763.078125,
+      "rollout/episode_turns_mean": 22.328125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.5625,
+      "sampling/importance_sampling_ratio/max": 2.630308151245117,
+      "sampling/importance_sampling_ratio/mean": 0.4907624125480652,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.9331345558166504,
+      "sampling/sampling_logp_difference/mean": 0.022463176399469376,
+      "step": 36,
+      "step_time": 452.8674500970046
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.453125,
+      "completions/max_length": 1560.0,
+      "completions/max_terminated_length": 1173.0,
+      "completions/mean_length": 592.9375,
+      "completions/mean_terminated_length": 322.4285583496094,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
+      "entropy": 0.2960165496915579,
+      "epoch": 0.296,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.009033096954226494,
+      "kl": 0.0017318885384156602,
+      "learning_rate": 2.1214305562385592e-06,
+      "loss": 0.0028,
+      "num_tokens": 1909281.0,
+      "reward": 0.5143750309944153,
+      "reward_std": 0.3860568404197693,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5143750309944153,
+      "rewards/alfworld_rollout_reward_func/std": 0.5290928483009338,
+      "rollout/action_tokens_mean": 592.9375,
+      "rollout/episode_turns_mean": 19.015625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.453125,
+      "sampling/importance_sampling_ratio/max": 2.8567276000976562,
+      "sampling/importance_sampling_ratio/mean": 0.5531047582626343,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.304647445678711,
+      "sampling/sampling_logp_difference/mean": 0.01983683556318283,
+      "step": 37,
+      "step_time": 388.3879080370025
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.53125,
+      "completions/max_length": 1343.0,
+      "completions/max_terminated_length": 792.0,
+      "completions/mean_length": 662.140625,
+      "completions/mean_terminated_length": 343.8333435058594,
+      "completions/min_length": 77.0,
+      "completions/min_terminated_length": 77.0,
+      "entropy": 0.32609579246491194,
+      "epoch": 0.304,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.014773277565836906,
+      "kl": 0.0018760439170364407,
+      "learning_rate": 1.9967536997783495e-06,
+      "loss": 0.0014,
+      "num_tokens": 1951658.0,
+      "reward": 0.4350000023841858,
+      "reward_std": 0.3387959599494934,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4350000023841858,
+      "rewards/alfworld_rollout_reward_func/std": 0.5278016924858093,
+      "rollout/action_tokens_mean": 662.140625,
+      "rollout/episode_turns_mean": 21.40625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.53125,
+      "sampling/importance_sampling_ratio/max": 2.9260754585266113,
+      "sampling/importance_sampling_ratio/mean": 0.563366711139679,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.768939971923828,
+      "sampling/sampling_logp_difference/mean": 0.022423237562179565,
+      "step": 38,
+      "step_time": 436.1079989810023
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.734375,
+      "completions/max_length": 1869.0,
+      "completions/max_terminated_length": 1018.0,
+      "completions/mean_length": 943.40625,
+      "completions/mean_terminated_length": 584.4117431640625,
+      "completions/min_length": 237.0,
+      "completions/min_terminated_length": 237.0,
+      "entropy": 0.3758079996332526,
+      "epoch": 0.312,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01313227228820324,
+      "kl": 0.0022639515991613735,
+      "learning_rate": 1.8733686693531986e-06,
+      "loss": -0.0025,
+      "num_tokens": 2012036.0,
+      "reward": 0.21578124165534973,
+      "reward_std": 0.4323212504386902,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21578124165534973,
+      "rewards/alfworld_rollout_reward_func/std": 0.46428921818733215,
+      "rollout/action_tokens_mean": 943.40625,
+      "rollout/episode_turns_mean": 26.90625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.734375,
+      "sampling/importance_sampling_ratio/max": 2.5490753650665283,
+      "sampling/importance_sampling_ratio/mean": 0.3670840859413147,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.389880657196045,
+      "sampling/sampling_logp_difference/mean": 0.021743088960647583,
+      "step": 39,
+      "step_time": 354.37862886100265
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.671875,
+      "completions/max_length": 1334.0,
+      "completions/max_terminated_length": 1062.0,
+      "completions/mean_length": 701.15625,
+      "completions/mean_terminated_length": 308.1428527832031,
+      "completions/min_length": 155.0,
+      "completions/min_terminated_length": 155.0,
+      "entropy": 0.2994346688501537,
+      "epoch": 0.32,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.009722444228827953,
+      "kl": 0.0016241020402958384,
+      "learning_rate": 1.7515921925666053e-06,
+      "loss": -0.0206,
+      "num_tokens": 2056910.0,
+      "reward": 0.29640626907348633,
+      "reward_std": 0.23870769143104553,
+      "rewards/alfworld_rollout_reward_func/mean": 0.29640626907348633,
+      "rewards/alfworld_rollout_reward_func/std": 0.4918813109397888,
+      "rollout/action_tokens_mean": 701.15625,
+      "rollout/episode_turns_mean": 23.625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.671875,
+      "sampling/importance_sampling_ratio/max": 2.825843572616577,
+      "sampling/importance_sampling_ratio/mean": 0.5806776881217957,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3793482780456543,
+      "sampling/sampling_logp_difference/mean": 0.01931983232498169,
+      "step": 40,
+      "step_time": 355.1650720970065
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.71875,
+      "completions/max_length": 1877.0,
+      "completions/max_terminated_length": 997.0,
+      "completions/mean_length": 860.609375,
+      "completions/mean_terminated_length": 526.0,
+      "completions/min_length": 117.0,
+      "completions/min_terminated_length": 117.0,
+      "entropy": 0.33433423191308975,
+      "epoch": 0.328,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01665978506207466,
+      "kl": 0.001694458733254578,
+      "learning_rate": 1.6317368678879497e-06,
+      "loss": 0.0013,
+      "num_tokens": 2111989.0,
+      "reward": 0.23156249523162842,
+      "reward_std": 0.42284566164016724,
+      "rewards/alfworld_rollout_reward_func/mean": 0.23156249523162842,
+      "rewards/alfworld_rollout_reward_func/std": 0.4703366458415985,
+      "rollout/action_tokens_mean": 860.609375,
+      "rollout/episode_turns_mean": 26.859375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.71875,
+      "sampling/importance_sampling_ratio/max": 2.746248960494995,
+      "sampling/importance_sampling_ratio/mean": 0.47027722001075745,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.0510313510894775,
+      "sampling/sampling_logp_difference/mean": 0.019619014114141464,
+      "step": 41,
+      "step_time": 383.9065069159915
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.578125,
+      "completions/max_length": 1491.0,
+      "completions/max_terminated_length": 1206.0,
+      "completions/mean_length": 699.109375,
+      "completions/mean_terminated_length": 410.9259338378906,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
+      "entropy": 0.3300422574393451,
+      "epoch": 0.336,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.011248192749917507,
+      "kl": 0.0020856883202213794,
+      "learning_rate": 1.5141103622167042e-06,
+      "loss": 0.0075,
+      "num_tokens": 2156732.0,
+      "reward": 0.37812501192092896,
+      "reward_std": 0.3792918920516968,
+      "rewards/alfworld_rollout_reward_func/mean": 0.37812501192092896,
+      "rewards/alfworld_rollout_reward_func/std": 0.5151386857032776,
+      "rollout/action_tokens_mean": 699.109375,
+      "rollout/episode_turns_mean": 23.015625,
+      "rollout/mismatch_mean": 0.03125,
+      "rollout/mismatch_ratio": 0.03125,
+      "rollout/truncated_ratio": 0.578125,
+      "sampling/importance_sampling_ratio/max": 2.8479621410369873,
+      "sampling/importance_sampling_ratio/mean": 0.49075883626937866,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.1700198650360107,
+      "sampling/sampling_logp_difference/mean": 0.02142202854156494,
+      "step": 42,
+      "step_time": 342.9056702470225
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.71875,
+      "completions/max_length": 1746.0,
+      "completions/max_terminated_length": 864.0,
+      "completions/mean_length": 780.5,
+      "completions/mean_terminated_length": 496.6111145019531,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
+      "entropy": 0.31365567492321134,
+      "epoch": 0.344,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.013282958418130875,
+      "kl": 0.0016229442680923967,
+      "learning_rate": 1.3990146211059141e-06,
+      "loss": -0.0022,
+      "num_tokens": 2206684.0,
+      "reward": 0.2423437535762787,
+      "reward_std": 0.44438114762306213,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2423437535762787,
+      "rewards/alfworld_rollout_reward_func/std": 0.47146087884902954,
+      "rollout/action_tokens_mean": 780.5,
+      "rollout/episode_turns_mean": 26.484375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.71875,
+      "sampling/importance_sampling_ratio/max": 2.7073540687561035,
+      "sampling/importance_sampling_ratio/mean": 0.548378586769104,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.89083194732666,
+      "sampling/sampling_logp_difference/mean": 0.020154234021902084,
+      "step": 43,
+      "step_time": 465.0683610840024
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 2028.0,
+      "completions/max_terminated_length": 1179.0,
+      "completions/mean_length": 882.0625,
+      "completions/mean_terminated_length": 563.0,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
+      "entropy": 0.30738420970737934,
+      "epoch": 0.352,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.011168000288307667,
+      "kl": 0.001302105893046246,
+      "learning_rate": 1.286745093672298e-06,
+      "loss": 0.0006,
+      "num_tokens": 2263136.0,
+      "reward": 0.18671876192092896,
+      "reward_std": 0.23547054827213287,
+      "rewards/alfworld_rollout_reward_func/mean": 0.18671876192092896,
+      "rewards/alfworld_rollout_reward_func/std": 0.4461868703365326,
+      "rollout/action_tokens_mean": 882.0625,
+      "rollout/episode_turns_mean": 26.578125,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.5664165019989014,
+      "sampling/importance_sampling_ratio/mean": 0.4952065646648407,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 8.617661476135254,
+      "sampling/sampling_logp_difference/mean": 0.019173569977283478,
+      "step": 44,
+      "step_time": 361.35297568800524
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.65625,
+      "completions/max_length": 1390.0,
+      "completions/max_terminated_length": 1011.0,
+      "completions/mean_length": 807.96875,
+      "completions/mean_terminated_length": 565.9091186523438,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
+      "entropy": 0.33469977881759405,
+      "epoch": 0.36,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.005941484589129686,
+      "kl": 0.001596126568983891,
+      "learning_rate": 1.1775899741825947e-06,
+      "loss": 0.0031,
+      "num_tokens": 2314846.0,
+      "reward": 0.2996875047683716,
+      "reward_std": 0.383698046207428,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2996875047683716,
+      "rewards/alfworld_rollout_reward_func/std": 0.48867806792259216,
+      "rollout/action_tokens_mean": 807.96875,
+      "rollout/episode_turns_mean": 25.65625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.65625,
+      "sampling/importance_sampling_ratio/max": 1.910278081893921,
+      "sampling/importance_sampling_ratio/mean": 0.4718725383281708,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.8846027851104736,
+      "sampling/sampling_logp_difference/mean": 0.021218668669462204,
+      "step": 45,
+      "step_time": 477.9487361019965
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.609375,
+      "completions/max_length": 1381.0,
+      "completions/max_terminated_length": 827.0,
+      "completions/mean_length": 659.234375,
+      "completions/mean_terminated_length": 271.7200012207031,
+      "completions/min_length": 106.0,
+      "completions/min_terminated_length": 106.0,
+      "entropy": 0.3191372868604958,
+      "epoch": 0.368,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.01500615943223238,
+      "kl": 0.0017743182979756966,
+      "learning_rate": 1.0718294622630188e-06,
+      "loss": -0.007,
+      "num_tokens": 2357037.0,
+      "reward": 0.3592187464237213,
+      "reward_std": 0.3250337839126587,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3592187464237213,
+      "rewards/alfworld_rollout_reward_func/std": 0.5135136246681213,
+      "rollout/action_tokens_mean": 659.234375,
+      "rollout/episode_turns_mean": 21.640625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.609375,
+      "sampling/importance_sampling_ratio/max": 2.703761100769043,
+      "sampling/importance_sampling_ratio/mean": 0.6121923923492432,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.5571508407592773,
+      "sampling/sampling_logp_difference/mean": 0.02017129212617874,
+      "step": 46,
+      "step_time": 368.9170330479901
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.46875,
+      "completions/max_length": 1449.0,
+      "completions/max_terminated_length": 879.0,
+      "completions/mean_length": 630.171875,
+      "completions/mean_terminated_length": 346.20587158203125,
+      "completions/min_length": 75.0,
+      "completions/min_terminated_length": 75.0,
+      "entropy": 0.33235412929207087,
+      "epoch": 0.376,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.014015787281095982,
+      "kl": 0.001809486606362043,
+      "learning_rate": 9.697350436308428e-07,
+      "loss": 0.0171,
+      "num_tokens": 2397368.0,
+      "reward": 0.4931250214576721,
+      "reward_std": 0.29200682044029236,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4931250214576721,
+      "rewards/alfworld_rollout_reward_func/std": 0.5321918725967407,
+      "rollout/action_tokens_mean": 630.171875,
+      "rollout/episode_turns_mean": 19.6875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.46875,
+      "sampling/importance_sampling_ratio/max": 2.7606446743011475,
+      "sampling/importance_sampling_ratio/mean": 0.6627721190452576,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.4474594593048096,
+      "sampling/sampling_logp_difference/mean": 0.02060038223862648,
+      "step": 47,
+      "step_time": 259.2473690390034
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.875,
+      "completions/max_length": 2126.0,
+      "completions/max_terminated_length": 1218.0,
+      "completions/mean_length": 943.59375,
+      "completions/mean_terminated_length": 424.0,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
+      "entropy": 0.3577259639278054,
+      "epoch": 0.384,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.014235694892704487,
+      "kl": 0.0015301553139579482,
+      "learning_rate": 8.71568793194445e-07,
+      "loss": 0.0326,
+      "num_tokens": 2457758.0,
+      "reward": 0.06859375536441803,
+      "reward_std": 0.24723710119724274,
+      "rewards/alfworld_rollout_reward_func/mean": 0.06859375536441803,
+      "rewards/alfworld_rollout_reward_func/std": 0.3545883595943451,
+      "rollout/action_tokens_mean": 943.59375,
+      "rollout/episode_turns_mean": 27.734375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.875,
+      "sampling/importance_sampling_ratio/max": 2.791482925415039,
+      "sampling/importance_sampling_ratio/mean": 0.5324603319168091,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.8059403896331787,
+      "sampling/sampling_logp_difference/mean": 0.020999200642108917,
+      "step": 48,
+      "step_time": 384.36283810400346
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.734375,
+      "completions/max_length": 1635.0,
+      "completions/max_terminated_length": 980.0,
+      "completions/mean_length": 861.203125,
+      "completions/mean_terminated_length": 481.4705810546875,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
+      "entropy": 0.36975698871538043,
+      "epoch": 0.392,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.006656537763774395,
+      "kl": 0.0018834842448995914,
+      "learning_rate": 7.775827023107835e-07,
+      "loss": 0.0044,
+      "num_tokens": 2512875.0,
+      "reward": 0.21015624701976776,
+      "reward_std": 0.3197978734970093,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21015626192092896,
+      "rewards/alfworld_rollout_reward_func/std": 0.4713321030139923,
+      "rollout/action_tokens_mean": 861.203125,
+      "rollout/episode_turns_mean": 26.59375,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.734375,
+      "sampling/importance_sampling_ratio/max": 2.697384834289551,
+      "sampling/importance_sampling_ratio/mean": 0.4099380373954773,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.545639991760254,
+      "sampling/sampling_logp_difference/mean": 0.02258184552192688,
+      "step": 49,
+      "step_time": 397.8871677600073
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.75,
+      "completions/max_length": 1624.0,
+      "completions/max_terminated_length": 910.0,
+      "completions/mean_length": 887.515625,
+      "completions/mean_terminated_length": 580.0625,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 0.3667739760130644,
+      "epoch": 0.4,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.013544962741434574,
+      "kl": 0.0015946346466080286,
+      "learning_rate": 6.880180319272006e-07,
+      "loss": 0.0071,
+      "num_tokens": 2569676.0,
+      "reward": 0.1901562511920929,
+      "reward_std": 0.45852425694465637,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1901562511920929,
+      "rewards/alfworld_rollout_reward_func/std": 0.4570210576057434,
+      "rollout/action_tokens_mean": 887.515625,
+      "rollout/episode_turns_mean": 26.828125,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.75,
+      "sampling/importance_sampling_ratio/max": 2.99810791015625,
+      "sampling/importance_sampling_ratio/mean": 0.5466522574424744,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.044807434082031,
+      "sampling/sampling_logp_difference/mean": 0.02207544818520546,
+      "step": 50,
+      "step_time": 363.5887358939908
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.65625,
+      "completions/max_length": 1596.0,
+      "completions/max_terminated_length": 1015.0,
+      "completions/mean_length": 773.125,
+      "completions/mean_terminated_length": 392.4090881347656,
+      "completions/min_length": 110.0,
+      "completions/min_terminated_length": 110.0,
+      "entropy": 0.3592249136418104,
+      "epoch": 0.408,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01083005964756012,
+      "kl": 0.002072367888104054,
+      "learning_rate": 6.031046932680229e-07,
+      "loss": 0.0148,
+      "num_tokens": 2619156.0,
+      "reward": 0.30000001192092896,
+      "reward_std": 0.43087929487228394,
+      "rewards/alfworld_rollout_reward_func/mean": 0.30000001192092896,
+      "rewards/alfworld_rollout_reward_func/std": 0.500076174736023,
+      "rollout/action_tokens_mean": 773.125,
+      "rollout/episode_turns_mean": 24.046875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.65625,
+      "sampling/importance_sampling_ratio/max": 2.189059257507324,
+      "sampling/importance_sampling_ratio/mean": 0.4208452105522156,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.6584203243255615,
+      "sampling/sampling_logp_difference/mean": 0.022034121677279472,
+      "step": 51,
+      "step_time": 381.66528784099137
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.59375,
+      "completions/max_length": 1738.0,
+      "completions/max_terminated_length": 1079.0,
+      "completions/mean_length": 780.3125,
+      "completions/mean_terminated_length": 445.5769348144531,
+      "completions/min_length": 98.0,
+      "completions/min_terminated_length": 98.0,
+      "entropy": 0.3415462737902999,
+      "epoch": 0.416,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.005050386302173138,
+      "kl": 0.0017878458802442765,
+      "learning_rate": 5.23060657655754e-07,
+      "loss": -0.0079,
+      "num_tokens": 2669096.0,
+      "reward": 0.3726562559604645,
+      "reward_std": 0.36754581332206726,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3726562559604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.5065045356750488,
+      "rollout/action_tokens_mean": 780.3125,
+      "rollout/episode_turns_mean": 23.90625,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.59375,
+      "sampling/importance_sampling_ratio/max": 2.1458444595336914,
+      "sampling/importance_sampling_ratio/mean": 0.4066748023033142,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.5774283409118652,
+      "sampling/sampling_logp_difference/mean": 0.022033972665667534,
+      "step": 52,
+      "step_time": 482.9236067970087
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.734375,
+      "completions/max_length": 1387.0,
+      "completions/max_terminated_length": 1161.0,
+      "completions/mean_length": 819.3125,
+      "completions/mean_terminated_length": 579.8823852539062,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
+      "entropy": 0.33714828826487064,
+      "epoch": 0.424,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010500618256628513,
+      "kl": 0.0019351580704096705,
+      "learning_rate": 4.480913969818099e-07,
+      "loss": -0.0017,
+      "num_tokens": 2721532.0,
+      "reward": 0.21718750894069672,
+      "reward_std": 0.38691049814224243,
+      "rewards/alfworld_rollout_reward_func/mean": 0.21718750894069672,
+      "rewards/alfworld_rollout_reward_func/std": 0.4590102434158325,
+      "rollout/action_tokens_mean": 819.3125,
+      "rollout/episode_turns_mean": 27.015625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.734375,
+      "sampling/importance_sampling_ratio/max": 2.4449539184570312,
+      "sampling/importance_sampling_ratio/mean": 0.38772356510162354,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.6585437059402466,
+      "sampling/sampling_logp_difference/mean": 0.020567741245031357,
+      "step": 53,
+      "step_time": 440.7185651119944
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.765625,
+      "completions/max_length": 1538.0,
+      "completions/max_terminated_length": 1077.0,
+      "completions/mean_length": 858.40625,
+      "completions/mean_terminated_length": 563.4666748046875,
+      "completions/min_length": 196.0,
+      "completions/min_terminated_length": 196.0,
+      "entropy": 0.3832448711618781,
+      "epoch": 0.432,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00624613743275404,
+      "kl": 0.0019235094769101124,
+      "learning_rate": 3.7838935626312246e-07,
+      "loss": -0.003,
+      "num_tokens": 2776470.0,
+      "reward": 0.18359375,
+      "reward_std": 0.32859885692596436,
+      "rewards/alfworld_rollout_reward_func/mean": 0.18359375,
+      "rewards/alfworld_rollout_reward_func/std": 0.44527989625930786,
+      "rollout/action_tokens_mean": 858.40625,
+      "rollout/episode_turns_mean": 26.546875,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.765625,
+      "sampling/importance_sampling_ratio/max": 2.0156307220458984,
+      "sampling/importance_sampling_ratio/mean": 0.2892346978187561,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.6584253311157227,
+      "sampling/sampling_logp_difference/mean": 0.022003933787345886,
+      "step": 54,
+      "step_time": 433.99726870600716
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.640625,
+      "completions/max_length": 1862.0,
+      "completions/max_terminated_length": 1001.0,
+      "completions/mean_length": 774.25,
+      "completions/mean_terminated_length": 319.9565124511719,
+      "completions/min_length": 103.0,
+      "completions/min_terminated_length": 103.0,
+      "entropy": 0.3484980883076787,
+      "epoch": 0.44,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.019602373242378235,
+      "kl": 0.0015943678226904012,
+      "learning_rate": 3.141334596385448e-07,
+      "loss": 0.0162,
+      "num_tokens": 2826022.0,
+      "reward": 0.30687499046325684,
+      "reward_std": 0.2975793182849884,
+      "rewards/alfworld_rollout_reward_func/mean": 0.30687499046325684,
+      "rewards/alfworld_rollout_reward_func/std": 0.5158684253692627,
+      "rollout/action_tokens_mean": 774.25,
+      "rollout/episode_turns_mean": 22.890625,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.640625,
+      "sampling/importance_sampling_ratio/max": 2.4803109169006348,
+      "sampling/importance_sampling_ratio/mean": 0.5007343292236328,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.6167523860931396,
+      "sampling/sampling_logp_difference/mean": 0.021400175988674164,
+      "step": 55,
+      "step_time": 371.2160142980065
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.734375,
+      "completions/max_length": 1620.0,
+      "completions/max_terminated_length": 957.0,
+      "completions/mean_length": 795.484375,
+      "completions/mean_terminated_length": 376.058837890625,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
+      "entropy": 0.32310054916888475,
+      "epoch": 0.448,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01010950654745102,
+      "kl": 0.0015955448616296053,
+      "learning_rate": 2.5548865107314606e-07,
+      "loss": 0.0093,
+      "num_tokens": 2876933.0,
+      "reward": 0.22109374403953552,
+      "reward_std": 0.31529495120048523,
+      "rewards/alfworld_rollout_reward_func/mean": 0.22109374403953552,
+      "rewards/alfworld_rollout_reward_func/std": 0.4678628444671631,
+      "rollout/action_tokens_mean": 795.484375,
+      "rollout/episode_turns_mean": 25.328125,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.734375,
+      "sampling/importance_sampling_ratio/max": 2.8859243392944336,
+      "sampling/importance_sampling_ratio/mean": 0.5188072919845581,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 5.368804454803467,
+      "sampling/sampling_logp_difference/mean": 0.020041368901729584,
+      "step": 56,
+      "step_time": 424.94623591700656
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.71875,
+      "completions/max_length": 1384.0,
+      "completions/max_terminated_length": 1021.0,
+      "completions/mean_length": 818.9375,
+      "completions/mean_terminated_length": 486.0,
+      "completions/min_length": 213.0,
+      "completions/min_terminated_length": 213.0,
+      "entropy": 0.35508036240935326,
+      "epoch": 0.456,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.010526562109589577,
+      "kl": 0.0017983552861551289,
+      "learning_rate": 2.026054709494235e-07,
+      "loss": -0.0097,
+      "num_tokens": 2929345.0,
+      "reward": 0.23046875,
+      "reward_std": 0.4045883119106293,
+      "rewards/alfworld_rollout_reward_func/mean": 0.23046875,
+      "rewards/alfworld_rollout_reward_func/std": 0.475816547870636,
+      "rollout/action_tokens_mean": 818.9375,
+      "rollout/episode_turns_mean": 25.953125,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.71875,
+      "sampling/importance_sampling_ratio/max": 2.9720585346221924,
+      "sampling/importance_sampling_ratio/mean": 0.5196441411972046,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.8799049854278564,
+      "sampling/sampling_logp_difference/mean": 0.022473536431789398,
+      "step": 57,
+      "step_time": 439.1710239440072
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5625,
+      "completions/max_length": 1419.0,
+      "completions/max_terminated_length": 765.0,
+      "completions/mean_length": 738.4375,
+      "completions/mean_terminated_length": 387.2500305175781,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
+      "entropy": 0.3470684285275638,
+      "epoch": 0.464,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.00702791940420866,
+      "kl": 0.0016746247110859258,
+      "learning_rate": 1.5561966963229925e-07,
+      "loss": -0.0058,
+      "num_tokens": 2976605.0,
+      "reward": 0.3982812762260437,
+      "reward_std": 0.37868061661720276,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3982812464237213,
+      "rewards/alfworld_rollout_reward_func/std": 0.5275837779045105,
+      "rollout/action_tokens_mean": 738.4375,
+      "rollout/episode_turns_mean": 22.078125,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.5625,
+      "sampling/importance_sampling_ratio/max": 1.7666839361190796,
+      "sampling/importance_sampling_ratio/mean": 0.42546379566192627,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.1239027976989746,
+      "sampling/sampling_logp_difference/mean": 0.02082432433962822,
+      "step": 58,
+      "step_time": 360.1903216189894
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.4375,
+      "completions/max_length": 1488.0,
+      "completions/max_terminated_length": 1112.0,
+      "completions/mean_length": 657.859375,
+      "completions/mean_terminated_length": 380.02777099609375,
+      "completions/min_length": 81.0,
+      "completions/min_terminated_length": 81.0,
+      "entropy": 0.3138961121439934,
+      "epoch": 0.472,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.016277866438031197,
+      "kl": 0.0016064580941019813,
+      "learning_rate": 1.1465185899987797e-07,
+      "loss": 0.0023,
+      "num_tokens": 3018708.0,
+      "reward": 0.5278124809265137,
+      "reward_std": 0.386292964220047,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5278124809265137,
+      "rewards/alfworld_rollout_reward_func/std": 0.5249382853507996,
+      "rollout/action_tokens_mean": 657.859375,
+      "rollout/episode_turns_mean": 19.984375,
+      "rollout/mismatch_mean": 0.0,
+      "rollout/mismatch_ratio": 0.0,
+      "rollout/truncated_ratio": 0.4375,
+      "sampling/importance_sampling_ratio/max": 2.904142379760742,
+      "sampling/importance_sampling_ratio/mean": 0.5997130274772644,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.4242944717407227,
+      "sampling/sampling_logp_difference/mean": 0.01990351639688015,
+      "step": 59,
+      "step_time": 324.923169317015
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.65625,
+      "completions/max_length": 1532.0,
+      "completions/max_terminated_length": 1118.0,
+      "completions/mean_length": 825.765625,
+      "completions/mean_terminated_length": 551.6818237304688,
+      "completions/min_length": 132.0,
+      "completions/min_terminated_length": 132.0,
+      "entropy": 0.3388841552659869,
+      "epoch": 0.48,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.01379719190299511,
+      "kl": 0.001881336524093058,
+      "learning_rate": 7.980720283448957e-08,
+      "loss": 0.0022,
+      "num_tokens": 3071557.0,
+      "reward": 0.28171876072883606,
+      "reward_std": 0.49070847034454346,
+      "rewards/alfworld_rollout_reward_func/mean": 0.28171873092651367,
+      "rewards/alfworld_rollout_reward_func/std": 0.50815749168396,
+      "rollout/action_tokens_mean": 825.765625,
+      "rollout/episode_turns_mean": 25.703125,
+      "rollout/mismatch_mean": 0.015625,
+      "rollout/mismatch_ratio": 0.015625,
+      "rollout/truncated_ratio": 0.65625,
+      "sampling/importance_sampling_ratio/max": 2.6620919704437256,
+      "sampling/importance_sampling_ratio/mean": 0.47809475660324097,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.668699741363525,
+      "sampling/sampling_logp_difference/mean": 0.02150684781372547,
+      "step": 60,
+      "step_time": 377.0457431760042
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 64,
+  "num_input_tokens_seen": 3071557,
+  "num_train_epochs": 1,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-64/README.md

@@ -0,0 +1,210 @@
+---
+base_model: /cache/models/Qwen--Qwen2.5-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-64/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-64/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-64/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-64/ref/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-64/special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-64/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

config.json

@@ -0,0 +1,66 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151645,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 70,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

ref/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}