Upload alfworld-v3.0.0 model

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-20/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,68 @@
+---
+library_name: transformers
+model_name: app/checkpoints/1/alfworld-v3.0.0
+tags:
+- generated_from_trainer
+- axolotl
+- grpo
+- trl
+licence: license
+---
+
+# Model Card for app/checkpoints/1/alfworld-v3.0.0
+
+This model is a fine-tuned version of [None](https://huggingface.co/None).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/iamPi-0519/Gradients-Rollout/runs/v89kzb8a) 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.27.0.dev0
+- Transformers: 4.57.6
+- Pytorch: 2.9.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{shao2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b878ff20d084d009a6f57bea4813e4afefd51a1703b7783226841a37ec00fedb
+size 119801528

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-20/README.md

@@ -0,0 +1,210 @@
+---
+base_model: /cache/models/Qwen--Qwen2.5-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-20/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-20/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9832dc1bc2023d5f3837556ccda2868270f77d4d6f58612c28d622015c5f420
+size 119801528

checkpoint-20/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-20/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-20/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c58dcc3771032c6173598b076cb68a1f664661be00fe1e5910df9105bef62a03
+size 61392581

checkpoint-20/ref/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-20/ref/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a73e8a3357e88549443ed67169c44587d8849b718265a36909b7373da2322a00
+size 59934640

checkpoint-20/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:617359dfa3f981ca698950bb07b7755a5d3b648c9110431d345731aec957bafd
+size 14853

checkpoint-20/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce14c9f96a0aa5a11ec1906c8b06547968496f7f4f6c975b4a3a9e1bd70674ce
+size 14917

checkpoint-20/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20e6cb87497d4549ec3c18de3c76741e527086c1a2492b0bd96b9fe6d8fdaaab
+size 1465

checkpoint-20/special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-20/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoint-20/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-20/trainer_state.json

@@ -0,0 +1,694 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.3225806451612903,
+  "eval_steps": 500,
+  "global_step": 20,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.53125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 487.0,
+      "completions/mean_length": 370.3359375,
+      "completions/mean_terminated_length": 209.78334045410156,
+      "completions/min_length": 66.0,
+      "completions/min_terminated_length": 66.0,
+      "entropy": 0.40961652249097824,
+      "epoch": 0.016129032258064516,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1428968608379364,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": -0.0891,
+      "num_tokens": 110863.0,
+      "reward": 0.09531250596046448,
+      "reward_std": 2.4042975902557373,
+      "rewards/alfworld_rollout_reward_func/mean": 0.09531247615814209,
+      "rewards/alfworld_rollout_reward_func/std": 2.9919276237487793,
+      "sampling/importance_sampling_ratio/max": 2.9990427494049072,
+      "sampling/importance_sampling_ratio/mean": 0.6262807846069336,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.450258255004883,
+      "sampling/sampling_logp_difference/mean": 0.024438712745904922,
+      "step": 1,
+      "step_time": 309.93943907799985
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5703125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 509.0,
+      "completions/mean_length": 380.1015625,
+      "completions/mean_terminated_length": 205.03636169433594,
+      "completions/min_length": 79.0,
+      "completions/min_terminated_length": 79.0,
+      "entropy": 0.40012454986572266,
+      "epoch": 0.03225806451612903,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12177681177854538,
+      "kl": 0.0,
+      "learning_rate": 2.5e-05,
+      "loss": -0.085,
+      "num_tokens": 222814.0,
+      "reward": 0.5320311784744263,
+      "reward_std": 2.642284393310547,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5320311784744263,
+      "rewards/alfworld_rollout_reward_func/std": 3.4178764820098877,
+      "sampling/importance_sampling_ratio/max": 2.909560441970825,
+      "sampling/importance_sampling_ratio/mean": 0.5676747560501099,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.949212551116943,
+      "sampling/sampling_logp_difference/mean": 0.023133086040616035,
+      "step": 2,
+      "step_time": 315.77915475400005
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6015625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 497.0,
+      "completions/mean_length": 378.34375,
+      "completions/mean_terminated_length": 176.5490264892578,
+      "completions/min_length": 76.0,
+      "completions/min_terminated_length": 76.0,
+      "entropy": 0.4211720749735832,
+      "epoch": 0.04838709677419355,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13358080387115479,
+      "kl": 0.00272539074649103,
+      "learning_rate": 2.493586654239869e-05,
+      "loss": -0.0053,
+      "num_tokens": 332775.0,
+      "reward": 0.925000011920929,
+      "reward_std": 3.4297051429748535,
+      "rewards/alfworld_rollout_reward_func/mean": 0.925000011920929,
+      "rewards/alfworld_rollout_reward_func/std": 3.9396629333496094,
+      "sampling/importance_sampling_ratio/max": 2.511305809020996,
+      "sampling/importance_sampling_ratio/mean": 0.5851866602897644,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.5749268531799316,
+      "sampling/sampling_logp_difference/mean": 0.023533061146736145,
+      "step": 3,
+      "step_time": 296.23798338899906
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.53125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 511.0,
+      "completions/mean_length": 357.84375,
+      "completions/mean_terminated_length": 183.1333465576172,
+      "completions/min_length": 66.0,
+      "completions/min_terminated_length": 66.0,
+      "entropy": 0.3783623240888119,
+      "epoch": 0.06451612903225806,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15261100232601166,
+      "kl": 0.002557590647484176,
+      "learning_rate": 2.474412426565618e-05,
+      "loss": -0.0004,
+      "num_tokens": 441773.0,
+      "reward": 0.45625001192092896,
+      "reward_std": 2.879180908203125,
+      "rewards/alfworld_rollout_reward_func/mean": 0.45625001192092896,
+      "rewards/alfworld_rollout_reward_func/std": 3.427947759628296,
+      "sampling/importance_sampling_ratio/max": 2.91353440284729,
+      "sampling/importance_sampling_ratio/mean": 0.6235978603363037,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.593559741973877,
+      "sampling/sampling_logp_difference/mean": 0.023225955665111542,
+      "step": 4,
+      "step_time": 322.2789937999987
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3984375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 504.0,
+      "completions/mean_length": 298.53125,
+      "completions/mean_terminated_length": 157.14285278320312,
+      "completions/min_length": 64.0,
+      "completions/min_terminated_length": 64.0,
+      "entropy": 0.4225323870778084,
+      "epoch": 0.08064516129032258,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11338824778795242,
+      "kl": 0.003911561769200489,
+      "learning_rate": 2.442674070500061e-05,
+      "loss": 0.0261,
+      "num_tokens": 540709.0,
+      "reward": 0.48515623807907104,
+      "reward_std": 2.928551197052002,
+      "rewards/alfworld_rollout_reward_func/mean": 0.48515623807907104,
+      "rewards/alfworld_rollout_reward_func/std": 3.3146796226501465,
+      "sampling/importance_sampling_ratio/max": 2.970187187194824,
+      "sampling/importance_sampling_ratio/mean": 0.5831338763237,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 6.362844944000244,
+      "sampling/sampling_logp_difference/mean": 0.024303022772073746,
+      "step": 5,
+      "step_time": 281.38914586900137
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.34375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 476.0,
+      "completions/mean_length": 280.25,
+      "completions/mean_terminated_length": 158.85714721679688,
+      "completions/min_length": 79.0,
+      "completions/min_terminated_length": 79.0,
+      "entropy": 0.4307485744357109,
+      "epoch": 0.0967741935483871,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.840685248374939,
+      "kl": 0.035365400719456375,
+      "learning_rate": 2.3986972645252883e-05,
+      "loss": 0.0634,
+      "num_tokens": 641237.0,
+      "reward": 1.1078124046325684,
+      "reward_std": 3.4873318672180176,
+      "rewards/alfworld_rollout_reward_func/mean": 1.107812523841858,
+      "rewards/alfworld_rollout_reward_func/std": 3.958895206451416,
+      "sampling/importance_sampling_ratio/max": 2.7673635482788086,
+      "sampling/importance_sampling_ratio/mean": 0.561335563659668,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 11.066271781921387,
+      "sampling/sampling_logp_difference/mean": 0.02636803314089775,
+      "step": 6,
+      "step_time": 306.572960538002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.28125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 474.0,
+      "completions/mean_length": 248.328125,
+      "completions/mean_terminated_length": 145.1521759033203,
+      "completions/min_length": 61.0,
+      "completions/min_terminated_length": 61.0,
+      "entropy": 0.4486994035542011,
+      "epoch": 0.11290322580645161,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13363927602767944,
+      "kl": 0.010801714262925088,
+      "learning_rate": 2.342933270180728e-05,
+      "loss": 0.1147,
+      "num_tokens": 734359.0,
+      "reward": 0.7398437261581421,
+      "reward_std": 3.073120594024658,
+      "rewards/alfworld_rollout_reward_func/mean": 0.7398437261581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.5458602905273438,
+      "sampling/importance_sampling_ratio/max": 2.4107882976531982,
+      "sampling/importance_sampling_ratio/mean": 0.5770797729492188,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 7.637388229370117,
+      "sampling/sampling_logp_difference/mean": 0.027358299121260643,
+      "step": 7,
+      "step_time": 291.4309037350031
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2109375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 487.0,
+      "completions/mean_length": 224.53125,
+      "completions/mean_terminated_length": 147.68316650390625,
+      "completions/min_length": 71.0,
+      "completions/min_terminated_length": 71.0,
+      "entropy": 0.4458822049200535,
+      "epoch": 0.12903225806451613,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1380084902048111,
+      "kl": 0.018724333960562944,
+      "learning_rate": 2.2759543015090955e-05,
+      "loss": 0.0322,
+      "num_tokens": 827827.0,
+      "reward": 0.8648437261581421,
+      "reward_std": 3.5357110500335693,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8648437857627869,
+      "rewards/alfworld_rollout_reward_func/std": 3.7438578605651855,
+      "sampling/importance_sampling_ratio/max": 2.939615249633789,
+      "sampling/importance_sampling_ratio/mean": 0.5691805481910706,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 5.754482269287109,
+      "sampling/sampling_logp_difference/mean": 0.028573205694556236,
+      "step": 8,
+      "step_time": 305.87279431799834
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 504.0,
+      "completions/mean_length": 218.171875,
+      "completions/mean_terminated_length": 150.36538696289062,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
+      "entropy": 0.4581918604671955,
+      "epoch": 0.14516129032258066,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11444258689880371,
+      "kl": 0.02050452691037208,
+      "learning_rate": 2.1984476533659888e-05,
+      "loss": 0.0214,
+      "num_tokens": 918938.0,
+      "reward": 1.025781273841858,
+      "reward_std": 3.5701520442962646,
+      "rewards/alfworld_rollout_reward_func/mean": 1.0257811546325684,
+      "rewards/alfworld_rollout_reward_func/std": 3.894923686981201,
+      "sampling/importance_sampling_ratio/max": 2.700753688812256,
+      "sampling/importance_sampling_ratio/mean": 0.6189267635345459,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.1386852264404297,
+      "sampling/sampling_logp_difference/mean": 0.0270779300481081,
+      "step": 9,
+      "step_time": 308.85608878800485
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0546875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 432.0,
+      "completions/mean_length": 150.1875,
+      "completions/mean_terminated_length": 129.25619506835938,
+      "completions/min_length": 62.0,
+      "completions/min_terminated_length": 62.0,
+      "entropy": 0.5325295478105545,
+      "epoch": 0.16129032258064516,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2502661645412445,
+      "kl": 0.03952266392298043,
+      "learning_rate": 2.1112086488446085e-05,
+      "loss": 0.029,
+      "num_tokens": 1000430.0,
+      "reward": 0.8335937261581421,
+      "reward_std": 3.1759183406829834,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8335937261581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.7348177433013916,
+      "sampling/importance_sampling_ratio/max": 2.995654344558716,
+      "sampling/importance_sampling_ratio/mean": 0.7903203964233398,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.775932788848877,
+      "sampling/sampling_logp_difference/mean": 0.03168691694736481,
+      "step": 10,
+      "step_time": 285.58172405699224
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0859375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 325.0,
+      "completions/mean_length": 156.7734375,
+      "completions/mean_terminated_length": 123.3760757446289,
+      "completions/min_length": 64.0,
+      "completions/min_terminated_length": 64.0,
+      "entropy": 0.5142566300928593,
+      "epoch": 0.1774193548387097,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18933017551898956,
+      "kl": 0.04493421735242009,
+      "learning_rate": 2.0151324781845787e-05,
+      "loss": 0.0496,
+      "num_tokens": 1082621.0,
+      "reward": 0.9093749523162842,
+      "reward_std": 3.467135429382324,
+      "rewards/alfworld_rollout_reward_func/mean": 0.909375011920929,
+      "rewards/alfworld_rollout_reward_func/std": 3.8583264350891113,
+      "sampling/importance_sampling_ratio/max": 2.9975228309631348,
+      "sampling/importance_sampling_ratio/mean": 0.8283847570419312,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.3669824600219727,
+      "sampling/sampling_logp_difference/mean": 0.028965603560209274,
+      "step": 11,
+      "step_time": 300.83864827199795
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.046875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 343.0,
+      "completions/mean_length": 140.28125,
+      "completions/mean_terminated_length": 121.99999237060547,
+      "completions/min_length": 51.0,
+      "completions/min_terminated_length": 51.0,
+      "entropy": 0.519548587501049,
+      "epoch": 0.1935483870967742,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2622426152229309,
+      "kl": 0.05217514652758837,
+      "learning_rate": 1.911205012908703e-05,
+      "loss": -0.0594,
+      "num_tokens": 1162516.0,
+      "reward": 0.8148437738418579,
+      "reward_std": 3.1582674980163574,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8148437142372131,
+      "rewards/alfworld_rollout_reward_func/std": 3.7535512447357178,
+      "sampling/importance_sampling_ratio/max": 2.8224730491638184,
+      "sampling/importance_sampling_ratio/mean": 0.7709423899650574,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.520348072052002,
+      "sampling/sampling_logp_difference/mean": 0.031547725200653076,
+      "step": 12,
+      "step_time": 287.3358050610004
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0703125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 256.0,
+      "completions/mean_length": 151.046875,
+      "completions/mean_terminated_length": 123.7479019165039,
+      "completions/min_length": 60.0,
+      "completions/min_terminated_length": 60.0,
+      "entropy": 0.5261144414544106,
+      "epoch": 0.20967741935483872,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1863553375005722,
+      "kl": 0.06452470645308495,
+      "learning_rate": 1.800492689447043e-05,
+      "loss": -0.0566,
+      "num_tokens": 1243615.0,
+      "reward": 0.8781249523162842,
+      "reward_std": 3.3634252548217773,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8781249523162842,
+      "rewards/alfworld_rollout_reward_func/std": 3.842200994491577,
+      "sampling/importance_sampling_ratio/max": 2.4663472175598145,
+      "sampling/importance_sampling_ratio/mean": 0.7910538911819458,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.4453465938568115,
+      "sampling/sampling_logp_difference/mean": 0.02982635796070099,
+      "step": 13,
+      "step_time": 293.8646844759969
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0078125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 380.0,
+      "completions/mean_length": 129.8671875,
+      "completions/mean_terminated_length": 126.85826873779297,
+      "completions/min_length": 56.0,
+      "completions/min_terminated_length": 56.0,
+      "entropy": 0.5344252400100231,
+      "epoch": 0.22580645161290322,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.24170275032520294,
+      "kl": 0.06391164055094123,
+      "learning_rate": 1.6841315660560252e-05,
+      "loss": -0.1138,
+      "num_tokens": 1321112.0,
+      "reward": 0.9546874761581421,
+      "reward_std": 3.586996078491211,
+      "rewards/alfworld_rollout_reward_func/mean": 0.9546874761581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.9051523208618164,
+      "sampling/importance_sampling_ratio/max": 2.8537843227386475,
+      "sampling/importance_sampling_ratio/mean": 0.7751458883285522,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.7649002075195312,
+      "sampling/sampling_logp_difference/mean": 0.03092074766755104,
+      "step": 14,
+      "step_time": 283.45238025300205
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0546875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 457.0,
+      "completions/mean_length": 149.8515625,
+      "completions/mean_terminated_length": 128.90081787109375,
+      "completions/min_length": 70.0,
+      "completions/min_terminated_length": 70.0,
+      "entropy": 0.5125062428414822,
+      "epoch": 0.24193548387096775,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16989152133464813,
+      "kl": 0.053708435501903296,
+      "learning_rate": 1.563315665323401e-05,
+      "loss": 0.0051,
+      "num_tokens": 1403786.0,
+      "reward": 0.34921878576278687,
+      "reward_std": 2.4881625175476074,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3492187559604645,
+      "rewards/alfworld_rollout_reward_func/std": 3.1670050621032715,
+      "sampling/importance_sampling_ratio/max": 2.4405319690704346,
+      "sampling/importance_sampling_ratio/mean": 0.7845873832702637,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.7177300453186035,
+      "sampling/sampling_logp_difference/mean": 0.029662001878023148,
+      "step": 15,
+      "step_time": 308.28231060100006
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0703125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 422.0,
+      "completions/mean_length": 156.3359375,
+      "completions/mean_terminated_length": 129.43698120117188,
+      "completions/min_length": 71.0,
+      "completions/min_terminated_length": 71.0,
+      "entropy": 0.5052059814333916,
+      "epoch": 0.25806451612903225,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18680106103420258,
+      "kl": 0.052992088720202446,
+      "learning_rate": 1.439284721880721e-05,
+      "loss": -0.0061,
+      "num_tokens": 1485910.0,
+      "reward": 1.005468726158142,
+      "reward_std": 2.9901723861694336,
+      "rewards/alfworld_rollout_reward_func/mean": 1.005468726158142,
+      "rewards/alfworld_rollout_reward_func/std": 3.869877815246582,
+      "sampling/importance_sampling_ratio/max": 2.6479995250701904,
+      "sampling/importance_sampling_ratio/mean": 0.8335222005844116,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2812881469726562,
+      "sampling/sampling_logp_difference/mean": 0.02822798863053322,
+      "step": 16,
+      "step_time": 288.45321577499635
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 436.0,
+      "completions/mean_length": 155.78125,
+      "completions/mean_terminated_length": 132.03334045410156,
+      "completions/min_length": 62.0,
+      "completions/min_terminated_length": 62.0,
+      "entropy": 0.5273009687662125,
+      "epoch": 0.27419354838709675,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17423537373542786,
+      "kl": 0.07150922995060682,
+      "learning_rate": 1.3133114610483909e-05,
+      "loss": -0.0034,
+      "num_tokens": 1567778.0,
+      "reward": 0.25390625,
+      "reward_std": 2.4427285194396973,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2539062201976776,
+      "rewards/alfworld_rollout_reward_func/std": 3.2506790161132812,
+      "sampling/importance_sampling_ratio/max": 2.4126062393188477,
+      "sampling/importance_sampling_ratio/mean": 0.8037413954734802,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.5358574390411377,
+      "sampling/sampling_logp_difference/mean": 0.029498092830181122,
+      "step": 17,
+      "step_time": 281.3368725319997
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1484375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 477.0,
+      "completions/mean_length": 186.5,
+      "completions/mean_terminated_length": 129.76145935058594,
+      "completions/min_length": 73.0,
+      "completions/min_terminated_length": 73.0,
+      "entropy": 0.47341491281986237,
+      "epoch": 0.2903225806451613,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1518625020980835,
+      "kl": 0.049451613100245595,
+      "learning_rate": 1.1866885389516092e-05,
+      "loss": 0.0897,
+      "num_tokens": 1656705.0,
+      "reward": 0.80859375,
+      "reward_std": 3.4441990852355957,
+      "rewards/alfworld_rollout_reward_func/mean": 0.80859375,
+      "rewards/alfworld_rollout_reward_func/std": 3.8008499145507812,
+      "sampling/importance_sampling_ratio/max": 2.9426238536834717,
+      "sampling/importance_sampling_ratio/mean": 0.7826383113861084,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3363547325134277,
+      "sampling/sampling_logp_difference/mean": 0.027312669903039932,
+      "step": 18,
+      "step_time": 334.05939280299935
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1015625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 499.0,
+      "completions/mean_length": 166.0859375,
+      "completions/mean_terminated_length": 126.98260498046875,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
+      "entropy": 0.47634539753198624,
+      "epoch": 0.3064516129032258,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2188945859670639,
+      "kl": 0.0531598380766809,
+      "learning_rate": 1.0607152781192796e-05,
+      "loss": 0.0078,
+      "num_tokens": 1741466.0,
+      "reward": 0.6265625357627869,
+      "reward_std": 2.6468114852905273,
+      "rewards/alfworld_rollout_reward_func/mean": 0.6265624761581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.4735782146453857,
+      "sampling/importance_sampling_ratio/max": 2.799043655395508,
+      "sampling/importance_sampling_ratio/mean": 0.8041483163833618,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.4288792610168457,
+      "sampling/sampling_logp_difference/mean": 0.02712005004286766,
+      "step": 19,
+      "step_time": 303.70944481100014
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0859375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 403.0,
+      "completions/mean_length": 160.9375,
+      "completions/mean_terminated_length": 127.93163299560547,
+      "completions/min_length": 76.0,
+      "completions/min_terminated_length": 76.0,
+      "entropy": 0.46492085233330727,
+      "epoch": 0.3225806451612903,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7110471725463867,
+      "kl": 0.09847234468907118,
+      "learning_rate": 9.366843346765992e-06,
+      "loss": 0.0238,
+      "num_tokens": 1823852.0,
+      "reward": 0.819531261920929,
+      "reward_std": 3.2860288619995117,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8195312023162842,
+      "rewards/alfworld_rollout_reward_func/std": 3.780327320098877,
+      "sampling/importance_sampling_ratio/max": 2.7687222957611084,
+      "sampling/importance_sampling_ratio/mean": 0.7973555326461792,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 20.83700942993164,
+      "sampling/sampling_logp_difference/mean": 0.027603479102253914,
+      "step": 20,
+      "step_time": 296.20082658499814
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 32,
+  "num_input_tokens_seen": 1823852,
+  "num_train_epochs": 1,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-20/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe31dee94209c1973566ddbbc45e4eaa5758a9402713eebce4f579426cfdc6a3
+size 9361

checkpoint-32/README.md

@@ -0,0 +1,210 @@
+---
+base_model: /cache/models/Qwen--Qwen2.5-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-32/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-32/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b878ff20d084d009a6f57bea4813e4afefd51a1703b7783226841a37ec00fedb
+size 119801528

checkpoint-32/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-32/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-32/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eae68cc173b6507d8f069518b00419e9fe3a5d0ee5b2ff7e1205996be8c53bd7
+size 61392581

checkpoint-32/ref/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-32/ref/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a73e8a3357e88549443ed67169c44587d8849b718265a36909b7373da2322a00
+size 59934640

checkpoint-32/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9d3bb4ce845169fd3ce72597fab37a23453e900b23b17ba40fcfd1fa88b6e6f
+size 14917

checkpoint-32/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a00654d154682a9b49ef12ed206fff9ae0b9894c482c27008bf4244f61914c
+size 14981

checkpoint-32/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48ea6f0f38f84d67d36c14ee5cfbd976fee0d136a7e48f9dfe00eb968cff425e
+size 1465

checkpoint-32/special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-32/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoint-32/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-32/trainer_state.json

@@ -0,0 +1,1090 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5161290322580645,
+  "eval_steps": 500,
+  "global_step": 32,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.53125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 487.0,
+      "completions/mean_length": 370.3359375,
+      "completions/mean_terminated_length": 209.78334045410156,
+      "completions/min_length": 66.0,
+      "completions/min_terminated_length": 66.0,
+      "entropy": 0.40961652249097824,
+      "epoch": 0.016129032258064516,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1428968608379364,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": -0.0891,
+      "num_tokens": 110863.0,
+      "reward": 0.09531250596046448,
+      "reward_std": 2.4042975902557373,
+      "rewards/alfworld_rollout_reward_func/mean": 0.09531247615814209,
+      "rewards/alfworld_rollout_reward_func/std": 2.9919276237487793,
+      "sampling/importance_sampling_ratio/max": 2.9990427494049072,
+      "sampling/importance_sampling_ratio/mean": 0.6262807846069336,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.450258255004883,
+      "sampling/sampling_logp_difference/mean": 0.024438712745904922,
+      "step": 1,
+      "step_time": 309.93943907799985
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.5703125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 509.0,
+      "completions/mean_length": 380.1015625,
+      "completions/mean_terminated_length": 205.03636169433594,
+      "completions/min_length": 79.0,
+      "completions/min_terminated_length": 79.0,
+      "entropy": 0.40012454986572266,
+      "epoch": 0.03225806451612903,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12177681177854538,
+      "kl": 0.0,
+      "learning_rate": 2.5e-05,
+      "loss": -0.085,
+      "num_tokens": 222814.0,
+      "reward": 0.5320311784744263,
+      "reward_std": 2.642284393310547,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5320311784744263,
+      "rewards/alfworld_rollout_reward_func/std": 3.4178764820098877,
+      "sampling/importance_sampling_ratio/max": 2.909560441970825,
+      "sampling/importance_sampling_ratio/mean": 0.5676747560501099,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.949212551116943,
+      "sampling/sampling_logp_difference/mean": 0.023133086040616035,
+      "step": 2,
+      "step_time": 315.77915475400005
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6015625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 497.0,
+      "completions/mean_length": 378.34375,
+      "completions/mean_terminated_length": 176.5490264892578,
+      "completions/min_length": 76.0,
+      "completions/min_terminated_length": 76.0,
+      "entropy": 0.4211720749735832,
+      "epoch": 0.04838709677419355,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13358080387115479,
+      "kl": 0.00272539074649103,
+      "learning_rate": 2.493586654239869e-05,
+      "loss": -0.0053,
+      "num_tokens": 332775.0,
+      "reward": 0.925000011920929,
+      "reward_std": 3.4297051429748535,
+      "rewards/alfworld_rollout_reward_func/mean": 0.925000011920929,
+      "rewards/alfworld_rollout_reward_func/std": 3.9396629333496094,
+      "sampling/importance_sampling_ratio/max": 2.511305809020996,
+      "sampling/importance_sampling_ratio/mean": 0.5851866602897644,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.5749268531799316,
+      "sampling/sampling_logp_difference/mean": 0.023533061146736145,
+      "step": 3,
+      "step_time": 296.23798338899906
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.53125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 511.0,
+      "completions/mean_length": 357.84375,
+      "completions/mean_terminated_length": 183.1333465576172,
+      "completions/min_length": 66.0,
+      "completions/min_terminated_length": 66.0,
+      "entropy": 0.3783623240888119,
+      "epoch": 0.06451612903225806,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15261100232601166,
+      "kl": 0.002557590647484176,
+      "learning_rate": 2.474412426565618e-05,
+      "loss": -0.0004,
+      "num_tokens": 441773.0,
+      "reward": 0.45625001192092896,
+      "reward_std": 2.879180908203125,
+      "rewards/alfworld_rollout_reward_func/mean": 0.45625001192092896,
+      "rewards/alfworld_rollout_reward_func/std": 3.427947759628296,
+      "sampling/importance_sampling_ratio/max": 2.91353440284729,
+      "sampling/importance_sampling_ratio/mean": 0.6235978603363037,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.593559741973877,
+      "sampling/sampling_logp_difference/mean": 0.023225955665111542,
+      "step": 4,
+      "step_time": 322.2789937999987
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3984375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 504.0,
+      "completions/mean_length": 298.53125,
+      "completions/mean_terminated_length": 157.14285278320312,
+      "completions/min_length": 64.0,
+      "completions/min_terminated_length": 64.0,
+      "entropy": 0.4225323870778084,
+      "epoch": 0.08064516129032258,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11338824778795242,
+      "kl": 0.003911561769200489,
+      "learning_rate": 2.442674070500061e-05,
+      "loss": 0.0261,
+      "num_tokens": 540709.0,
+      "reward": 0.48515623807907104,
+      "reward_std": 2.928551197052002,
+      "rewards/alfworld_rollout_reward_func/mean": 0.48515623807907104,
+      "rewards/alfworld_rollout_reward_func/std": 3.3146796226501465,
+      "sampling/importance_sampling_ratio/max": 2.970187187194824,
+      "sampling/importance_sampling_ratio/mean": 0.5831338763237,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 6.362844944000244,
+      "sampling/sampling_logp_difference/mean": 0.024303022772073746,
+      "step": 5,
+      "step_time": 281.38914586900137
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.34375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 476.0,
+      "completions/mean_length": 280.25,
+      "completions/mean_terminated_length": 158.85714721679688,
+      "completions/min_length": 79.0,
+      "completions/min_terminated_length": 79.0,
+      "entropy": 0.4307485744357109,
+      "epoch": 0.0967741935483871,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.840685248374939,
+      "kl": 0.035365400719456375,
+      "learning_rate": 2.3986972645252883e-05,
+      "loss": 0.0634,
+      "num_tokens": 641237.0,
+      "reward": 1.1078124046325684,
+      "reward_std": 3.4873318672180176,
+      "rewards/alfworld_rollout_reward_func/mean": 1.107812523841858,
+      "rewards/alfworld_rollout_reward_func/std": 3.958895206451416,
+      "sampling/importance_sampling_ratio/max": 2.7673635482788086,
+      "sampling/importance_sampling_ratio/mean": 0.561335563659668,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 11.066271781921387,
+      "sampling/sampling_logp_difference/mean": 0.02636803314089775,
+      "step": 6,
+      "step_time": 306.572960538002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.28125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 474.0,
+      "completions/mean_length": 248.328125,
+      "completions/mean_terminated_length": 145.1521759033203,
+      "completions/min_length": 61.0,
+      "completions/min_terminated_length": 61.0,
+      "entropy": 0.4486994035542011,
+      "epoch": 0.11290322580645161,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13363927602767944,
+      "kl": 0.010801714262925088,
+      "learning_rate": 2.342933270180728e-05,
+      "loss": 0.1147,
+      "num_tokens": 734359.0,
+      "reward": 0.7398437261581421,
+      "reward_std": 3.073120594024658,
+      "rewards/alfworld_rollout_reward_func/mean": 0.7398437261581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.5458602905273438,
+      "sampling/importance_sampling_ratio/max": 2.4107882976531982,
+      "sampling/importance_sampling_ratio/mean": 0.5770797729492188,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 7.637388229370117,
+      "sampling/sampling_logp_difference/mean": 0.027358299121260643,
+      "step": 7,
+      "step_time": 291.4309037350031
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2109375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 487.0,
+      "completions/mean_length": 224.53125,
+      "completions/mean_terminated_length": 147.68316650390625,
+      "completions/min_length": 71.0,
+      "completions/min_terminated_length": 71.0,
+      "entropy": 0.4458822049200535,
+      "epoch": 0.12903225806451613,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1380084902048111,
+      "kl": 0.018724333960562944,
+      "learning_rate": 2.2759543015090955e-05,
+      "loss": 0.0322,
+      "num_tokens": 827827.0,
+      "reward": 0.8648437261581421,
+      "reward_std": 3.5357110500335693,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8648437857627869,
+      "rewards/alfworld_rollout_reward_func/std": 3.7438578605651855,
+      "sampling/importance_sampling_ratio/max": 2.939615249633789,
+      "sampling/importance_sampling_ratio/mean": 0.5691805481910706,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 5.754482269287109,
+      "sampling/sampling_logp_difference/mean": 0.028573205694556236,
+      "step": 8,
+      "step_time": 305.87279431799834
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 504.0,
+      "completions/mean_length": 218.171875,
+      "completions/mean_terminated_length": 150.36538696289062,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
+      "entropy": 0.4581918604671955,
+      "epoch": 0.14516129032258066,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11444258689880371,
+      "kl": 0.02050452691037208,
+      "learning_rate": 2.1984476533659888e-05,
+      "loss": 0.0214,
+      "num_tokens": 918938.0,
+      "reward": 1.025781273841858,
+      "reward_std": 3.5701520442962646,
+      "rewards/alfworld_rollout_reward_func/mean": 1.0257811546325684,
+      "rewards/alfworld_rollout_reward_func/std": 3.894923686981201,
+      "sampling/importance_sampling_ratio/max": 2.700753688812256,
+      "sampling/importance_sampling_ratio/mean": 0.6189267635345459,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.1386852264404297,
+      "sampling/sampling_logp_difference/mean": 0.0270779300481081,
+      "step": 9,
+      "step_time": 308.85608878800485
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0546875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 432.0,
+      "completions/mean_length": 150.1875,
+      "completions/mean_terminated_length": 129.25619506835938,
+      "completions/min_length": 62.0,
+      "completions/min_terminated_length": 62.0,
+      "entropy": 0.5325295478105545,
+      "epoch": 0.16129032258064516,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2502661645412445,
+      "kl": 0.03952266392298043,
+      "learning_rate": 2.1112086488446085e-05,
+      "loss": 0.029,
+      "num_tokens": 1000430.0,
+      "reward": 0.8335937261581421,
+      "reward_std": 3.1759183406829834,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8335937261581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.7348177433013916,
+      "sampling/importance_sampling_ratio/max": 2.995654344558716,
+      "sampling/importance_sampling_ratio/mean": 0.7903203964233398,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.775932788848877,
+      "sampling/sampling_logp_difference/mean": 0.03168691694736481,
+      "step": 10,
+      "step_time": 285.58172405699224
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0859375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 325.0,
+      "completions/mean_length": 156.7734375,
+      "completions/mean_terminated_length": 123.3760757446289,
+      "completions/min_length": 64.0,
+      "completions/min_terminated_length": 64.0,
+      "entropy": 0.5142566300928593,
+      "epoch": 0.1774193548387097,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18933017551898956,
+      "kl": 0.04493421735242009,
+      "learning_rate": 2.0151324781845787e-05,
+      "loss": 0.0496,
+      "num_tokens": 1082621.0,
+      "reward": 0.9093749523162842,
+      "reward_std": 3.467135429382324,
+      "rewards/alfworld_rollout_reward_func/mean": 0.909375011920929,
+      "rewards/alfworld_rollout_reward_func/std": 3.8583264350891113,
+      "sampling/importance_sampling_ratio/max": 2.9975228309631348,
+      "sampling/importance_sampling_ratio/mean": 0.8283847570419312,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.3669824600219727,
+      "sampling/sampling_logp_difference/mean": 0.028965603560209274,
+      "step": 11,
+      "step_time": 300.83864827199795
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.046875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 343.0,
+      "completions/mean_length": 140.28125,
+      "completions/mean_terminated_length": 121.99999237060547,
+      "completions/min_length": 51.0,
+      "completions/min_terminated_length": 51.0,
+      "entropy": 0.519548587501049,
+      "epoch": 0.1935483870967742,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2622426152229309,
+      "kl": 0.05217514652758837,
+      "learning_rate": 1.911205012908703e-05,
+      "loss": -0.0594,
+      "num_tokens": 1162516.0,
+      "reward": 0.8148437738418579,
+      "reward_std": 3.1582674980163574,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8148437142372131,
+      "rewards/alfworld_rollout_reward_func/std": 3.7535512447357178,
+      "sampling/importance_sampling_ratio/max": 2.8224730491638184,
+      "sampling/importance_sampling_ratio/mean": 0.7709423899650574,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.520348072052002,
+      "sampling/sampling_logp_difference/mean": 0.031547725200653076,
+      "step": 12,
+      "step_time": 287.3358050610004
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0703125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 256.0,
+      "completions/mean_length": 151.046875,
+      "completions/mean_terminated_length": 123.7479019165039,
+      "completions/min_length": 60.0,
+      "completions/min_terminated_length": 60.0,
+      "entropy": 0.5261144414544106,
+      "epoch": 0.20967741935483872,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1863553375005722,
+      "kl": 0.06452470645308495,
+      "learning_rate": 1.800492689447043e-05,
+      "loss": -0.0566,
+      "num_tokens": 1243615.0,
+      "reward": 0.8781249523162842,
+      "reward_std": 3.3634252548217773,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8781249523162842,
+      "rewards/alfworld_rollout_reward_func/std": 3.842200994491577,
+      "sampling/importance_sampling_ratio/max": 2.4663472175598145,
+      "sampling/importance_sampling_ratio/mean": 0.7910538911819458,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.4453465938568115,
+      "sampling/sampling_logp_difference/mean": 0.02982635796070099,
+      "step": 13,
+      "step_time": 293.8646844759969
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0078125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 380.0,
+      "completions/mean_length": 129.8671875,
+      "completions/mean_terminated_length": 126.85826873779297,
+      "completions/min_length": 56.0,
+      "completions/min_terminated_length": 56.0,
+      "entropy": 0.5344252400100231,
+      "epoch": 0.22580645161290322,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.24170275032520294,
+      "kl": 0.06391164055094123,
+      "learning_rate": 1.6841315660560252e-05,
+      "loss": -0.1138,
+      "num_tokens": 1321112.0,
+      "reward": 0.9546874761581421,
+      "reward_std": 3.586996078491211,
+      "rewards/alfworld_rollout_reward_func/mean": 0.9546874761581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.9051523208618164,
+      "sampling/importance_sampling_ratio/max": 2.8537843227386475,
+      "sampling/importance_sampling_ratio/mean": 0.7751458883285522,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.7649002075195312,
+      "sampling/sampling_logp_difference/mean": 0.03092074766755104,
+      "step": 14,
+      "step_time": 283.45238025300205
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0546875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 457.0,
+      "completions/mean_length": 149.8515625,
+      "completions/mean_terminated_length": 128.90081787109375,
+      "completions/min_length": 70.0,
+      "completions/min_terminated_length": 70.0,
+      "entropy": 0.5125062428414822,
+      "epoch": 0.24193548387096775,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16989152133464813,
+      "kl": 0.053708435501903296,
+      "learning_rate": 1.563315665323401e-05,
+      "loss": 0.0051,
+      "num_tokens": 1403786.0,
+      "reward": 0.34921878576278687,
+      "reward_std": 2.4881625175476074,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3492187559604645,
+      "rewards/alfworld_rollout_reward_func/std": 3.1670050621032715,
+      "sampling/importance_sampling_ratio/max": 2.4405319690704346,
+      "sampling/importance_sampling_ratio/mean": 0.7845873832702637,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.7177300453186035,
+      "sampling/sampling_logp_difference/mean": 0.029662001878023148,
+      "step": 15,
+      "step_time": 308.28231060100006
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0703125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 422.0,
+      "completions/mean_length": 156.3359375,
+      "completions/mean_terminated_length": 129.43698120117188,
+      "completions/min_length": 71.0,
+      "completions/min_terminated_length": 71.0,
+      "entropy": 0.5052059814333916,
+      "epoch": 0.25806451612903225,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18680106103420258,
+      "kl": 0.052992088720202446,
+      "learning_rate": 1.439284721880721e-05,
+      "loss": -0.0061,
+      "num_tokens": 1485910.0,
+      "reward": 1.005468726158142,
+      "reward_std": 2.9901723861694336,
+      "rewards/alfworld_rollout_reward_func/mean": 1.005468726158142,
+      "rewards/alfworld_rollout_reward_func/std": 3.869877815246582,
+      "sampling/importance_sampling_ratio/max": 2.6479995250701904,
+      "sampling/importance_sampling_ratio/mean": 0.8335222005844116,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.2812881469726562,
+      "sampling/sampling_logp_difference/mean": 0.02822798863053322,
+      "step": 16,
+      "step_time": 288.45321577499635
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 436.0,
+      "completions/mean_length": 155.78125,
+      "completions/mean_terminated_length": 132.03334045410156,
+      "completions/min_length": 62.0,
+      "completions/min_terminated_length": 62.0,
+      "entropy": 0.5273009687662125,
+      "epoch": 0.27419354838709675,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17423537373542786,
+      "kl": 0.07150922995060682,
+      "learning_rate": 1.3133114610483909e-05,
+      "loss": -0.0034,
+      "num_tokens": 1567778.0,
+      "reward": 0.25390625,
+      "reward_std": 2.4427285194396973,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2539062201976776,
+      "rewards/alfworld_rollout_reward_func/std": 3.2506790161132812,
+      "sampling/importance_sampling_ratio/max": 2.4126062393188477,
+      "sampling/importance_sampling_ratio/mean": 0.8037413954734802,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.5358574390411377,
+      "sampling/sampling_logp_difference/mean": 0.029498092830181122,
+      "step": 17,
+      "step_time": 281.3368725319997
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1484375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 477.0,
+      "completions/mean_length": 186.5,
+      "completions/mean_terminated_length": 129.76145935058594,
+      "completions/min_length": 73.0,
+      "completions/min_terminated_length": 73.0,
+      "entropy": 0.47341491281986237,
+      "epoch": 0.2903225806451613,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1518625020980835,
+      "kl": 0.049451613100245595,
+      "learning_rate": 1.1866885389516092e-05,
+      "loss": 0.0897,
+      "num_tokens": 1656705.0,
+      "reward": 0.80859375,
+      "reward_std": 3.4441990852355957,
+      "rewards/alfworld_rollout_reward_func/mean": 0.80859375,
+      "rewards/alfworld_rollout_reward_func/std": 3.8008499145507812,
+      "sampling/importance_sampling_ratio/max": 2.9426238536834717,
+      "sampling/importance_sampling_ratio/mean": 0.7826383113861084,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3363547325134277,
+      "sampling/sampling_logp_difference/mean": 0.027312669903039932,
+      "step": 18,
+      "step_time": 334.05939280299935
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1015625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 499.0,
+      "completions/mean_length": 166.0859375,
+      "completions/mean_terminated_length": 126.98260498046875,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
+      "entropy": 0.47634539753198624,
+      "epoch": 0.3064516129032258,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2188945859670639,
+      "kl": 0.0531598380766809,
+      "learning_rate": 1.0607152781192796e-05,
+      "loss": 0.0078,
+      "num_tokens": 1741466.0,
+      "reward": 0.6265625357627869,
+      "reward_std": 2.6468114852905273,
+      "rewards/alfworld_rollout_reward_func/mean": 0.6265624761581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.4735782146453857,
+      "sampling/importance_sampling_ratio/max": 2.799043655395508,
+      "sampling/importance_sampling_ratio/mean": 0.8041483163833618,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.4288792610168457,
+      "sampling/sampling_logp_difference/mean": 0.02712005004286766,
+      "step": 19,
+      "step_time": 303.70944481100014
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0859375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 403.0,
+      "completions/mean_length": 160.9375,
+      "completions/mean_terminated_length": 127.93163299560547,
+      "completions/min_length": 76.0,
+      "completions/min_terminated_length": 76.0,
+      "entropy": 0.46492085233330727,
+      "epoch": 0.3225806451612903,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7110471725463867,
+      "kl": 0.09847234468907118,
+      "learning_rate": 9.366843346765992e-06,
+      "loss": 0.0238,
+      "num_tokens": 1823852.0,
+      "reward": 0.819531261920929,
+      "reward_std": 3.2860288619995117,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8195312023162842,
+      "rewards/alfworld_rollout_reward_func/std": 3.780327320098877,
+      "sampling/importance_sampling_ratio/max": 2.7687222957611084,
+      "sampling/importance_sampling_ratio/mean": 0.7973555326461792,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 20.83700942993164,
+      "sampling/sampling_logp_difference/mean": 0.027603479102253914,
+      "step": 20,
+      "step_time": 296.20082658499814
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.078125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 356.0,
+      "completions/mean_length": 154.0625,
+      "completions/mean_terminated_length": 123.72881317138672,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.4891073890030384,
+      "epoch": 0.3387096774193548,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3204660415649414,
+      "kl": 0.06161752389743924,
+      "learning_rate": 8.158684339439748e-06,
+      "loss": 0.1854,
+      "num_tokens": 1905839.0,
+      "reward": 0.66796875,
+      "reward_std": 2.7234749794006348,
+      "rewards/alfworld_rollout_reward_func/mean": 0.66796875,
+      "rewards/alfworld_rollout_reward_func/std": 3.580017566680908,
+      "sampling/importance_sampling_ratio/max": 2.9959678649902344,
+      "sampling/importance_sampling_ratio/mean": 0.9344255328178406,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.3660998344421387,
+      "sampling/sampling_logp_difference/mean": 0.029398081824183464,
+      "step": 21,
+      "step_time": 279.7738521129977
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.171875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 446.0,
+      "completions/mean_length": 197.1015625,
+      "completions/mean_terminated_length": 131.7452850341797,
+      "completions/min_length": 65.0,
+      "completions/min_terminated_length": 65.0,
+      "entropy": 0.4758484289050102,
+      "epoch": 0.3548387096774194,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16225755214691162,
+      "kl": 0.054194105323404074,
+      "learning_rate": 6.99507310552957e-06,
+      "loss": 0.0319,
+      "num_tokens": 1991294.0,
+      "reward": 0.69921875,
+      "reward_std": 3.503169298171997,
+      "rewards/alfworld_rollout_reward_func/mean": 0.69921875,
+      "rewards/alfworld_rollout_reward_func/std": 3.726823329925537,
+      "sampling/importance_sampling_ratio/max": 2.5170819759368896,
+      "sampling/importance_sampling_ratio/mean": 0.777429461479187,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.413573741912842,
+      "sampling/sampling_logp_difference/mean": 0.027101196348667145,
+      "step": 22,
+      "step_time": 285.0048562720003
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1171875,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 471.0,
+      "completions/mean_length": 169.8359375,
+      "completions/mean_terminated_length": 124.41593170166016,
+      "completions/min_length": 57.0,
+      "completions/min_terminated_length": 57.0,
+      "entropy": 0.5033188574016094,
+      "epoch": 0.3709677419354839,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18970800936222076,
+      "kl": 0.04351181210950017,
+      "learning_rate": 5.8879498709129735e-06,
+      "loss": -0.0057,
+      "num_tokens": 2075803.0,
+      "reward": 0.8039063215255737,
+      "reward_std": 3.560253620147705,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8039062023162842,
+      "rewards/alfworld_rollout_reward_func/std": 3.759028911590576,
+      "sampling/importance_sampling_ratio/max": 2.6937098503112793,
+      "sampling/importance_sampling_ratio/mean": 0.7721619606018066,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.6162972450256348,
+      "sampling/sampling_logp_difference/mean": 0.02900775894522667,
+      "step": 23,
+      "step_time": 298.5376572299974
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1484375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 451.0,
+      "completions/mean_length": 186.1640625,
+      "completions/mean_terminated_length": 129.36697387695312,
+      "completions/min_length": 60.0,
+      "completions/min_terminated_length": 60.0,
+      "entropy": 0.4917047880589962,
+      "epoch": 0.3870967741935484,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18956582248210907,
+      "kl": 0.04409886850044131,
+      "learning_rate": 4.848675218154214e-06,
+      "loss": -0.0152,
+      "num_tokens": 2160777.0,
+      "reward": 0.675000011920929,
+      "reward_std": 2.953065872192383,
+      "rewards/alfworld_rollout_reward_func/mean": 0.675000011920929,
+      "rewards/alfworld_rollout_reward_func/std": 3.6408443450927734,
+      "sampling/importance_sampling_ratio/max": 2.5711066722869873,
+      "sampling/importance_sampling_ratio/mean": 0.6986918449401855,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.6658549308776855,
+      "sampling/sampling_logp_difference/mean": 0.02768370695412159,
+      "step": 24,
+      "step_time": 286.79983711899695
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.140625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 436.0,
+      "completions/mean_length": 176.5703125,
+      "completions/mean_terminated_length": 121.68181610107422,
+      "completions/min_length": 54.0,
+      "completions/min_terminated_length": 54.0,
+      "entropy": 0.4625844620168209,
+      "epoch": 0.4032258064516129,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1729527860879898,
+      "kl": 0.046662998385727406,
+      "learning_rate": 3.887913511553917e-06,
+      "loss": 0.0258,
+      "num_tokens": 2247234.0,
+      "reward": 0.8726562261581421,
+      "reward_std": 3.0254807472229004,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8726562261581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.7554731369018555,
+      "sampling/importance_sampling_ratio/max": 2.884495496749878,
+      "sampling/importance_sampling_ratio/mean": 0.7411025762557983,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.431777000427246,
+      "sampling/sampling_logp_difference/mean": 0.02642218954861164,
+      "step": 25,
+      "step_time": 331.329842252002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1953125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 437.0,
+      "completions/mean_length": 203.78125,
+      "completions/mean_terminated_length": 128.9708709716797,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
+      "entropy": 0.4392157681286335,
+      "epoch": 0.41935483870967744,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18243148922920227,
+      "kl": 0.03993735695257783,
+      "learning_rate": 3.0155234663401146e-06,
+      "loss": 0.1456,
+      "num_tokens": 2332769.0,
+      "reward": 2.0328125953674316,
+      "reward_std": 4.510900020599365,
+      "rewards/alfworld_rollout_reward_func/mean": 2.0328123569488525,
+      "rewards/alfworld_rollout_reward_func/std": 4.624057769775391,
+      "sampling/importance_sampling_ratio/max": 2.9809186458587646,
+      "sampling/importance_sampling_ratio/mean": 0.8058295845985413,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.739076852798462,
+      "sampling/sampling_logp_difference/mean": 0.025428924709558487,
+      "step": 26,
+      "step_time": 262.9066606970009
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 387.0,
+      "completions/mean_length": 173.7890625,
+      "completions/mean_terminated_length": 125.47322082519531,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.46685169637203217,
+      "epoch": 0.43548387096774194,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17903129756450653,
+      "kl": 0.046850728802382946,
+      "learning_rate": 2.240456984909049e-06,
+      "loss": 0.0513,
+      "num_tokens": 2415919.0,
+      "reward": 1.5,
+      "reward_std": 4.049795627593994,
+      "rewards/alfworld_rollout_reward_func/mean": 1.5,
+      "rewards/alfworld_rollout_reward_func/std": 4.301400661468506,
+      "sampling/importance_sampling_ratio/max": 2.8721108436584473,
+      "sampling/importance_sampling_ratio/mean": 0.813679575920105,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3308191299438477,
+      "sampling/sampling_logp_difference/mean": 0.027277380228042603,
+      "step": 27,
+      "step_time": 287.4597008519995
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1640625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 253.0,
+      "completions/mean_length": 182.4140625,
+      "completions/mean_terminated_length": 117.72896575927734,
+      "completions/min_length": 67.0,
+      "completions/min_terminated_length": 67.0,
+      "entropy": 0.4677879251539707,
+      "epoch": 0.45161290322580644,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13941031694412231,
+      "kl": 0.042057349579408765,
+      "learning_rate": 1.570667298192724e-06,
+      "loss": 0.0048,
+      "num_tokens": 2500100.0,
+      "reward": 0.5835937261581421,
+      "reward_std": 2.6255595684051514,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5835937261581421,
+      "rewards/alfworld_rollout_reward_func/std": 3.450979232788086,
+      "sampling/importance_sampling_ratio/max": 2.854583501815796,
+      "sampling/importance_sampling_ratio/mean": 0.7393490076065063,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 5.095232963562012,
+      "sampling/sampling_logp_difference/mean": 0.027033191174268723,
+      "step": 28,
+      "step_time": 285.79294812399894
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2109375,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 479.0,
+      "completions/mean_length": 212.203125,
+      "completions/mean_terminated_length": 132.0594024658203,
+      "completions/min_length": 58.0,
+      "completions/min_terminated_length": 58.0,
+      "entropy": 0.4555159918963909,
+      "epoch": 0.46774193548387094,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16522692143917084,
+      "kl": 0.04728960315696895,
+      "learning_rate": 1.0130273547471176e-06,
+      "loss": 0.0531,
+      "num_tokens": 2589663.0,
+      "reward": 0.828906238079071,
+      "reward_std": 3.300809860229492,
+      "rewards/alfworld_rollout_reward_func/mean": 0.8289061784744263,
+      "rewards/alfworld_rollout_reward_func/std": 3.700098991394043,
+      "sampling/importance_sampling_ratio/max": 2.9519269466400146,
+      "sampling/importance_sampling_ratio/mean": 0.7473565340042114,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.6239898204803467,
+      "sampling/sampling_logp_difference/mean": 0.024686329066753387,
+      "step": 29,
+      "step_time": 316.5334539249998
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.140625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 404.0,
+      "completions/mean_length": 182.484375,
+      "completions/mean_terminated_length": 128.56362915039062,
+      "completions/min_length": 69.0,
+      "completions/min_terminated_length": 69.0,
+      "entropy": 0.4658549278974533,
+      "epoch": 0.4838709677419355,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1560395509004593,
+      "kl": 0.05294563015922904,
+      "learning_rate": 5.732592949993898e-07,
+      "loss": 0.0688,
+      "num_tokens": 2675234.0,
+      "reward": 0.84765625,
+      "reward_std": 3.315725803375244,
+      "rewards/alfworld_rollout_reward_func/mean": 0.84765625,
+      "rewards/alfworld_rollout_reward_func/std": 3.8803963661193848,
+      "sampling/importance_sampling_ratio/max": 2.622981548309326,
+      "sampling/importance_sampling_ratio/mean": 0.7878429293632507,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.5301332473754883,
+      "sampling/sampling_logp_difference/mean": 0.026188407093286514,
+      "step": 30,
+      "step_time": 298.4171370410004
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1328125,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 446.0,
+      "completions/mean_length": 179.0078125,
+      "completions/mean_terminated_length": 128.00901794433594,
+      "completions/min_length": 63.0,
+      "completions/min_terminated_length": 63.0,
+      "entropy": 0.5069509670138359,
+      "epoch": 0.5,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20257869362831116,
+      "kl": 0.055831174831837416,
+      "learning_rate": 2.5587573434381895e-07,
+      "loss": 0.0626,
+      "num_tokens": 2762673.0,
+      "reward": 0.6875,
+      "reward_std": 3.218989372253418,
+      "rewards/alfworld_rollout_reward_func/mean": 0.6875,
+      "rewards/alfworld_rollout_reward_func/std": 3.5843491554260254,
+      "sampling/importance_sampling_ratio/max": 2.8179516792297363,
+      "sampling/importance_sampling_ratio/mean": 0.7038029432296753,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.102773666381836,
+      "sampling/sampling_logp_difference/mean": 0.029051225632429123,
+      "step": 31,
+      "step_time": 305.9211267239989
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.140625,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 486.0,
+      "completions/mean_length": 193.7734375,
+      "completions/mean_terminated_length": 141.6999969482422,
+      "completions/min_length": 64.0,
+      "completions/min_terminated_length": 64.0,
+      "entropy": 0.50175466760993,
+      "epoch": 0.5161290322580645,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14924179017543793,
+      "kl": 0.03636577841825783,
+      "learning_rate": 6.413345760131057e-08,
+      "loss": 0.1286,
+      "num_tokens": 2849119.0,
+      "reward": 0.9367187023162842,
+      "reward_std": 3.7098560333251953,
+      "rewards/alfworld_rollout_reward_func/mean": 0.9367187023162842,
+      "rewards/alfworld_rollout_reward_func/std": 3.8140616416931152,
+      "sampling/importance_sampling_ratio/max": 2.708343744277954,
+      "sampling/importance_sampling_ratio/mean": 0.8295129537582397,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.4670004844665527,
+      "sampling/sampling_logp_difference/mean": 0.027453571557998657,
+      "step": 32,
+      "step_time": 294.9108767129983
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 32,
+  "num_input_tokens_seen": 2849119,
+  "num_train_epochs": 1,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-32/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe31dee94209c1973566ddbbc45e4eaa5758a9402713eebce4f579426cfdc6a3
+size 9361

config.json

@@ -0,0 +1,66 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151645,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 70,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

debug.log

@@ -0,0 +1,13 @@
+[2026-01-26 07:21:36,287] [WARNING] [py.warnings._showwarnmsg:110] [PID:300] /root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/backends/__init__.py:46: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.)
+  self.setter(val)
+
+
+
+[2026-01-26 07:21:58,152] [WARNING] [py.warnings._showwarnmsg:110] [PID:300] <string>:246: FutureWarning: The `max_prompt_length` argument is deprecated and will be removed in version 0.28.0. You should instead filter your dataset before training to ensure that prompts do not exceed your desired length.
+
+[2026-01-26 07:21:58,799] [WARNING] [py.warnings._showwarnmsg:110] [PID:300] /workspace/axolotl/src/axolotl/core/trainers/mixins/optimizer.py:209: UserWarning: You are importing from 'rollout_func', which is an experimental feature. This API may change or be removed at any time without prior notice. Silence this warning by setting environment variable TRL_EXPERIMENTAL_SILENCE=1.
+  super().__init__(*args, **kwargs)
+
+2026-01-26 07:22:41,128 - INFO - autotuner.py:256 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
+2026-01-26 07:22:41,141 - INFO - autotuner.py:262 - flashinfer.jit: [Autotuner]: Autotuning process ends
+AlfWorld endpoint initialized on rank 1 at http://environment-server-1:8000

ref/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

ref/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a73e8a3357e88549443ed67169c44587d8849b718265a36909b7373da2322a00
+size 59934640

special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}