Upload task output 1

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,210 @@
+---
+base_model: Qwen/Qwen2.5-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:/cache/models/Qwen--Qwen2.5-3B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:137c7779b7d79c0a055cdfc8d4733150a72a9d00dc69cce5b2004a1a395d7fb6
+size 119801528

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:339306587efc75e93b6aa072198ee9462e266edd0eb1b0db69e78067a0d92522
+size 61392581

ref/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj",
+    "up_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

ref/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f5c43d294c90dcd8aa53b8f4358d140af282467938de22109f1ded1570a6cc9
+size 59934640

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f328d924e63f6fa99a7ad32932184a17174c355a0b58f761130deaad0631a8d
+size 14917

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bf6c99a153c612933624ce605c135e0dd3c6296a9d76861879874b2e26da0aa
+size 14917

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2b7f9ea74826c1502a61c3de41cc35c375d36aa942e516022944ddcf38f06ba
+size 1465

special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": "<|im_end|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,2014 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.24,
+  "eval_steps": 500,
+  "global_step": 60,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9999.0,
+      "completions/max_terminated_length": 9999.0,
+      "completions/mean_length": 4554.453125,
+      "completions/mean_terminated_length": 4554.453125,
+      "completions/min_length": 551.0,
+      "completions/min_terminated_length": 551.0,
+      "entropy": 0.31497739674523473,
+      "epoch": 0.004,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.27671217918395996,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.1899,
+      "num_tokens": 332413.0,
+      "reward": 0.5768750309944153,
+      "reward_std": 0.4988396167755127,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5768749713897705,
+      "rewards/alfworld_rollout_reward_func/std": 0.5170210003852844,
+      "sampling/importance_sampling_ratio/max": 2.406592845916748,
+      "sampling/importance_sampling_ratio/mean": 0.6213822364807129,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 55.82071304321289,
+      "sampling/sampling_logp_difference/mean": 0.023054132238030434,
+      "step": 1,
+      "step_time": 293.2645074960601
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8320.0,
+      "completions/max_terminated_length": 8320.0,
+      "completions/mean_length": 5251.21875,
+      "completions/mean_terminated_length": 5251.21875,
+      "completions/min_length": 1861.0,
+      "completions/min_terminated_length": 1861.0,
+      "entropy": 0.3733639745041728,
+      "epoch": 0.008,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.26362094283103943,
+      "kl": 0.0,
+      "learning_rate": 1.5e-06,
+      "loss": 0.0961,
+      "num_tokens": 704651.0,
+      "reward": 0.13640625774860382,
+      "reward_std": 0.43152421712875366,
+      "rewards/alfworld_rollout_reward_func/mean": 0.13640624284744263,
+      "rewards/alfworld_rollout_reward_func/std": 0.4294232726097107,
+      "sampling/importance_sampling_ratio/max": 2.881873846054077,
+      "sampling/importance_sampling_ratio/mean": 0.4364074170589447,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.0964527130126953,
+      "sampling/sampling_logp_difference/mean": 0.019750293344259262,
+      "step": 2,
+      "step_time": 281.28423442596977
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10722.0,
+      "completions/max_terminated_length": 10722.0,
+      "completions/mean_length": 6977.828125,
+      "completions/mean_terminated_length": 6977.828125,
+      "completions/min_length": 1154.0,
+      "completions/min_terminated_length": 1154.0,
+      "entropy": 0.3223036811687052,
+      "epoch": 0.012,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19866329431533813,
+      "kl": 0.0017119673575507477,
+      "learning_rate": 3e-06,
+      "loss": 0.2902,
+      "num_tokens": 1196416.0,
+      "reward": 0.24703124165534973,
+      "reward_std": 0.4671919345855713,
+      "rewards/alfworld_rollout_reward_func/mean": 0.24703125655651093,
+      "rewards/alfworld_rollout_reward_func/std": 0.4794430732727051,
+      "sampling/importance_sampling_ratio/max": 2.9424545764923096,
+      "sampling/importance_sampling_ratio/mean": 0.46759214997291565,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.1991941928863525,
+      "sampling/sampling_logp_difference/mean": 0.019809413701295853,
+      "step": 3,
+      "step_time": 336.2479823809699
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9874.0,
+      "completions/max_terminated_length": 9874.0,
+      "completions/mean_length": 6289.03125,
+      "completions/mean_terminated_length": 6289.03125,
+      "completions/min_length": 3976.0,
+      "completions/min_terminated_length": 3976.0,
+      "entropy": 0.33153627486899495,
+      "epoch": 0.016,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20725910365581512,
+      "kl": 0.0019219094938307535,
+      "learning_rate": 2.9980747607565792e-06,
+      "loss": -0.0184,
+      "num_tokens": 1636162.0,
+      "reward": 0.0007812529802322388,
+      "reward_std": 0.17323368787765503,
+      "rewards/alfworld_rollout_reward_func/mean": 0.0007812529802322388,
+      "rewards/alfworld_rollout_reward_func/std": 0.2191564291715622,
+      "sampling/importance_sampling_ratio/max": 2.584482192993164,
+      "sampling/importance_sampling_ratio/mean": 0.38795873522758484,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.953186511993408,
+      "sampling/sampling_logp_difference/mean": 0.020250197499990463,
+      "step": 4,
+      "step_time": 313.9296533779707
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11285.0,
+      "completions/max_terminated_length": 11285.0,
+      "completions/mean_length": 7613.421875,
+      "completions/mean_terminated_length": 7613.421875,
+      "completions/min_length": 2227.0,
+      "completions/min_terminated_length": 2227.0,
+      "entropy": 0.34567126259207726,
+      "epoch": 0.02,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17643821239471436,
+      "kl": 0.0016085030692920554,
+      "learning_rate": 2.9923039850878425e-06,
+      "loss": 0.0963,
+      "num_tokens": 2170909.0,
+      "reward": 0.1875,
+      "reward_std": 0.4111865162849426,
+      "rewards/alfworld_rollout_reward_func/mean": 0.1875,
+      "rewards/alfworld_rollout_reward_func/std": 0.445791631937027,
+      "sampling/importance_sampling_ratio/max": 2.329904079437256,
+      "sampling/importance_sampling_ratio/mean": 0.4071568250656128,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 51.395050048828125,
+      "sampling/sampling_logp_difference/mean": 0.02332460880279541,
+      "step": 5,
+      "step_time": 396.88119790001656
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10068.0,
+      "completions/max_terminated_length": 10068.0,
+      "completions/mean_length": 7365.96875,
+      "completions/mean_terminated_length": 7365.96875,
+      "completions/min_length": 2236.0,
+      "completions/min_terminated_length": 2236.0,
+      "entropy": 0.33807497285306454,
+      "epoch": 0.024,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2105790078639984,
+      "kl": 0.002014314475673018,
+      "learning_rate": 2.982702486492167e-06,
+      "loss": 0.1834,
+      "num_tokens": 2690011.0,
+      "reward": 0.18296873569488525,
+      "reward_std": 0.34179380536079407,
+      "rewards/alfworld_rollout_reward_func/mean": 0.18296875059604645,
+      "rewards/alfworld_rollout_reward_func/std": 0.4501417279243469,
+      "sampling/importance_sampling_ratio/max": 2.981616735458374,
+      "sampling/importance_sampling_ratio/mean": 0.4046768844127655,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 5.697381019592285,
+      "sampling/sampling_logp_difference/mean": 0.020046193152666092,
+      "step": 6,
+      "step_time": 348.1233048220165
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9846.0,
+      "completions/max_terminated_length": 9846.0,
+      "completions/mean_length": 6610.140625,
+      "completions/mean_terminated_length": 6610.140625,
+      "completions/min_length": 808.0,
+      "completions/min_terminated_length": 808.0,
+      "entropy": 0.3595086098648608,
+      "epoch": 0.028,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.218948632478714,
+      "kl": 0.0016301772157021333,
+      "learning_rate": 2.969294911878742e-06,
+      "loss": 0.0717,
+      "num_tokens": 3155044.0,
+      "reward": 0.171875,
+      "reward_std": 0.41918015480041504,
+      "rewards/alfworld_rollout_reward_func/mean": 0.171875,
+      "rewards/alfworld_rollout_reward_func/std": 0.42613014578819275,
+      "sampling/importance_sampling_ratio/max": 2.5824413299560547,
+      "sampling/importance_sampling_ratio/mean": 0.3877456784248352,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.626394271850586,
+      "sampling/sampling_logp_difference/mean": 0.020398743450641632,
+      "step": 7,
+      "step_time": 351.79137006499514
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11160.0,
+      "completions/max_terminated_length": 11160.0,
+      "completions/mean_length": 7036.953125,
+      "completions/mean_terminated_length": 7036.953125,
+      "completions/min_length": 1917.0,
+      "completions/min_terminated_length": 1917.0,
+      "entropy": 0.3341470160521567,
+      "epoch": 0.032,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2093495875597,
+      "kl": 0.0016681601082382258,
+      "learning_rate": 2.9521156782993067e-06,
+      "loss": -0.0439,
+      "num_tokens": 3658497.0,
+      "reward": 0.41718751192092896,
+      "reward_std": 0.5516971349716187,
+      "rewards/alfworld_rollout_reward_func/mean": 0.41718751192092896,
+      "rewards/alfworld_rollout_reward_func/std": 0.5447016954421997,
+      "sampling/importance_sampling_ratio/max": 2.776736259460449,
+      "sampling/importance_sampling_ratio/mean": 0.4658258259296417,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 56.51785659790039,
+      "sampling/sampling_logp_difference/mean": 0.022869806736707687,
+      "step": 8,
+      "step_time": 329.31609062396456
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9616.0,
+      "completions/max_terminated_length": 9616.0,
+      "completions/mean_length": 8430.71875,
+      "completions/mean_terminated_length": 8430.71875,
+      "completions/min_length": 4983.0,
+      "completions/min_terminated_length": 4983.0,
+      "entropy": 0.3272924036718905,
+      "epoch": 0.036,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5628678202629089,
+      "kl": 0.0017659663353697397,
+      "learning_rate": 2.9312088846000733e-06,
+      "loss": -0.2308,
+      "num_tokens": 4245423.0,
+      "reward": 0.032499998807907104,
+      "reward_std": 0.24380016326904297,
+      "rewards/alfworld_rollout_reward_func/mean": 0.032499998807907104,
+      "rewards/alfworld_rollout_reward_func/std": 0.2736351490020752,
+      "sampling/importance_sampling_ratio/max": 2.395029067993164,
+      "sampling/importance_sampling_ratio/mean": 0.3855500817298889,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.2965643405914307,
+      "sampling/sampling_logp_difference/mean": 0.020408470183610916,
+      "step": 9,
+      "step_time": 363.9418078730232
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6815.0,
+      "completions/max_terminated_length": 6815.0,
+      "completions/mean_length": 4436.71875,
+      "completions/mean_terminated_length": 4436.71875,
+      "completions/min_length": 1645.0,
+      "completions/min_terminated_length": 1645.0,
+      "entropy": 0.3489171755500138,
+      "epoch": 0.04,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.28452208638191223,
+      "kl": 0.0015779180102981627,
+      "learning_rate": 2.906628198220621e-06,
+      "loss": 0.1568,
+      "num_tokens": 4563965.0,
+      "reward": 0.22546875476837158,
+      "reward_std": 0.2872272729873657,
+      "rewards/alfworld_rollout_reward_func/mean": 0.22546875476837158,
+      "rewards/alfworld_rollout_reward_func/std": 0.4715692698955536,
+      "sampling/importance_sampling_ratio/max": 2.9534051418304443,
+      "sampling/importance_sampling_ratio/mean": 0.4985596835613251,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.560643196105957,
+      "sampling/sampling_logp_difference/mean": 0.020065899938344955,
+      "step": 10,
+      "step_time": 263.016338995003
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9583.0,
+      "completions/max_terminated_length": 9583.0,
+      "completions/mean_length": 6498.203125,
+      "completions/mean_terminated_length": 6498.203125,
+      "completions/min_length": 630.0,
+      "completions/min_terminated_length": 630.0,
+      "entropy": 0.34817238664254546,
+      "epoch": 0.044,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.218714639544487,
+      "kl": 0.0016383687889174325,
+      "learning_rate": 2.878436717430346e-06,
+      "loss": 0.187,
+      "num_tokens": 5023626.0,
+      "reward": 0.18468749523162842,
+      "reward_std": 0.4014522433280945,
+      "rewards/alfworld_rollout_reward_func/mean": 0.18468749523162842,
+      "rewards/alfworld_rollout_reward_func/std": 0.4519691467285156,
+      "sampling/importance_sampling_ratio/max": 2.0133447647094727,
+      "sampling/importance_sampling_ratio/mean": 0.48143666982650757,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.304382801055908,
+      "sampling/sampling_logp_difference/mean": 0.020289719104766846,
+      "step": 11,
+      "step_time": 358.22142862899636
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10214.0,
+      "completions/max_terminated_length": 10214.0,
+      "completions/mean_length": 8234.421875,
+      "completions/mean_terminated_length": 8234.421875,
+      "completions/min_length": 2923.0,
+      "completions/min_terminated_length": 2923.0,
+      "entropy": 0.2907608295790851,
+      "epoch": 0.048,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.26984360814094543,
+      "kl": 0.0016117684463097248,
+      "learning_rate": 2.846706809356113e-06,
+      "loss": 0.0146,
+      "num_tokens": 5600485.0,
+      "reward": 0.11812499165534973,
+      "reward_std": 0.35846489667892456,
+      "rewards/alfworld_rollout_reward_func/mean": 0.11812499910593033,
+      "rewards/alfworld_rollout_reward_func/std": 0.37784212827682495,
+      "sampling/importance_sampling_ratio/max": 2.9857699871063232,
+      "sampling/importance_sampling_ratio/mean": 0.5836184024810791,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 57.58735656738281,
+      "sampling/sampling_logp_difference/mean": 0.01956798881292343,
+      "step": 12,
+      "step_time": 354.4028672409913
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 5514.0,
+      "completions/max_terminated_length": 5514.0,
+      "completions/mean_length": 2816.578125,
+      "completions/mean_terminated_length": 2816.578125,
+      "completions/min_length": 899.0,
+      "completions/min_terminated_length": 899.0,
+      "entropy": 0.37038205191493034,
+      "epoch": 0.052,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.21182094514369965,
+      "kl": 0.0029761453515675385,
+      "learning_rate": 2.811519924216873e-06,
+      "loss": -0.0192,
+      "num_tokens": 5812938.0,
+      "reward": 0.7221874594688416,
+      "reward_std": 0.4521871507167816,
+      "rewards/alfworld_rollout_reward_func/mean": 0.7221875190734863,
+      "rewards/alfworld_rollout_reward_func/std": 0.4576561152935028,
+      "sampling/importance_sampling_ratio/max": 2.8400075435638428,
+      "sampling/importance_sampling_ratio/mean": 0.3720712661743164,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 54.35240936279297,
+      "sampling/sampling_logp_difference/mean": 0.04739231616258621,
+      "step": 13,
+      "step_time": 176.69907861396496
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11718.0,
+      "completions/max_terminated_length": 11718.0,
+      "completions/mean_length": 8725.5,
+      "completions/mean_terminated_length": 8725.5,
+      "completions/min_length": 2386.0,
+      "completions/min_terminated_length": 2386.0,
+      "entropy": 0.34669697750359774,
+      "epoch": 0.056,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2509883940219879,
+      "kl": 0.002024810473812977,
+      "learning_rate": 2.7729663862421267e-06,
+      "loss": 0.3618,
+      "num_tokens": 6423274.0,
+      "reward": 0.1317187398672104,
+      "reward_std": 0.27824532985687256,
+      "rewards/alfworld_rollout_reward_func/mean": 0.13171875476837158,
+      "rewards/alfworld_rollout_reward_func/std": 0.3909270763397217,
+      "sampling/importance_sampling_ratio/max": 2.5765702724456787,
+      "sampling/importance_sampling_ratio/mean": 0.4724542498588562,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.3789246082305908,
+      "sampling/sampling_logp_difference/mean": 0.021418465301394463,
+      "step": 14,
+      "step_time": 375.7027179340075
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9823.0,
+      "completions/max_terminated_length": 9823.0,
+      "completions/mean_length": 3761.203125,
+      "completions/mean_terminated_length": 3761.203125,
+      "completions/min_length": 710.0,
+      "completions/min_terminated_length": 710.0,
+      "entropy": 0.2707831831648946,
+      "epoch": 0.06,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18436089158058167,
+      "kl": 0.0015840599007788114,
+      "learning_rate": 2.731145161810915e-06,
+      "loss": 0.0515,
+      "num_tokens": 6700631.0,
+      "reward": 0.6873437166213989,
+      "reward_std": 0.3868533968925476,
+      "rewards/alfworld_rollout_reward_func/mean": 0.6873437166213989,
+      "rewards/alfworld_rollout_reward_func/std": 0.48384782671928406,
+      "sampling/importance_sampling_ratio/max": 2.213698625564575,
+      "sampling/importance_sampling_ratio/mean": 0.573298454284668,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 56.33928680419922,
+      "sampling/sampling_logp_difference/mean": 0.019625678658485413,
+      "step": 15,
+      "step_time": 284.1389054669853
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6007.0,
+      "completions/max_terminated_length": 6007.0,
+      "completions/mean_length": 4786.0,
+      "completions/mean_terminated_length": 4786.0,
+      "completions/min_length": 1835.0,
+      "completions/min_terminated_length": 1835.0,
+      "entropy": 0.42686293832957745,
+      "epoch": 0.064,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5065476298332214,
+      "kl": 0.004679446868976811,
+      "learning_rate": 2.6861636054065477e-06,
+      "loss": -0.0551,
+      "num_tokens": 7042679.0,
+      "reward": 0.23718750476837158,
+      "reward_std": 0.3366313576698303,
+      "rewards/alfworld_rollout_reward_func/mean": 0.23718750476837158,
+      "rewards/alfworld_rollout_reward_func/std": 0.4672325551509857,
+      "sampling/importance_sampling_ratio/max": 1.987215518951416,
+      "sampling/importance_sampling_ratio/mean": 0.3825452923774719,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 48.08681869506836,
+      "sampling/sampling_logp_difference/mean": 0.02753990702331066,
+      "step": 16,
+      "step_time": 253.53596857101365
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 13026.0,
+      "completions/max_terminated_length": 13026.0,
+      "completions/mean_length": 7801.28125,
+      "completions/mean_terminated_length": 7801.28125,
+      "completions/min_length": 996.0,
+      "completions/min_terminated_length": 996.0,
+      "entropy": 0.38106839125975966,
+      "epoch": 0.068,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13401441276073456,
+      "kl": 0.0019643659798020963,
+      "learning_rate": 2.6381371840391863e-06,
+      "loss": 0.2576,
+      "num_tokens": 7587913.0,
+      "reward": 0.08687499165534973,
+      "reward_std": 0.31602421402931213,
+      "rewards/alfworld_rollout_reward_func/mean": 0.08687499165534973,
+      "rewards/alfworld_rollout_reward_func/std": 0.37281376123428345,
+      "sampling/importance_sampling_ratio/max": 1.947364330291748,
+      "sampling/importance_sampling_ratio/mean": 0.346585750579834,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 50.33766174316406,
+      "sampling/sampling_logp_difference/mean": 0.024903230369091034,
+      "step": 17,
+      "step_time": 438.1667110319977
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 7932.0,
+      "completions/max_terminated_length": 7932.0,
+      "completions/mean_length": 5135.984375,
+      "completions/mean_terminated_length": 5135.984375,
+      "completions/min_length": 2158.0,
+      "completions/min_terminated_length": 2158.0,
+      "entropy": 0.38822565879672766,
+      "epoch": 0.072,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2989322245121002,
+      "kl": 0.001747572790918639,
+      "learning_rate": 2.58718918084368e-06,
+      "loss": -0.1124,
+      "num_tokens": 7953544.0,
+      "reward": 0.277031272649765,
+      "reward_std": 0.3521837890148163,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2770312428474426,
+      "rewards/alfworld_rollout_reward_func/std": 0.4784753620624542,
+      "sampling/importance_sampling_ratio/max": 2.394239902496338,
+      "sampling/importance_sampling_ratio/mean": 0.37263223528862,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.304655075073242,
+      "sampling/sampling_logp_difference/mean": 0.02233094722032547,
+      "step": 18,
+      "step_time": 256.8168452990067
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 14106.0,
+      "completions/max_terminated_length": 14106.0,
+      "completions/mean_length": 10424.546875,
+      "completions/mean_terminated_length": 10424.546875,
+      "completions/min_length": 2464.0,
+      "completions/min_terminated_length": 2464.0,
+      "entropy": 0.3585456218570471,
+      "epoch": 0.076,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16573461890220642,
+      "kl": 0.0020609867096936796,
+      "learning_rate": 2.53345037861353e-06,
+      "loss": 0.1829,
+      "num_tokens": 8681771.0,
+      "reward": 0.17765626311302185,
+      "reward_std": 0.4618713855743408,
+      "rewards/alfworld_rollout_reward_func/mean": 0.17765626311302185,
+      "rewards/alfworld_rollout_reward_func/std": 0.46532371640205383,
+      "sampling/importance_sampling_ratio/max": 2.4118423461914062,
+      "sampling/importance_sampling_ratio/mean": 0.23810628056526184,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 54.374839782714844,
+      "sampling/sampling_logp_difference/mean": 0.026143547147512436,
+      "step": 19,
+      "step_time": 470.0736003620259
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6909.0,
+      "completions/max_terminated_length": 6909.0,
+      "completions/mean_length": 3685.25,
+      "completions/mean_terminated_length": 3685.25,
+      "completions/min_length": 621.0,
+      "completions/min_terminated_length": 621.0,
+      "entropy": 0.34135088324546814,
+      "epoch": 0.08,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1747477948665619,
+      "kl": 0.001954720755747985,
+      "learning_rate": 2.477058724083334e-06,
+      "loss": 0.1728,
+      "num_tokens": 8952283.0,
+      "reward": 0.4632812738418579,
+      "reward_std": 0.3338242769241333,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4632812738418579,
+      "rewards/alfworld_rollout_reward_func/std": 0.5396590828895569,
+      "sampling/importance_sampling_ratio/max": 2.4462101459503174,
+      "sampling/importance_sampling_ratio/mean": 0.5562223792076111,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 57.48880386352539,
+      "sampling/sampling_logp_difference/mean": 0.025760911405086517,
+      "step": 20,
+      "step_time": 243.09781286600628
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9257.0,
+      "completions/max_terminated_length": 9257.0,
+      "completions/mean_length": 5892.0,
+      "completions/mean_terminated_length": 5892.0,
+      "completions/min_length": 2008.0,
+      "completions/min_terminated_length": 2008.0,
+      "entropy": 0.317400851752609,
+      "epoch": 0.084,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2818284034729004,
+      "kl": 0.0034878615733759943,
+      "learning_rate": 2.4181589738214946e-06,
+      "loss": -0.1166,
+      "num_tokens": 9367611.0,
+      "reward": 0.13109375536441803,
+      "reward_std": 0.4177722930908203,
+      "rewards/alfworld_rollout_reward_func/mean": 0.13109375536441803,
+      "rewards/alfworld_rollout_reward_func/std": 0.41239187121391296,
+      "sampling/importance_sampling_ratio/max": 2.3060288429260254,
+      "sampling/importance_sampling_ratio/mean": 0.40336811542510986,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.0395994186401367,
+      "sampling/sampling_logp_difference/mean": 0.01938227005302906,
+      "step": 21,
+      "step_time": 322.3771183890203
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 5404.0,
+      "completions/max_terminated_length": 5404.0,
+      "completions/mean_length": 2908.671875,
+      "completions/mean_terminated_length": 2908.671875,
+      "completions/min_length": 504.0,
+      "completions/min_terminated_length": 504.0,
+      "entropy": 0.33527113823220134,
+      "epoch": 0.088,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.28914618492126465,
+      "kl": 0.0015201076712401118,
+      "learning_rate": 2.3569023226421886e-06,
+      "loss": 0.4174,
+      "num_tokens": 9586310.0,
+      "reward": 0.5940625071525574,
+      "reward_std": 0.4888940155506134,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5940624475479126,
+      "rewards/alfworld_rollout_reward_func/std": 0.5188471078872681,
+      "sampling/importance_sampling_ratio/max": 2.96110200881958,
+      "sampling/importance_sampling_ratio/mean": 0.5440493822097778,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2356529235839844,
+      "sampling/sampling_logp_difference/mean": 0.02087477594614029,
+      "step": 22,
+      "step_time": 220.2282471570361
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11264.0,
+      "completions/max_terminated_length": 11264.0,
+      "completions/mean_length": 4416.765625,
+      "completions/mean_terminated_length": 4416.765625,
+      "completions/min_length": 548.0,
+      "completions/min_terminated_length": 548.0,
+      "entropy": 0.37216851580888033,
+      "epoch": 0.092,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.21720609068870544,
+      "kl": 0.0015780055618961342,
+      "learning_rate": 2.2934460154904436e-06,
+      "loss": -0.1388,
+      "num_tokens": 9904855.0,
+      "reward": 0.44140625,
+      "reward_std": 0.5448532104492188,
+      "rewards/alfworld_rollout_reward_func/mean": 0.44140625,
+      "rewards/alfworld_rollout_reward_func/std": 0.5440409779548645,
+      "sampling/importance_sampling_ratio/max": 2.4073739051818848,
+      "sampling/importance_sampling_ratio/mean": 0.5275839567184448,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.4305299520492554,
+      "sampling/sampling_logp_difference/mean": 0.020915336906909943,
+      "step": 23,
+      "step_time": 276.8828063480614
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8603.0,
+      "completions/max_terminated_length": 8603.0,
+      "completions/mean_length": 5866.765625,
+      "completions/mean_terminated_length": 5866.765625,
+      "completions/min_length": 638.0,
+      "completions/min_terminated_length": 638.0,
+      "entropy": 0.38163668289780617,
+      "epoch": 0.096,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1996874064207077,
+      "kl": 0.0022600124138989486,
+      "learning_rate": 2.227952943796622e-06,
+      "loss": 0.0203,
+      "num_tokens": 10321992.0,
+      "reward": 0.3020312488079071,
+      "reward_std": 0.32399436831474304,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3020312488079071,
+      "rewards/alfworld_rollout_reward_func/std": 0.4859981834888458,
+      "sampling/importance_sampling_ratio/max": 2.0370097160339355,
+      "sampling/importance_sampling_ratio/mean": 0.322752445936203,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.6053457260131836,
+      "sampling/sampling_logp_difference/mean": 0.023195665329694748,
+      "step": 24,
+      "step_time": 326.98625825899944
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.015625,
+      "completions/max_length": 11175.0,
+      "completions/max_terminated_length": 11175.0,
+      "completions/mean_length": 8415.28125,
+      "completions/mean_terminated_length": 8421.2705078125,
+      "completions/min_length": 3084.0,
+      "completions/min_terminated_length": 3084.0,
+      "entropy": 0.3829211392439902,
+      "epoch": 0.1,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15396606922149658,
+      "kl": 0.0019763640011660755,
+      "learning_rate": 2.160591227336452e-06,
+      "loss": 0.0503,
+      "num_tokens": 10909370.0,
+      "reward": 0.10234375298023224,
+      "reward_std": 0.32216501235961914,
+      "rewards/alfworld_rollout_reward_func/mean": 0.10234374552965164,
+      "rewards/alfworld_rollout_reward_func/std": 0.3721825182437897,
+      "sampling/importance_sampling_ratio/max": 2.063018560409546,
+      "sampling/importance_sampling_ratio/mean": 0.26939719915390015,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 6.286721229553223,
+      "sampling/sampling_logp_difference/mean": 0.021831313148140907,
+      "step": 25,
+      "step_time": 399.5593527499732
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6961.0,
+      "completions/max_terminated_length": 6961.0,
+      "completions/mean_length": 4205.625,
+      "completions/mean_terminated_length": 4205.625,
+      "completions/min_length": 601.0,
+      "completions/min_terminated_length": 601.0,
+      "entropy": 0.3060068585909903,
+      "epoch": 0.104,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.26218128204345703,
+      "kl": 0.0015425982546730665,
+      "learning_rate": 2.091533782669978e-06,
+      "loss": 0.2487,
+      "num_tokens": 11216162.0,
+      "reward": 0.43156248331069946,
+      "reward_std": 0.4275887608528137,
+      "rewards/alfworld_rollout_reward_func/mean": 0.43156248331069946,
+      "rewards/alfworld_rollout_reward_func/std": 0.5302896499633789,
+      "sampling/importance_sampling_ratio/max": 2.158494472503662,
+      "sampling/importance_sampling_ratio/mean": 0.5278518795967102,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.369166374206543,
+      "sampling/sampling_logp_difference/mean": 0.019138336181640625,
+      "step": 26,
+      "step_time": 264.540892480014
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6184.0,
+      "completions/max_terminated_length": 6184.0,
+      "completions/mean_length": 4046.3125,
+      "completions/mean_terminated_length": 4046.3125,
+      "completions/min_length": 1216.0,
+      "completions/min_terminated_length": 1216.0,
+      "entropy": 0.35598616674542427,
+      "epoch": 0.108,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.568981409072876,
+      "kl": 0.001767738020134857,
+      "learning_rate": 2.0209578792672304e-06,
+      "loss": 0.764,
+      "num_tokens": 11507734.0,
+      "reward": 0.2854687571525574,
+      "reward_std": 0.27528271079063416,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2854687571525574,
+      "rewards/alfworld_rollout_reward_func/std": 0.4897940456867218,
+      "sampling/importance_sampling_ratio/max": 2.9929444789886475,
+      "sampling/importance_sampling_ratio/mean": 0.5946164131164551,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 7.642696380615234,
+      "sampling/sampling_logp_difference/mean": 0.021483708173036575,
+      "step": 27,
+      "step_time": 238.4292279400688
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11353.0,
+      "completions/max_terminated_length": 11353.0,
+      "completions/mean_length": 6085.71875,
+      "completions/mean_terminated_length": 6085.71875,
+      "completions/min_length": 442.0,
+      "completions/min_terminated_length": 442.0,
+      "entropy": 0.34554752754047513,
+      "epoch": 0.112,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20419467985630035,
+      "kl": 0.0022789983468101127,
+      "learning_rate": 1.9490446844600373e-06,
+      "loss": 0.0168,
+      "num_tokens": 11941124.0,
+      "reward": 0.3648437559604645,
+      "reward_std": 0.3334695100784302,
+      "rewards/alfworld_rollout_reward_func/mean": 0.36484378576278687,
+      "rewards/alfworld_rollout_reward_func/std": 0.5057353973388672,
+      "sampling/importance_sampling_ratio/max": 2.2758615016937256,
+      "sampling/importance_sampling_ratio/mean": 0.6538110375404358,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.1021443605422974,
+      "sampling/sampling_logp_difference/mean": 0.02113598957657814,
+      "step": 28,
+      "step_time": 396.3338410100696
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11143.0,
+      "completions/max_terminated_length": 11143.0,
+      "completions/mean_length": 8093.265625,
+      "completions/mean_terminated_length": 8093.265625,
+      "completions/min_length": 2100.0,
+      "completions/min_terminated_length": 2100.0,
+      "entropy": 0.30172505462542176,
+      "epoch": 0.116,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.23860569298267365,
+      "kl": 0.0015739148620923515,
+      "learning_rate": 1.875978798388081e-06,
+      "loss": 0.1597,
+      "num_tokens": 12510485.0,
+      "reward": 0.2574999928474426,
+      "reward_std": 0.27106666564941406,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2574999928474426,
+      "rewards/alfworld_rollout_reward_func/std": 0.4856676161289215,
+      "sampling/importance_sampling_ratio/max": 2.100985527038574,
+      "sampling/importance_sampling_ratio/mean": 0.4662199318408966,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.5363552570343018,
+      "sampling/sampling_logp_difference/mean": 0.0185923483222723,
+      "step": 29,
+      "step_time": 398.5051951840287
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11407.0,
+      "completions/max_terminated_length": 11407.0,
+      "completions/mean_length": 7339.609375,
+      "completions/mean_terminated_length": 7339.609375,
+      "completions/min_length": 3248.0,
+      "completions/min_terminated_length": 3248.0,
+      "entropy": 0.30220934841781855,
+      "epoch": 0.12,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2525518834590912,
+      "kl": 0.0015497524218517356,
+      "learning_rate": 1.8019477801329903e-06,
+      "loss": 0.0827,
+      "num_tokens": 13022268.0,
+      "reward": 0.15546876192092896,
+      "reward_std": 0.3895842730998993,
+      "rewards/alfworld_rollout_reward_func/mean": 0.15546876192092896,
+      "rewards/alfworld_rollout_reward_func/std": 0.398253858089447,
+      "sampling/importance_sampling_ratio/max": 2.3416731357574463,
+      "sampling/importance_sampling_ratio/mean": 0.5051580667495728,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.9200248718261719,
+      "sampling/sampling_logp_difference/mean": 0.01827174611389637,
+      "step": 30,
+      "step_time": 324.4753994320199
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 12941.0,
+      "completions/max_terminated_length": 12941.0,
+      "completions/mean_length": 8962.4375,
+      "completions/mean_terminated_length": 8962.4375,
+      "completions/min_length": 2241.0,
+      "completions/min_terminated_length": 2241.0,
+      "entropy": 0.31774380756542087,
+      "epoch": 0.124,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22262242436408997,
+      "kl": 0.0023503736083512194,
+      "learning_rate": 1.7271416662568652e-06,
+      "loss": -0.2728,
+      "num_tokens": 13653560.0,
+      "reward": 0.3187499940395355,
+      "reward_std": 0.46891656517982483,
+      "rewards/alfworld_rollout_reward_func/mean": 0.3187500238418579,
+      "rewards/alfworld_rollout_reward_func/std": 0.49869832396507263,
+      "sampling/importance_sampling_ratio/max": 2.161463975906372,
+      "sampling/importance_sampling_ratio/mean": 0.36309731006622314,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.3483941555023193,
+      "sampling/sampling_logp_difference/mean": 0.02068442292511463,
+      "step": 31,
+      "step_time": 430.4778445490374
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 7700.0,
+      "completions/max_terminated_length": 7700.0,
+      "completions/mean_length": 4740.34375,
+      "completions/mean_terminated_length": 4740.34375,
+      "completions/min_length": 751.0,
+      "completions/min_terminated_length": 751.0,
+      "entropy": 0.3920795237645507,
+      "epoch": 0.128,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3321034610271454,
+      "kl": 0.0023466993261536118,
+      "learning_rate": 1.6517524829811483e-06,
+      "loss": 0.2419,
+      "num_tokens": 13993934.0,
+      "reward": 0.25953125953674316,
+      "reward_std": 0.27672427892684937,
+      "rewards/alfworld_rollout_reward_func/mean": 0.25953125953674316,
+      "rewards/alfworld_rollout_reward_func/std": 0.4774650037288666,
+      "sampling/importance_sampling_ratio/max": 2.515002965927124,
+      "sampling/importance_sampling_ratio/mean": 0.5241488218307495,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.658240556716919,
+      "sampling/sampling_logp_difference/mean": 0.023516817018389702,
+      "step": 32,
+      "step_time": 244.8374374089908
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9641.0,
+      "completions/max_terminated_length": 9641.0,
+      "completions/mean_length": 5815.34375,
+      "completions/mean_terminated_length": 5815.34375,
+      "completions/min_length": 1387.0,
+      "completions/min_terminated_length": 1387.0,
+      "entropy": 0.3165481761097908,
+      "epoch": 0.132,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2114800661802292,
+      "kl": 0.0015353642320405925,
+      "learning_rate": 1.5759737532580691e-06,
+      "loss": 0.3504,
+      "num_tokens": 14409316.0,
+      "reward": 0.27562499046325684,
+      "reward_std": 0.48886263370513916,
+      "rewards/alfworld_rollout_reward_func/mean": 0.27562499046325684,
+      "rewards/alfworld_rollout_reward_func/std": 0.5106294751167297,
+      "sampling/importance_sampling_ratio/max": 1.6374565362930298,
+      "sampling/importance_sampling_ratio/mean": 0.5097586512565613,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.133441925048828,
+      "sampling/sampling_logp_difference/mean": 0.01870131492614746,
+      "step": 33,
+      "step_time": 331.69739585091884
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10654.0,
+      "completions/max_terminated_length": 10654.0,
+      "completions/mean_length": 7535.609375,
+      "completions/mean_terminated_length": 7535.609375,
+      "completions/min_length": 2351.0,
+      "completions/min_terminated_length": 2351.0,
+      "entropy": 0.33530362136662006,
+      "epoch": 0.136,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.263776034116745,
+      "kl": 0.0015736176555947168,
+      "learning_rate": 1.5e-06,
+      "loss": 0.1335,
+      "num_tokens": 14939979.0,
+      "reward": 0.18921875953674316,
+      "reward_std": 0.44627484679222107,
+      "rewards/alfworld_rollout_reward_func/mean": 0.18921875953674316,
+      "rewards/alfworld_rollout_reward_func/std": 0.4631647765636444,
+      "sampling/importance_sampling_ratio/max": 2.777794122695923,
+      "sampling/importance_sampling_ratio/mean": 0.5164031982421875,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 55.2677001953125,
+      "sampling/sampling_logp_difference/mean": 0.021179549396038055,
+      "step": 34,
+      "step_time": 358.0179667470511
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10147.0,
+      "completions/max_terminated_length": 10147.0,
+      "completions/mean_length": 5750.734375,
+      "completions/mean_terminated_length": 5750.734375,
+      "completions/min_length": 2291.0,
+      "completions/min_terminated_length": 2291.0,
+      "entropy": 0.32300999434664845,
+      "epoch": 0.14,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19500228762626648,
+      "kl": 0.0016171741226571612,
+      "learning_rate": 1.4240262467419312e-06,
+      "loss": 0.1209,
+      "num_tokens": 15353146.0,
+      "reward": 0.2917187511920929,
+      "reward_std": 0.40162450075149536,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2917187809944153,
+      "rewards/alfworld_rollout_reward_func/std": 0.48963722586631775,
+      "sampling/importance_sampling_ratio/max": 1.8978744745254517,
+      "sampling/importance_sampling_ratio/mean": 0.44630223512649536,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 53.83928298950195,
+      "sampling/sampling_logp_difference/mean": 0.021219002082943916,
+      "step": 35,
+      "step_time": 295.46302833205846
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10553.0,
+      "completions/max_terminated_length": 10553.0,
+      "completions/mean_length": 7239.625,
+      "completions/mean_terminated_length": 7239.625,
+      "completions/min_length": 2030.0,
+      "completions/min_terminated_length": 2030.0,
+      "entropy": 0.31878431234508753,
+      "epoch": 0.144,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.21166330575942993,
+      "kl": 0.0018054660904454067,
+      "learning_rate": 1.348247517018852e-06,
+      "loss": 0.2801,
+      "num_tokens": 15866850.0,
+      "reward": 0.2901562452316284,
+      "reward_std": 0.47076618671417236,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2901562452316284,
+      "rewards/alfworld_rollout_reward_func/std": 0.5097291469573975,
+      "sampling/importance_sampling_ratio/max": 2.524914503097534,
+      "sampling/importance_sampling_ratio/mean": 0.5144106149673462,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.984360694885254,
+      "sampling/sampling_logp_difference/mean": 0.020332355052232742,
+      "step": 36,
+      "step_time": 350.01136623701314
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 5913.0,
+      "completions/max_terminated_length": 5913.0,
+      "completions/mean_length": 4780.84375,
+      "completions/mean_terminated_length": 4780.84375,
+      "completions/min_length": 579.0,
+      "completions/min_terminated_length": 579.0,
+      "entropy": 0.36987574864178896,
+      "epoch": 0.148,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4266482889652252,
+      "kl": 0.002036058562225662,
+      "learning_rate": 1.2728583337431355e-06,
+      "loss": -0.1097,
+      "num_tokens": 16207096.0,
+      "reward": 0.07484374940395355,
+      "reward_std": 0.2557505965232849,
+      "rewards/alfworld_rollout_reward_func/mean": 0.07484374940395355,
+      "rewards/alfworld_rollout_reward_func/std": 0.34834298491477966,
+      "sampling/importance_sampling_ratio/max": 2.214252233505249,
+      "sampling/importance_sampling_ratio/mean": 0.5370051860809326,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.3917155265808105,
+      "sampling/sampling_logp_difference/mean": 0.02142323926091194,
+      "step": 37,
+      "step_time": 247.20503475998703
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6967.0,
+      "completions/max_terminated_length": 6967.0,
+      "completions/mean_length": 4629.875,
+      "completions/mean_terminated_length": 4629.875,
+      "completions/min_length": 999.0,
+      "completions/min_terminated_length": 999.0,
+      "entropy": 0.3442773185670376,
+      "epoch": 0.152,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.31260499358177185,
+      "kl": 0.0025437602398596937,
+      "learning_rate": 1.1980522198670096e-06,
+      "loss": 0.0576,
+      "num_tokens": 16541488.0,
+      "reward": 0.44578126072883606,
+      "reward_std": 0.5412331819534302,
+      "rewards/alfworld_rollout_reward_func/mean": 0.44578126072883606,
+      "rewards/alfworld_rollout_reward_func/std": 0.5354713201522827,
+      "sampling/importance_sampling_ratio/max": 2.806023597717285,
+      "sampling/importance_sampling_ratio/mean": 0.517693281173706,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.0272293090820312,
+      "sampling/sampling_logp_difference/mean": 0.020449213683605194,
+      "step": 38,
+      "step_time": 244.387926218973
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 13688.0,
+      "completions/max_terminated_length": 13688.0,
+      "completions/mean_length": 8324.359375,
+      "completions/mean_terminated_length": 8324.359375,
+      "completions/min_length": 3154.0,
+      "completions/min_terminated_length": 3154.0,
+      "entropy": 0.3674225895665586,
+      "epoch": 0.156,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.36787140369415283,
+      "kl": 0.0025533495027048048,
+      "learning_rate": 1.1240212016119191e-06,
+      "loss": 0.3241,
+      "num_tokens": 17125223.0,
+      "reward": 0.09515625238418579,
+      "reward_std": 0.3095976710319519,
+      "rewards/alfworld_rollout_reward_func/mean": 0.09515625238418579,
+      "rewards/alfworld_rollout_reward_func/std": 0.365721732378006,
+      "sampling/importance_sampling_ratio/max": 2.9900293350219727,
+      "sampling/importance_sampling_ratio/mean": 0.3794456422328949,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 46.29445266723633,
+      "sampling/sampling_logp_difference/mean": 0.027921315282583237,
+      "step": 39,
+      "step_time": 463.61854148590646
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6432.0,
+      "completions/max_terminated_length": 6432.0,
+      "completions/mean_length": 4293.5625,
+      "completions/mean_terminated_length": 4293.5625,
+      "completions/min_length": 552.0,
+      "completions/min_terminated_length": 552.0,
+      "entropy": 0.38137416914105415,
+      "epoch": 0.16,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4678648114204407,
+      "kl": 0.001712255700113019,
+      "learning_rate": 1.050955315539963e-06,
+      "loss": -0.2122,
+      "num_tokens": 17436395.0,
+      "reward": 0.31953126192092896,
+      "reward_std": 0.3283727169036865,
+      "rewards/alfworld_rollout_reward_func/mean": 0.31953126192092896,
+      "rewards/alfworld_rollout_reward_func/std": 0.510062038898468,
+      "sampling/importance_sampling_ratio/max": 2.993114948272705,
+      "sampling/importance_sampling_ratio/mean": 0.6158585548400879,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 48.348148345947266,
+      "sampling/sampling_logp_difference/mean": 0.023211471736431122,
+      "step": 40,
+      "step_time": 256.8888109070249
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10452.0,
+      "completions/max_terminated_length": 10452.0,
+      "completions/mean_length": 7975.96875,
+      "completions/mean_terminated_length": 7975.96875,
+      "completions/min_length": 1643.0,
+      "completions/min_terminated_length": 1643.0,
+      "entropy": 0.3266838057897985,
+      "epoch": 0.164,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2977025508880615,
+      "kl": 0.0014619105677411426,
+      "learning_rate": 9.790421207327699e-07,
+      "loss": 0.0016,
+      "num_tokens": 17994953.0,
+      "reward": 0.12093749642372131,
+      "reward_std": 0.26076871156692505,
+      "rewards/alfworld_rollout_reward_func/mean": 0.12093749642372131,
+      "rewards/alfworld_rollout_reward_func/std": 0.37234872579574585,
+      "sampling/importance_sampling_ratio/max": 2.8854687213897705,
+      "sampling/importance_sampling_ratio/mean": 0.5439540147781372,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.1864867210388184,
+      "sampling/sampling_logp_difference/mean": 0.01889047771692276,
+      "step": 41,
+      "step_time": 425.9135665790236
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10013.0,
+      "completions/max_terminated_length": 10013.0,
+      "completions/mean_length": 5837.21875,
+      "completions/mean_terminated_length": 5837.21875,
+      "completions/min_length": 812.0,
+      "completions/min_terminated_length": 812.0,
+      "entropy": 0.3718068078160286,
+      "epoch": 0.168,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.27091944217681885,
+      "kl": 0.0016485519881825894,
+      "learning_rate": 9.084662173300225e-07,
+      "loss": -0.1063,
+      "num_tokens": 18409943.0,
+      "reward": 0.32343751192092896,
+      "reward_std": 0.5115345120429993,
+      "rewards/alfworld_rollout_reward_func/mean": 0.32343751192092896,
+      "rewards/alfworld_rollout_reward_func/std": 0.5089429616928101,
+      "sampling/importance_sampling_ratio/max": 2.761387586593628,
+      "sampling/importance_sampling_ratio/mean": 0.5661083459854126,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2433140277862549,
+      "sampling/sampling_logp_difference/mean": 0.020694797858595848,
+      "step": 42,
+      "step_time": 350.4842578980315
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9433.0,
+      "completions/max_terminated_length": 9433.0,
+      "completions/mean_length": 7310.640625,
+      "completions/mean_terminated_length": 7310.640625,
+      "completions/min_length": 2144.0,
+      "completions/min_terminated_length": 2144.0,
+      "entropy": 0.3453982062637806,
+      "epoch": 0.172,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3637518882751465,
+      "kl": 0.002296481339726597,
+      "learning_rate": 8.394087726635485e-07,
+      "loss": -0.1358,
+      "num_tokens": 18925600.0,
+      "reward": 0.2357812523841858,
+      "reward_std": 0.44348853826522827,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2357812523841858,
+      "rewards/alfworld_rollout_reward_func/std": 0.4680924117565155,
+      "sampling/importance_sampling_ratio/max": 2.9779891967773438,
+      "sampling/importance_sampling_ratio/mean": 0.5728085041046143,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.363124132156372,
+      "sampling/sampling_logp_difference/mean": 0.021319666877388954,
+      "step": 43,
+      "step_time": 331.5770332479151
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 8984.0,
+      "completions/max_terminated_length": 8984.0,
+      "completions/mean_length": 6977.109375,
+      "completions/mean_terminated_length": 6977.109375,
+      "completions/min_length": 3575.0,
+      "completions/min_terminated_length": 3575.0,
+      "entropy": 0.36568512208759785,
+      "epoch": 0.176,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20232941210269928,
+      "kl": 0.0016346415723091923,
+      "learning_rate": 7.720470562033787e-07,
+      "loss": -0.0301,
+      "num_tokens": 19414471.0,
+      "reward": 0.04781249910593033,
+      "reward_std": 0.2049415409564972,
+      "rewards/alfworld_rollout_reward_func/mean": 0.04781249910593033,
+      "rewards/alfworld_rollout_reward_func/std": 0.2761898934841156,
+      "sampling/importance_sampling_ratio/max": 2.573187828063965,
+      "sampling/importance_sampling_ratio/mean": 0.40001991391181946,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.278357982635498,
+      "sampling/sampling_logp_difference/mean": 0.02066657692193985,
+      "step": 44,
+      "step_time": 335.4528454130341
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 15070.0,
+      "completions/max_terminated_length": 15070.0,
+      "completions/mean_length": 8883.3125,
+      "completions/mean_terminated_length": 8883.3125,
+      "completions/min_length": 4182.0,
+      "completions/min_terminated_length": 4182.0,
+      "entropy": 0.31764299469068646,
+      "epoch": 0.18,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1879281848669052,
+      "kl": 0.0016870554245542735,
+      "learning_rate": 7.065539845095568e-07,
+      "loss": 0.205,
+      "num_tokens": 20034107.0,
+      "reward": 0.10062500834465027,
+      "reward_std": 0.3260836601257324,
+      "rewards/alfworld_rollout_reward_func/mean": 0.10062500089406967,
+      "rewards/alfworld_rollout_reward_func/std": 0.3762761652469635,
+      "sampling/importance_sampling_ratio/max": 2.5225887298583984,
+      "sampling/importance_sampling_ratio/mean": 0.5068827271461487,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.621875524520874,
+      "sampling/sampling_logp_difference/mean": 0.019450519233942032,
+      "step": 45,
+      "step_time": 501.09887950192206
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9683.0,
+      "completions/max_terminated_length": 9683.0,
+      "completions/mean_length": 6441.609375,
+      "completions/mean_terminated_length": 6441.609375,
+      "completions/min_length": 2937.0,
+      "completions/min_terminated_length": 2937.0,
+      "entropy": 0.3502948391251266,
+      "epoch": 0.184,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1859540343284607,
+      "kl": 0.0018878296577895526,
+      "learning_rate": 6.430976773578113e-07,
+      "loss": 0.0278,
+      "num_tokens": 20485858.0,
+      "reward": 0.0010937508195638657,
+      "reward_std": 0.23143835365772247,
+      "rewards/alfworld_rollout_reward_func/mean": 0.0010937508195638657,
+      "rewards/alfworld_rollout_reward_func/std": 0.2593160569667816,
+      "sampling/importance_sampling_ratio/max": 2.3244879245758057,
+      "sampling/importance_sampling_ratio/mean": 0.34531980752944946,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.0072948932647705,
+      "sampling/sampling_logp_difference/mean": 0.020865384489297867,
+      "step": 46,
+      "step_time": 366.95768191007664
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 5392.0,
+      "completions/max_terminated_length": 5392.0,
+      "completions/mean_length": 3591.0625,
+      "completions/mean_terminated_length": 3591.0625,
+      "completions/min_length": 517.0,
+      "completions/min_terminated_length": 517.0,
+      "entropy": 0.3532397085800767,
+      "epoch": 0.188,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.26849785447120667,
+      "kl": 0.0017250235250685364,
+      "learning_rate": 5.818410261785057e-07,
+      "loss": 0.1737,
+      "num_tokens": 20748838.0,
+      "reward": 0.41593751311302185,
+      "reward_std": 0.5181204080581665,
+      "rewards/alfworld_rollout_reward_func/mean": 0.41593748331069946,
+      "rewards/alfworld_rollout_reward_func/std": 0.5215962529182434,
+      "sampling/importance_sampling_ratio/max": 2.7028884887695312,
+      "sampling/importance_sampling_ratio/mean": 0.5614103078842163,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.718446731567383,
+      "sampling/sampling_logp_difference/mean": 0.021135177463293076,
+      "step": 47,
+      "step_time": 194.4773051920347
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 5232.0,
+      "completions/max_terminated_length": 5232.0,
+      "completions/mean_length": 2274.453125,
+      "completions/mean_terminated_length": 2274.453125,
+      "completions/min_length": 498.0,
+      "completions/min_terminated_length": 498.0,
+      "entropy": 0.3223302629776299,
+      "epoch": 0.192,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20919622480869293,
+      "kl": 0.001539721906738123,
+      "learning_rate": 5.22941275916667e-07,
+      "loss": 0.3358,
+      "num_tokens": 20925091.0,
+      "reward": 0.6996874809265137,
+      "reward_std": 0.4721481502056122,
+      "rewards/alfworld_rollout_reward_func/mean": 0.6996875405311584,
+      "rewards/alfworld_rollout_reward_func/std": 0.4819766581058502,
+      "sampling/importance_sampling_ratio/max": 2.5296757221221924,
+      "sampling/importance_sampling_ratio/mean": 0.6417877674102783,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.344658613204956,
+      "sampling/sampling_logp_difference/mean": 0.019674479961395264,
+      "step": 48,
+      "step_time": 168.07057143107522
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6957.0,
+      "completions/max_terminated_length": 6957.0,
+      "completions/mean_length": 5750.375,
+      "completions/mean_terminated_length": 5750.375,
+      "completions/min_length": 3849.0,
+      "completions/min_terminated_length": 3849.0,
+      "entropy": 0.31901704519987106,
+      "epoch": 0.196,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.28379565477371216,
+      "kl": 0.0021226651278993813,
+      "learning_rate": 4.6654962138647007e-07,
+      "loss": 0.0489,
+      "num_tokens": 21330971.0,
+      "reward": 0.02437499538064003,
+      "reward_std": 0.186305969953537,
+      "rewards/alfworld_rollout_reward_func/mean": 0.02437499724328518,
+      "rewards/alfworld_rollout_reward_func/std": 0.2512942850589752,
+      "sampling/importance_sampling_ratio/max": 2.9523849487304688,
+      "sampling/importance_sampling_ratio/mean": 0.4695657193660736,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 53.46038055419922,
+      "sampling/sampling_logp_difference/mean": 0.019282279536128044,
+      "step": 49,
+      "step_time": 266.79126943598385
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9327.0,
+      "completions/max_terminated_length": 9327.0,
+      "completions/mean_length": 5568.34375,
+      "completions/mean_terminated_length": 5568.34375,
+      "completions/min_length": 544.0,
+      "completions/min_terminated_length": 544.0,
+      "entropy": 0.3215307700447738,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.319042831659317,
+      "kl": 0.0018373212151345797,
+      "learning_rate": 4.1281081915632036e-07,
+      "loss": 0.3955,
+      "num_tokens": 21725489.0,
+      "reward": 0.22624999284744263,
+      "reward_std": 0.40438205003738403,
+      "rewards/alfworld_rollout_reward_func/mean": 0.22624999284744263,
+      "rewards/alfworld_rollout_reward_func/std": 0.4729659855365753,
+      "sampling/importance_sampling_ratio/max": 2.5651261806488037,
+      "sampling/importance_sampling_ratio/mean": 0.5718743801116943,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.8857594728469849,
+      "sampling/sampling_logp_difference/mean": 0.019382823258638382,
+      "step": 50,
+      "step_time": 338.1545116149937
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11776.0,
+      "completions/max_terminated_length": 11776.0,
+      "completions/mean_length": 8995.296875,
+      "completions/mean_terminated_length": 8995.296875,
+      "completions/min_length": 2626.0,
+      "completions/min_terminated_length": 2626.0,
+      "entropy": 0.35890309792011976,
+      "epoch": 0.204,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16795441508293152,
+      "kl": 0.002312422751856502,
+      "learning_rate": 3.618628159608137e-07,
+      "loss": 0.1271,
+      "num_tokens": 22352804.0,
+      "reward": 0.06656250357627869,
+      "reward_std": 0.3205060660839081,
+      "rewards/alfworld_rollout_reward_func/mean": 0.06656249612569809,
+      "rewards/alfworld_rollout_reward_func/std": 0.32436618208885193,
+      "sampling/importance_sampling_ratio/max": 2.878086566925049,
+      "sampling/importance_sampling_ratio/mean": 0.4224788546562195,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.016578197479248,
+      "sampling/sampling_logp_difference/mean": 0.022725660353899002,
+      "step": 51,
+      "step_time": 381.1358364249754
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6659.0,
+      "completions/max_terminated_length": 6659.0,
+      "completions/mean_length": 4975.1875,
+      "completions/mean_terminated_length": 4975.1875,
+      "completions/min_length": 2482.0,
+      "completions/min_terminated_length": 2482.0,
+      "entropy": 0.3571747145615518,
+      "epoch": 0.208,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19965247809886932,
+      "kl": 0.001949786972545553,
+      "learning_rate": 3.1383639459345236e-07,
+      "loss": -0.0391,
+      "num_tokens": 22705936.0,
+      "reward": 0.10843749344348907,
+      "reward_std": 0.2537260055541992,
+      "rewards/alfworld_rollout_reward_func/mean": 0.10843749344348907,
+      "rewards/alfworld_rollout_reward_func/std": 0.3537294268608093,
+      "sampling/importance_sampling_ratio/max": 2.9369618892669678,
+      "sampling/importance_sampling_ratio/mean": 0.33827975392341614,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.243539333343506,
+      "sampling/sampling_logp_difference/mean": 0.020905133336782455,
+      "step": 52,
+      "step_time": 256.6296185880783
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9607.0,
+      "completions/max_terminated_length": 9607.0,
+      "completions/mean_length": 5973.78125,
+      "completions/mean_terminated_length": 5973.78125,
+      "completions/min_length": 1849.0,
+      "completions/min_terminated_length": 1849.0,
+      "entropy": 0.3052480248734355,
+      "epoch": 0.212,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.27389758825302124,
+      "kl": 0.0034342061553616077,
+      "learning_rate": 2.688548381890859e-07,
+      "loss": 0.0311,
+      "num_tokens": 23130466.0,
+      "reward": 0.19437500834465027,
+      "reward_std": 0.4533996284008026,
+      "rewards/alfworld_rollout_reward_func/mean": 0.19437500834465027,
+      "rewards/alfworld_rollout_reward_func/std": 0.45724809169769287,
+      "sampling/importance_sampling_ratio/max": 2.6141345500946045,
+      "sampling/importance_sampling_ratio/mean": 0.41237422823905945,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 3.5475544929504395,
+      "sampling/sampling_logp_difference/mean": 0.020619163289666176,
+      "step": 53,
+      "step_time": 327.58865647405037
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9492.0,
+      "completions/max_terminated_length": 9492.0,
+      "completions/mean_length": 6306.28125,
+      "completions/mean_terminated_length": 6306.28125,
+      "completions/min_length": 1341.0,
+      "completions/min_terminated_length": 1341.0,
+      "entropy": 0.3394437450915575,
+      "epoch": 0.216,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19828611612319946,
+      "kl": 0.0016102603804029059,
+      "learning_rate": 2.2703361375787346e-07,
+      "loss": 0.3936,
+      "num_tokens": 23577428.0,
+      "reward": 0.16374999284744263,
+      "reward_std": 0.4402611553668976,
+      "rewards/alfworld_rollout_reward_func/mean": 0.16374999284744263,
+      "rewards/alfworld_rollout_reward_func/std": 0.44003063440322876,
+      "sampling/importance_sampling_ratio/max": 2.7105014324188232,
+      "sampling/importance_sampling_ratio/mean": 0.5148346424102783,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.6567630767822266,
+      "sampling/sampling_logp_difference/mean": 0.019922319799661636,
+      "step": 54,
+      "step_time": 330.50816768797813
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 5940.0,
+      "completions/max_terminated_length": 5940.0,
+      "completions/mean_length": 3369.0,
+      "completions/mean_terminated_length": 3369.0,
+      "completions/min_length": 498.0,
+      "completions/min_terminated_length": 498.0,
+      "entropy": 0.36095575196668506,
+      "epoch": 0.22,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3427432179450989,
+      "kl": 0.0036608128975785803,
+      "learning_rate": 1.8848007578312686e-07,
+      "loss": 0.4046,
+      "num_tokens": 23825684.0,
+      "reward": 0.5928125381469727,
+      "reward_std": 0.5227715969085693,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5928125381469727,
+      "rewards/alfworld_rollout_reward_func/std": 0.511535108089447,
+      "sampling/importance_sampling_ratio/max": 2.6692662239074707,
+      "sampling/importance_sampling_ratio/mean": 0.5836673378944397,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 4.313566207885742,
+      "sampling/sampling_logp_difference/mean": 0.021921399980783463,
+      "step": 55,
+      "step_time": 209.76535455693374
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 7371.0,
+      "completions/max_terminated_length": 7371.0,
+      "completions/mean_length": 5376.625,
+      "completions/mean_terminated_length": 5376.625,
+      "completions/min_length": 992.0,
+      "completions/min_terminated_length": 992.0,
+      "entropy": 0.335981338750571,
+      "epoch": 0.224,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2279304414987564,
+      "kl": 0.001375907209876459,
+      "learning_rate": 1.5329319064388763e-07,
+      "loss": 0.3754,
+      "num_tokens": 24206844.0,
+      "reward": 0.08656249940395355,
+      "reward_std": 0.34256887435913086,
+      "rewards/alfworld_rollout_reward_func/mean": 0.08656249940395355,
+      "rewards/alfworld_rollout_reward_func/std": 0.34272125363349915,
+      "sampling/importance_sampling_ratio/max": 2.9141616821289062,
+      "sampling/importance_sampling_ratio/mean": 0.5004754066467285,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.75748872756958,
+      "sampling/sampling_logp_difference/mean": 0.018594371154904366,
+      "step": 56,
+      "step_time": 289.04723902978003
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 10048.0,
+      "completions/max_terminated_length": 10048.0,
+      "completions/mean_length": 8312.671875,
+      "completions/mean_terminated_length": 8312.671875,
+      "completions/min_length": 2348.0,
+      "completions/min_terminated_length": 2348.0,
+      "entropy": 0.3852003728970885,
+      "epoch": 0.228,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22347784042358398,
+      "kl": 0.0018401142369839363,
+      "learning_rate": 1.215632825696541e-07,
+      "loss": 0.2461,
+      "num_tokens": 24789191.0,
+      "reward": 0.11437500268220901,
+      "reward_std": 0.3912443518638611,
+      "rewards/alfworld_rollout_reward_func/mean": 0.11437500268220901,
+      "rewards/alfworld_rollout_reward_func/std": 0.403114378452301,
+      "sampling/importance_sampling_ratio/max": 2.5475013256073,
+      "sampling/importance_sampling_ratio/mean": 0.45833104848861694,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.6942596435546875,
+      "sampling/sampling_logp_difference/mean": 0.022461190819740295,
+      "step": 57,
+      "step_time": 388.85974016188993
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 6720.0,
+      "completions/max_terminated_length": 6720.0,
+      "completions/mean_length": 5543.359375,
+      "completions/mean_terminated_length": 5543.359375,
+      "completions/min_length": 2344.0,
+      "completions/min_terminated_length": 2344.0,
+      "entropy": 0.32870627008378506,
+      "epoch": 0.232,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4650394320487976,
+      "kl": 0.001981915997021133,
+      "learning_rate": 9.337180177937954e-08,
+      "loss": 0.7758,
+      "num_tokens": 25182878.0,
+      "reward": 0.08546875417232513,
+      "reward_std": 0.2914150357246399,
+      "rewards/alfworld_rollout_reward_func/mean": 0.08546875417232513,
+      "rewards/alfworld_rollout_reward_func/std": 0.34556111693382263,
+      "sampling/importance_sampling_ratio/max": 2.719653367996216,
+      "sampling/importance_sampling_ratio/mean": 0.5657415390014648,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 5.367229461669922,
+      "sampling/sampling_logp_difference/mean": 0.0186595655977726,
+      "step": 58,
+      "step_time": 247.89911623601802
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 11541.0,
+      "completions/max_terminated_length": 11541.0,
+      "completions/mean_length": 5263.265625,
+      "completions/mean_terminated_length": 5263.265625,
+      "completions/min_length": 1418.0,
+      "completions/min_terminated_length": 1418.0,
+      "entropy": 0.361502249725163,
+      "epoch": 0.236,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2132435142993927,
+      "kl": 0.0016867622362042312,
+      "learning_rate": 6.879111539992677e-08,
+      "loss": 0.0722,
+      "num_tokens": 25557423.0,
+      "reward": 0.2345312535762787,
+      "reward_std": 0.4755549430847168,
+      "rewards/alfworld_rollout_reward_func/mean": 0.2345312535762787,
+      "rewards/alfworld_rollout_reward_func/std": 0.486049622297287,
+      "sampling/importance_sampling_ratio/max": 2.7570416927337646,
+      "sampling/importance_sampling_ratio/mean": 0.4710940420627594,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.1428396701812744,
+      "sampling/sampling_logp_difference/mean": 0.021931620314717293,
+      "step": 59,
+      "step_time": 334.8811481029552
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 9499.0,
+      "completions/max_terminated_length": 9499.0,
+      "completions/mean_length": 5835.5625,
+      "completions/mean_terminated_length": 5835.5625,
+      "completions/min_length": 1881.0,
+      "completions/min_terminated_length": 1881.0,
+      "entropy": 0.28580303583294153,
+      "epoch": 0.24,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2324402630329132,
+      "kl": 0.0014885484506521607,
+      "learning_rate": 4.788432170069373e-08,
+      "loss": 0.0045,
+      "num_tokens": 25969907.0,
+      "reward": 0.2704687714576721,
+      "reward_std": 0.48869654536247253,
+      "rewards/alfworld_rollout_reward_func/mean": 0.27046874165534973,
+      "rewards/alfworld_rollout_reward_func/std": 0.482570618391037,
+      "sampling/importance_sampling_ratio/max": 2.4066574573516846,
+      "sampling/importance_sampling_ratio/mean": 0.4883001446723938,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.3303287029266357,
+      "sampling/sampling_logp_difference/mean": 0.016779117286205292,
+      "step": 60,
+      "step_time": 319.99202060801326
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 64,
+  "num_input_tokens_seen": 25969907,
+  "num_train_epochs": 1,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3465d4107a8cecb08f7b1f693dff91800432dd1c9daa830c68e25daf6eecb8b3
+size 9425