debug.log

[2026-01-21 17:04:15,704] [WARNING] [py.warnings._showwarnmsg:110] [PID:252] /root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/backends/__init__.py:46: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.)
  self.setter(val)

[2026-01-21 17:04:15,706] [DEBUG] [axolotl.utils.config.resolve_dtype:66] [PID:251] bf16 support detected, enabling for this configuration.
[2026-01-21 17:04:15,707] [WARNING] [py.warnings._showwarnmsg:110] [PID:251] /root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/backends/__init__.py:46: UserWarning: Please use the new API settings to control TF32 behavior, such as torch.backends.cudnn.conv.fp32_precision = 'tf32' or torch.backends.cuda.matmul.fp32_precision = 'ieee'. Old settings, e.g, torch.backends.cuda.matmul.allow_tf32 = True, torch.backends.cudnn.allow_tf32 = True, allowTF32CuDNN() and allowTF32CuBLAS() will be deprecated after Pytorch 2.9. Please see https://pytorch.org/docs/main/notes/cuda.html#tensorfloat-32-tf32-on-ampere-and-later-devices (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:80.)
  self.setter(val)

[2026-01-21 17:04:15,709] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:251] baseline 0.000GB ()
[2026-01-21 17:04:15,709] [INFO] [axolotl.cli.config.load_cfg:259] [PID:251] config:
{
  "activation_offloading": false,
  "adapter": "lora",
  "axolotl_config_path": "/workspace/axolotl/configs/1.yml",
  "base_model": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
  "base_model_config": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
  "batch_size": 64,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_90",
    "fp8": true,
    "n_gpu": 2,
    "n_node": 1
  },
  "chat_template": "llama3",
  "context_parallel_size": 1,
  "dataloader_num_workers": 2,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_num_proc": 52,
  "datasets": [    {      "data_files": [        "1_train_data.json"      ],
      "ds_type": "json",
      "message_property_mappings": {
        "content": "content",
        "role": "role"
      },
      "path": "/workspace/axolotl/data",
      "split": "train",
      "trust_remote_code": false
    }
  ],
  "ddp": true,
  "device": "cuda:0",
  "device_map": {
    "": 0
  },
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "env_capabilities": {
    "torch_version": "2.9.0"
  },
  "eval_batch_size": 1,
  "eval_causal_lm_metrics": [    "sacrebleu",    "comet",    "ter",    "chrf"  ],
  "eval_max_new_tokens": 128,
  "eval_strategy": "no",
  "eval_table_size": 0,
  "experimental_skip_move_to_device": true,
  "flash_attention": false,
  "fp16": false,
  "gradient_accumulation_steps": 32,
  "gradient_checkpointing": true,
  "group_by_length": false,
  "hub_strategy": "every_save",
  "include_tkps": true,
  "is_falcon_derived_model": false,
  "is_llama_derived_model": false,
  "is_mistral_derived_model": false,
  "learning_rate": 5e-06,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": false,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "logging_steps": 1,
  "lora_alpha": 32,
  "lora_dropout": 0.0,
  "lora_r": 16,
  "lora_target_linear": true,
  "loraplus_lr_embedding": 1e-06,
  "lr_scheduler": "cosine",
  "max_grad_norm": 1.0,
  "max_steps": 64,
  "mean_resizing_embeddings": false,
  "micro_batch_size": 1,
  "mlflow_experiment_name": "/workspace/axolotl/data/1_train_data.json",
  "model_config_type": "qwen2",
  "num_epochs": 1.0,
  "optimizer": "adamw_bnb_8bit",
  "otel_metrics_host": "localhost",
  "otel_metrics_port": 8000,
  "output_dir": "/app/checkpoints/1/environment_test",
  "pad_to_sequence_len": true,
  "pretrain_multipack_attn": true,
  "profiler_steps_start": 0,
  "qlora_sharded_model_loading": false,
  "ray_num_workers": 1,
  "resources_per_worker": {
    "GPU": 1
  },
  "rl": "grpo",
  "sample_packing": false,
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_only_model": false,
  "save_safetensors": true,
  "save_steps": 20,
  "sequence_len": 24000,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": true,
  "skip_prepare_dataset": false,
  "special_tokens": {
    "bos_token": "<|im_end|>"
  },
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tf32": false,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
  "tokenizer_save_jinja_files": true,
  "tokenizer_type": "AutoTokenizer",
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "beta": 0.01,
    "log_completions": false,
    "mask_truncated_completions": false,
    "max_completion_length": 128,
    "num_generations": 8,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "reward_funcs": [      "alfworld.alfworld_rollout_reward_func"    ],
    "reward_weights": [      1.0    ],
    "rollout_func": "alfworld.alfworld_rollout_first_prompt_and_completion",
    "scale_rewards": true,
    "sync_ref_model": false,
    "temperature": 0.7,
    "use_vllm": true,
    "vllm_enable_sleep_mode": false,
    "vllm_mode": "colocate",
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "trust_remote_code": true,
  "type_of_model": "AutoModelForCausalLM",
  "use_mlflow": true,
  "use_otel_metrics": false,
  "use_ray": false,
  "use_wandb": true,
  "val_set_size": 0.0,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "enable_prefix_caching": false,
    "gpu_memory_utilization": 0.3,
    "host": "0.0.0.0",
    "max_model_len": 8192,
    "port": 8000,
    "tensor_parallel_size": 1
  },
  "wandb_mode": "online",
  "wandb_name": "1_environment_test",
  "wandb_project": "Gradients-Rollout",
  "weight_decay": 0.0,
  "world_size": 2
}
[2026-01-21 17:04:15,712] [WARNING] [axolotl.cli.checks.check_user_token:46] [PID:251] Error verifying HuggingFace token. Remember to log in using `huggingface-cli login` and get your access token from https://huggingface.co/settings/tokens if you want to use gated models or datasets.
[2026-01-21 17:04:15,939] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:251] EOS: 151645 / <|im_end|>
[2026-01-21 17:04:15,939] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:251] BOS: 151645 / <|im_end|>
[2026-01-21 17:04:15,939] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:251] PAD: 151643 / <|endoftext|>
[2026-01-21 17:04:15,939] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:251] UNK: None / None
[2026-01-21 17:04:15,939] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:251] Unable to find prepared dataset in last_run_prepared/ba0ae834220c702ae7aefbdbfde66c85

Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 1000 examples [00:00, 174806.37 examples/s]
[2026-01-21 17:04:16,306] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:251] EOS: 151645 / <|im_end|>
[2026-01-21 17:04:16,306] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:251] BOS: 151645 / <|im_end|>
[2026-01-21 17:04:16,306] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:251] PAD: 151643 / <|endoftext|>
[2026-01-21 17:04:16,306] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:251] UNK: None / None

Dropping Long Sequences (num_proc=52):   0%|          | 0/1000 [00:00<?, ? examples/s]
Dropping Long Sequences (num_proc=52):   2%|▏         | 20/1000 [00:00<00:44, 21.89 examples/s]
Dropping Long Sequences (num_proc=52):   6%|▌         | 60/1000 [00:01<00:13, 68.27 examples/s]
Dropping Long Sequences (num_proc=52):  10%|█         | 100/1000 [00:01<00:08, 110.14 examples/s]
Dropping Long Sequences (num_proc=52):  16%|█▌        | 160/1000 [00:01<00:04, 176.15 examples/s]
Dropping Long Sequences (num_proc=52):  20%|██        | 200/1000 [00:01<00:04, 197.33 examples/s]
Dropping Long Sequences (num_proc=52):  24%|██▍       | 240/1000 [00:01<00:03, 214.30 examples/s]
Dropping Long Sequences (num_proc=52):  28%|██▊       | 278/1000 [00:01<00:03, 225.12 examples/s]
Dropping Long Sequences (num_proc=52):  32%|███▏      | 316/1000 [00:01<00:02, 233.07 examples/s]
Dropping Long Sequences (num_proc=52):  35%|███▌      | 354/1000 [00:02<00:02, 237.03 examples/s]
Dropping Long Sequences (num_proc=52):  39%|███▉      | 392/1000 [00:02<00:02, 241.67 examples/s]
Dropping Long Sequences (num_proc=52):  43%|████▎     | 430/1000 [00:02<00:02, 242.51 examples/s]
Dropping Long Sequences (num_proc=52):  47%|████▋     | 468/1000 [00:02<00:02, 247.08 examples/s]
Dropping Long Sequences (num_proc=52):  51%|█████     | 506/1000 [00:02<00:02, 217.43 examples/s]
Dropping Long Sequences (num_proc=52):  54%|█████▍    | 544/1000 [00:02<00:02, 225.52 examples/s]
Dropping Long Sequences (num_proc=52):  60%|██████    | 601/1000 [00:03<00:01, 264.74 examples/s]
Dropping Long Sequences (num_proc=52):  64%|██████▍   | 639/1000 [00:03<00:01, 257.89 examples/s]
Dropping Long Sequences (num_proc=52):  68%|██████▊   | 677/1000 [00:03<00:01, 253.20 examples/s]
Dropping Long Sequences (num_proc=52):  72%|███████▏  | 715/1000 [00:03<00:01, 251.21 examples/s]
Dropping Long Sequences (num_proc=52):  75%|███████▌  | 753/1000 [00:03<00:00, 248.61 examples/s]
Dropping Long Sequences (num_proc=52):  79%|███████▉  | 791/1000 [00:03<00:00, 247.45 examples/s]
Dropping Long Sequences (num_proc=52):  83%|████████▎ | 829/1000 [00:04<00:00, 245.85 examples/s]
Dropping Long Sequences (num_proc=52):  87%|████████▋ | 867/1000 [00:04<00:00, 244.78 examples/s]
Dropping Long Sequences (num_proc=52):  90%|█████████ | 905/1000 [00:04<00:00, 245.83 examples/s]
Dropping Long Sequences (num_proc=52):  94%|█████████▍| 943/1000 [00:04<00:00, 248.93 examples/s]
Dropping Long Sequences (num_proc=52):  98%|█████████▊| 981/1000 [00:04<00:00, 254.19 examples/s]
Dropping Long Sequences (num_proc=52): 100%|██████████| 1000/1000 [00:06<00:00, 166.07 examples/s]

Saving the dataset (0/3 shards):   0%|          | 0/1000 [00:00<?, ? examples/s]
Saving the dataset (0/3 shards):  33%|███▎      | 334/1000 [00:00<00:00, 3300.07 examples/s]
Saving the dataset (1/3 shards):  33%|███▎      | 334/1000 [00:00<00:00, 3300.07 examples/s]
Saving the dataset (2/3 shards):  67%|██████▋   | 667/1000 [00:00<00:00, 3300.07 examples/s]
Saving the dataset (3/3 shards): 100%|██████████| 1000/1000 [00:00<00:00, 3300.07 examples/s]
Saving the dataset (3/3 shards): 100%|██████████| 1000/1000 [00:00<00:00, 6022.04 examples/s]
[2026-01-21 17:04:22,668] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:251] loading tokenizer... /cache/models/Qwen--Qwen2.5-3B-Instruct
[2026-01-21 17:04:22,876] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:251] EOS: 151645 / <|im_end|>
[2026-01-21 17:04:22,876] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:251] BOS: 151645 / <|im_end|>
[2026-01-21 17:04:22,876] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:251] PAD: 151643 / <|endoftext|>
[2026-01-21 17:04:22,876] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:251] UNK: None / None
[2026-01-21 17:04:22,876] [DEBUG] [axolotl.train.setup_model_and_tokenizer:82] [PID:251] Loading model
[2026-01-21 17:04:22,886] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:251] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-01-21 17:04:22,888] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:251] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Saving the dataset (0/3 shards):   0%|          | 0/1000 [00:00<?, ? examples/s]
Saving the dataset (1/3 shards):  33%|███▎      | 334/1000 [00:00<00:00, 5111.74 examples/s]
Saving the dataset (2/3 shards):  67%|██████▋   | 667/1000 [00:00<00:00, 9999.57 examples/s]
Saving the dataset (3/3 shards): 100%|██████████| 1000/1000 [00:00<00:00, 14853.51 examples/s]
Saving the dataset (3/3 shards): 100%|██████████| 1000/1000 [00:00<00:00, 7615.82 examples/s] 
Loading checkpoint shards:  50%|█████     | 1/2 [00:00<00:00,  1.51it/s]

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:01<00:00,  2.06it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:01<00:00,  1.96it/s]
[2026-01-21 17:04:24,179] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:347] [PID:251] Converting modules to torch.bfloat16
[2026-01-21 17:04:24,183] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:251] Memory usage after model load 6.907GB (+6.907GB allocated, +6.928GB reserved)
[2026-01-21 17:04:24,183] [INFO] [axolotl.loaders.adapter.load_lora:81] [PID:251] found linear modules: ['down_proj', 'gate_proj', 'k_proj', 'o_proj', 'q_proj', 'up_proj', 'v_proj']

Loading checkpoint shards:  50%|█████     | 1/2 [00:00<00:00,  1.57it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00,  2.13it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00,  2.02it/s]trainable params: 29,933,568 || all params: 3,115,872,256 || trainable%: 0.9607
[2026-01-21 17:04:24,841] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:251] after adapters 5.860GB (+5.860GB allocated, +7.039GB reserved)
[2026-01-21 17:04:25,424] [DEBUG] [axolotl.train.setup_reference_model:126] [PID:251] Passing model_ref: None to RL trainer

[2026-01-21 17:04:32,533] [WARNING] [py.warnings._showwarnmsg:110] [PID:252] <string>:246: FutureWarning: The `max_prompt_length` argument is deprecated and will be removed in version 0.28.0. You should instead filter your dataset before training to ensure that prompts do not exceed your desired length.

[2026-01-21 17:04:32,548] [WARNING] [py.warnings._showwarnmsg:110] [PID:251] <string>:246: FutureWarning: The `max_prompt_length` argument is deprecated and will be removed in version 0.28.0. You should instead filter your dataset before training to ensure that prompts do not exceed your desired length.

[2026-01-21 17:04:33,253] [WARNING] [py.warnings._showwarnmsg:110] [PID:252] /workspace/axolotl/src/axolotl/core/trainers/mixins/optimizer.py:209: UserWarning: You are importing from 'rollout_func', which is an experimental feature. This API may change or be removed at any time without prior notice. Silence this warning by setting environment variable TRL_EXPERIMENTAL_SILENCE=1.
  super().__init__(*args, **kwargs)

[2026-01-21 17:04:33,256] [WARNING] [py.warnings._showwarnmsg:110] [PID:251] /workspace/axolotl/src/axolotl/core/trainers/mixins/optimizer.py:209: UserWarning: You are importing from 'rollout_func', which is an experimental feature. This API may change or be removed at any time without prior notice. Silence this warning by setting environment variable TRL_EXPERIMENTAL_SILENCE=1.
  super().__init__(*args, **kwargs)


Loading safetensors checkpoint shards:   0% Completed | 0/2 [00:00<?, ?it/s]

Loading safetensors checkpoint shards:  50% Completed | 1/2 [00:00<00:00,  2.22it/s]

Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:01<00:00,  1.53it/s]

Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:01<00:00,  1.61it/s]
2026-01-21 17:05:20,527 - INFO - autotuner.py:256 - flashinfer.jit: [Autotuner]: Autotuning process starts ...

2026-01-21 17:05:20,527 - INFO - autotuner.py:256 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
2026-01-21 17:05:20,540 - INFO - autotuner.py:262 - flashinfer.jit: [Autotuner]: Autotuning process ends
2026-01-21 17:05:20,540 - INFO - autotuner.py:262 - flashinfer.jit: [Autotuner]: Autotuning process ends

Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/11 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  27%|██▋       | 3/11 [00:00<00:00, 21.35it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  55%|█████▍    | 6/11 [00:00<00:00, 23.66it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  82%|████████▏ | 9/11 [00:00<00:00, 25.27it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:00<00:00, 24.88it/s]

Capturing CUDA graphs (decode, FULL):   0%|          | 0/7 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  57%|█████▋    | 4/7 [00:00<00:00, 37.59it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 7/7 [00:00<00:00, 38.76it/s]
[2026-01-21 17:05:23,522] [INFO] [axolotl.train.save_initial_configs:413] [PID:251] Pre-saving adapter config to /app/checkpoints/1/environment_test...
[2026-01-21 17:05:23,523] [INFO] [axolotl.train.save_initial_configs:417] [PID:251] Pre-saving tokenizer to /app/checkpoints/1/environment_test...
[2026-01-21 17:05:23,651] [INFO] [axolotl.train.save_initial_configs:422] [PID:251] Pre-saving model config to /app/checkpoints/1/environment_test...
[2026-01-21 17:05:23,654] [INFO] [axolotl.train.execute_training:212] [PID:251] Starting trainer...
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
wandb: Currently logged in as: ductrung27-dt (ductrung27-dt-none) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.24.0
wandb: Run data is saved locally in /workspace/axolotl/wandb/run-20260121_170524-0vosajol
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run 1_environment_test
wandb: ⭐️ View project at https://wandb.ai/ductrung27-dt-none/Gradients-Rollout
wandb: 🚀 View run at https://wandb.ai/ductrung27-dt-none/Gradients-Rollout/runs/0vosajol
wandb: Detected [huggingface_hub.inference, openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
wandb: WARNING Saving files without folders. If you want to preserve subdirectories pass base_path to wandb.save, i.e. wandb.save("/mnt/folder/file.h5", base_path="/mnt")
wandb: WARNING Symlinked 1 file into the W&B run directory; call wandb.save again to sync new files.
[2026-01-21 17:05:27,447] [INFO] [axolotl.utils.callbacks.on_train_begin:757] [PID:251] The Axolotl config has been saved to the WandB run under files.
[2026-01-21 17:05:27,451] [INFO] [axolotl.utils.callbacks.mlflow_.on_train_begin:50] [PID:251] Skipping logging artifacts to MLflow (hf_mlflow_log_artifacts is false)

  0%|          | 0/64 [00:00<?, ?it/s]Initializing AlfWorld environment on rank 0 at http://environment-server-0:8000...
Environment initialized. ID: 0
Warning: BPE mismatch at turn 16 (expected prefix 5813, got 6130 tokens). Attempting delta recovery.

  2%|▏         | 1/64 [05:45<6:03:10, 345.89s/it]
                                                 
{'loss': 0.0003, 'grad_norm': 0.013934657908976078, 'learning_rate': 0.0, 'num_tokens': 52013.0, 'completions/mean_length': 812.703125, 'completions/min_length': 88.0, 'completions/max_length': 1707.0, 'completions/clipped_ratio': 0.6875, 'completions/mean_terminated_length': 386.1499938964844, 'completions/min_terminated_length': 88.0, 'completions/max_terminated_length': 915.0, 'rollout/action_tokens_mean': 812.703125, 'rollout/episode_turns_mean': 25.0625, 'rollout/truncated_ratio': 0.6875, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.26906251907348633, 'rewards/alfworld_rollout_reward_func/std': 0.48894354701042175, 'reward': 0.26906251907348633, 'reward_std': 0.43214160203933716, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020846465602517128, 'sampling/sampling_logp_difference/max': 2.9913136959075928, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5413336753845215, 'sampling/importance_sampling_ratio/max': 2.401656150817871, 'kl': 0.0, 'entropy': 0.319809939712286, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 340.2447626780008, 'epoch': 0.01}

  2%|▏         | 1/64 [05:45<6:03:10, 345.89s/it]
  3%|▎         | 2/64 [12:08<6:19:31, 367.28s/it]
                                                 
{'loss': 0.0064, 'grad_norm': 0.005984712392091751, 'learning_rate': 2.5e-06, 'num_tokens': 104679.0, 'completions/mean_length': 822.90625, 'completions/min_length': 240.0, 'completions/max_length': 1338.0, 'completions/clipped_ratio': 0.765625, 'completions/mean_terminated_length': 481.0000305175781, 'completions/min_terminated_length': 240.0, 'completions/max_terminated_length': 912.0, 'rollout/action_tokens_mean': 822.90625, 'rollout/episode_turns_mean': 26.75, 'rollout/truncated_ratio': 0.765625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.1837500035762787, 'rewards/alfworld_rollout_reward_func/std': 0.44909608364105225, 'reward': 0.1837500035762787, 'reward_std': 0.37211528420448303, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.021448608487844467, 'sampling/sampling_logp_difference/max': 2.2203328609466553, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.30532318353652954, 'sampling/importance_sampling_ratio/max': 2.0904664993286133, 'kl': 0.0, 'entropy': 0.33951753890141845, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 376.3160819019986, 'epoch': 0.02}

  3%|▎         | 2/64 [12:08<6:19:31, 367.28s/it]
  5%|▍         | 3/64 [18:12<6:11:57, 365.85s/it]
                                                 
{'loss': 0.0026, 'grad_norm': 0.0066122072748839855, 'learning_rate': 5e-06, 'num_tokens': 151761.0, 'completions/mean_length': 735.65625, 'completions/min_length': 145.0, 'completions/max_length': 1253.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 380.5, 'completions/min_terminated_length': 145.0, 'completions/max_terminated_length': 680.0, 'rollout/action_tokens_mean': 735.65625, 'rollout/episode_turns_mean': 26.03125, 'rollout/truncated_ratio': 0.75, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.21156249940395355, 'rewards/alfworld_rollout_reward_func/std': 0.44746631383895874, 'reward': 0.21156249940395355, 'reward_std': 0.2755644917488098, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020412152633070946, 'sampling/sampling_logp_difference/max': 1.5531697273254395, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5484369993209839, 'sampling/importance_sampling_ratio/max': 2.4706838130950928, 'kl': 0.001546632844110718, 'entropy': 0.32836992759257555, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 359.0840177950013, 'epoch': 0.02}

  5%|▍         | 3/64 [18:12<6:11:57, 365.85s/it]
  6%|▋         | 4/64 [23:42<5:51:48, 351.81s/it]
                                                 
{'loss': 0.0123, 'grad_norm': 0.00901270005851984, 'learning_rate': 4.996791267927632e-06, 'num_tokens': 197955.0, 'completions/mean_length': 721.78125, 'completions/min_length': 112.0, 'completions/max_length': 1880.0, 'completions/clipped_ratio': 0.546875, 'completions/mean_terminated_length': 425.72412109375, 'completions/min_terminated_length': 112.0, 'completions/max_terminated_length': 807.0, 'rollout/action_tokens_mean': 721.78125, 'rollout/episode_turns_mean': 22.859375, 'rollout/truncated_ratio': 0.546875, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.40953126549720764, 'rewards/alfworld_rollout_reward_func/std': 0.5298889875411987, 'reward': 0.40953126549720764, 'reward_std': 0.48288512229919434, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022099550813436508, 'sampling/sampling_logp_difference/max': 3.2973906993865967, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5041748881340027, 'sampling/importance_sampling_ratio/max': 2.607266664505005, 'kl': 0.001597001251866459, 'entropy': 0.3564013452269137, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 324.93751245399835, 'epoch': 0.03}

  6%|▋         | 4/64 [23:42<5:51:48, 351.81s/it]
  8%|▊         | 5/64 [30:59<6:16:03, 382.44s/it]
                                                 
{'loss': -0.0049, 'grad_norm': 0.006255371496081352, 'learning_rate': 4.987173308479738e-06, 'num_tokens': 250690.0, 'completions/mean_length': 823.984375, 'completions/min_length': 238.0, 'completions/max_length': 1772.0, 'completions/clipped_ratio': 0.6875, 'completions/mean_terminated_length': 511.25, 'completions/min_terminated_length': 238.0, 'completions/max_terminated_length': 1135.0, 'rollout/action_tokens_mean': 823.984375, 'rollout/episode_turns_mean': 25.65625, 'rollout/truncated_ratio': 0.6875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.2629687786102295, 'rewards/alfworld_rollout_reward_func/std': 0.48142871260643005, 'reward': 0.2629687488079071, 'reward_std': 0.3148724138736725, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022958241403102875, 'sampling/sampling_logp_difference/max': 1.3790063858032227, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.3966376781463623, 'sampling/importance_sampling_ratio/max': 2.4376842975616455, 'kl': 0.0018659899324120488, 'entropy': 0.37002173252403736, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 428.7077085350011, 'epoch': 0.04}

  8%|▊         | 5/64 [30:59<6:16:03, 382.44s/it]
  9%|▉         | 6/64 [36:33<5:53:48, 366.01s/it]
                                                 
{'loss': 0.0284, 'grad_norm': 0.014856722205877304, 'learning_rate': 4.971170810820279e-06, 'num_tokens': 300499.0, 'completions/mean_length': 778.265625, 'completions/min_length': 89.0, 'completions/max_length': 1754.0, 'completions/clipped_ratio': 0.546875, 'completions/mean_terminated_length': 513.0689697265625, 'completions/min_terminated_length': 89.0, 'completions/max_terminated_length': 1081.0, 'rollout/action_tokens_mean': 778.265625, 'rollout/episode_turns_mean': 24.0, 'rollout/truncated_ratio': 0.546875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.4112499952316284, 'rewards/alfworld_rollout_reward_func/std': 0.5246087312698364, 'reward': 0.4112499952316284, 'reward_std': 0.4292137622833252, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020635541528463364, 'sampling/sampling_logp_difference/max': 2.5756630897521973, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.41494274139404297, 'sampling/importance_sampling_ratio/max': 2.469376802444458, 'kl': 0.0018188257581641665, 'entropy': 0.33683515526354313, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 329.3498830619992, 'epoch': 0.05}

  9%|▉         | 6/64 [36:33<5:53:48, 366.01s/it]
 11%|█         | 7/64 [42:51<5:51:25, 369.92s/it]
                                                 
{'loss': 0.0135, 'grad_norm': 0.011092925444245338, 'learning_rate': 4.948824853131237e-06, 'num_tokens': 353898.0, 'completions/mean_length': 834.359375, 'completions/min_length': 142.0, 'completions/max_length': 1750.0, 'completions/clipped_ratio': 0.765625, 'completions/mean_terminated_length': 538.4000244140625, 'completions/min_terminated_length': 142.0, 'completions/max_terminated_length': 882.0, 'rollout/action_tokens_mean': 834.359375, 'rollout/episode_turns_mean': 27.0625, 'rollout/truncated_ratio': 0.765625, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.1875, 'rewards/alfworld_rollout_reward_func/std': 0.4403570294380188, 'reward': 0.1875, 'reward_std': 0.41170334815979004, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022584954276680946, 'sampling/sampling_logp_difference/max': 3.733177900314331, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.46820664405822754, 'sampling/importance_sampling_ratio/max': 2.2279603481292725, 'kl': 0.0017349459521938115, 'entropy': 0.37804630119353533, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 372.10799923799914, 'epoch': 0.06}

 11%|█         | 7/64 [42:51<5:51:25, 369.92s/it]
 12%|█▎        | 8/64 [50:14<6:07:05, 393.30s/it]
                                                 
{'loss': -0.0059, 'grad_norm': 0.010468119755387306, 'learning_rate': 4.920192797165511e-06, 'num_tokens': 404238.0, 'completions/mean_length': 786.5625, 'completions/min_length': 180.0, 'completions/max_length': 1682.0, 'completions/clipped_ratio': 0.765625, 'completions/mean_terminated_length': 380.8000183105469, 'completions/min_terminated_length': 180.0, 'completions/max_terminated_length': 697.0, 'rollout/action_tokens_mean': 786.5625, 'rollout/episode_turns_mean': 25.984375, 'rollout/truncated_ratio': 0.765625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.1939062476158142, 'rewards/alfworld_rollout_reward_func/std': 0.44521304965019226, 'reward': 0.1939062476158142, 'reward_std': 0.31658345460891724, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.0203598253428936, 'sampling/sampling_logp_difference/max': 2.8486413955688477, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.6038376092910767, 'sampling/importance_sampling_ratio/max': 2.8437609672546387, 'kl': 0.0017176041819766397, 'entropy': 0.32334376219660044, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 434.73393120799847, 'epoch': 0.06}

 12%|█▎        | 8/64 [50:14<6:07:05, 393.30s/it]
 14%|█▍        | 9/64 [57:00<6:04:01, 397.12s/it]
                                                 
{'loss': -0.0089, 'grad_norm': 0.006268487311899662, 'learning_rate': 4.8853481410001225e-06, 'num_tokens': 456612.0, 'completions/mean_length': 818.34375, 'completions/min_length': 192.0, 'completions/max_length': 1492.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 474.0625, 'completions/min_terminated_length': 192.0, 'completions/max_terminated_length': 1027.0, 'rollout/action_tokens_mean': 818.34375, 'rollout/episode_turns_mean': 26.109375, 'rollout/truncated_ratio': 0.75, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.19500000774860382, 'rewards/alfworld_rollout_reward_func/std': 0.46079298853874207, 'reward': 0.19500000774860382, 'reward_std': 0.29811495542526245, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.019470183178782463, 'sampling/sampling_logp_difference/max': 3.3629698753356934, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.42990007996559143, 'sampling/importance_sampling_ratio/max': 2.6596250534057617, 'kl': 0.0026504553206905257, 'entropy': 0.32139473129063845, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 399.7356371579963, 'epoch': 0.07}

 14%|█▍        | 9/64 [57:00<6:04:01, 397.12s/it]Warning: BPE mismatch at turn 1 (expected prefix 613, got 775 tokens). Attempting delta recovery.

 16%|█▌        | 10/64 [1:03:39<5:57:59, 397.77s/it]
                                                    
{'loss': 0.0042, 'grad_norm': 0.010129230096936226, 'learning_rate': 4.844380330367701e-06, 'num_tokens': 502071.0, 'completions/mean_length': 710.296875, 'completions/min_length': 193.0, 'completions/max_length': 1381.0, 'completions/clipped_ratio': 0.59375, 'completions/mean_terminated_length': 485.5384826660156, 'completions/min_terminated_length': 193.0, 'completions/max_terminated_length': 1015.0, 'rollout/action_tokens_mean': 710.296875, 'rollout/episode_turns_mean': 24.859375, 'rollout/truncated_ratio': 0.59375, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.3684375286102295, 'rewards/alfworld_rollout_reward_func/std': 0.5042713284492493, 'reward': 0.3684375286102295, 'reward_std': 0.3336261808872223, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022041790187358856, 'sampling/sampling_logp_difference/max': 3.2723636627197266, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5169863104820251, 'sampling/importance_sampling_ratio/max': 2.386782646179199, 'kl': 0.0015055066687637009, 'entropy': 0.34738565888255835, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 394.5470729360013, 'epoch': 0.08}

 16%|█▌        | 10/64 [1:03:39<5:57:59, 397.77s/it]
 17%|█▋        | 11/64 [1:10:22<5:52:38, 399.22s/it]
                                                    
{'loss': 0.0111, 'grad_norm': 0.00803722906857729, 'learning_rate': 4.797394529050577e-06, 'num_tokens': 558509.0, 'completions/mean_length': 881.84375, 'completions/min_length': 224.0, 'completions/max_length': 1352.0, 'completions/clipped_ratio': 0.859375, 'completions/mean_terminated_length': 523.4444580078125, 'completions/min_terminated_length': 224.0, 'completions/max_terminated_length': 782.0, 'rollout/action_tokens_mean': 881.84375, 'rollout/episode_turns_mean': 28.40625, 'rollout/truncated_ratio': 0.859375, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.08921875059604645, 'rewards/alfworld_rollout_reward_func/std': 0.36485835909843445, 'reward': 0.08921875059604645, 'reward_std': 0.2777005434036255, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02142297849059105, 'sampling/sampling_logp_difference/max': 3.038581132888794, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4752790629863739, 'sampling/importance_sampling_ratio/max': 2.7719504833221436, 'kl': 0.0017023698201228399, 'entropy': 0.3616171991452575, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 396.1487058620023, 'epoch': 0.09}

 17%|█▋        | 11/64 [1:10:22<5:52:38, 399.22s/it]
 19%|█▉        | 12/64 [1:17:30<5:53:36, 408.01s/it]
                                                    
{'loss': 0.0048, 'grad_norm': 0.011235510930418968, 'learning_rate': 4.744511348926855e-06, 'num_tokens': 613567.0, 'completions/mean_length': 860.28125, 'completions/min_length': 116.0, 'completions/max_length': 2292.0, 'completions/clipped_ratio': 0.671875, 'completions/mean_terminated_length': 460.76190185546875, 'completions/min_terminated_length': 116.0, 'completions/max_terminated_length': 901.0, 'rollout/action_tokens_mean': 860.28125, 'rollout/episode_turns_mean': 25.484375, 'rollout/truncated_ratio': 0.671875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.26890626549720764, 'rewards/alfworld_rollout_reward_func/std': 0.4936746656894684, 'reward': 0.26890623569488525, 'reward_std': 0.41516801714897156, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02156790718436241, 'sampling/sampling_logp_difference/max': 2.6376895904541016, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5103189945220947, 'sampling/importance_sampling_ratio/max': 2.7596683502197266, 'kl': 0.0016268756971840048, 'entropy': 0.35516145126894116, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 421.574981908002, 'epoch': 0.1}

 19%|█▉        | 12/64 [1:17:30<5:53:36, 408.01s/it]
 20%|██        | 13/64 [1:23:45<5:38:23, 398.10s/it]
                                                    
{'loss': 0.0068, 'grad_norm': 0.010335148312151432, 'learning_rate': 4.685866540361456e-06, 'num_tokens': 667105.0, 'completions/mean_length': 836.53125, 'completions/min_length': 402.0, 'completions/max_length': 1382.0, 'completions/clipped_ratio': 0.8125, 'completions/mean_terminated_length': 586.4166870117188, 'completions/min_terminated_length': 402.0, 'completions/max_terminated_length': 775.0, 'rollout/action_tokens_mean': 836.53125, 'rollout/episode_turns_mean': 28.453125, 'rollout/truncated_ratio': 0.8125, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.14281249046325684, 'rewards/alfworld_rollout_reward_func/std': 0.3984073996543884, 'reward': 0.14281249046325684, 'reward_std': 0.3198840916156769, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.021617209538817406, 'sampling/sampling_logp_difference/max': 2.14093017578125, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.6008636951446533, 'sampling/importance_sampling_ratio/max': 2.6870784759521484, 'kl': 0.0018077169770549517, 'entropy': 0.35410969890654087, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 369.507772967002, 'epoch': 0.1}

 20%|██        | 13/64 [1:23:45<5:38:23, 398.10s/it]
 22%|██▏       | 14/64 [1:29:18<5:15:23, 378.48s/it]
                                                    
{'loss': 0.0062, 'grad_norm': 0.00711849657818675, 'learning_rate': 4.621610643736878e-06, 'num_tokens': 717902.0, 'completions/mean_length': 793.703125, 'completions/min_length': 158.0, 'completions/max_length': 1584.0, 'completions/clipped_ratio': 0.6875, 'completions/mean_terminated_length': 525.4500122070312, 'completions/min_terminated_length': 158.0, 'completions/max_terminated_length': 1063.0, 'rollout/action_tokens_mean': 793.703125, 'rollout/episode_turns_mean': 26.28125, 'rollout/truncated_ratio': 0.6875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.26265624165534973, 'rewards/alfworld_rollout_reward_func/std': 0.489940881729126, 'reward': 0.2626562714576721, 'reward_std': 0.3680095076560974, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.0205212514847517, 'sampling/sampling_logp_difference/max': 2.7529265880584717, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.45154738426208496, 'sampling/importance_sampling_ratio/max': 2.815065860748291, 'kl': 0.0019163258293701801, 'entropy': 0.34002805408090353, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 327.6827599140015, 'epoch': 0.11}

 22%|██▏       | 14/64 [1:29:18<5:15:23, 378.48s/it]Warning: BPE mismatch at turn 25 (expected prefix 7684, got 8136 tokens). Attempting delta recovery.

 23%|██▎       | 15/64 [1:36:15<5:18:30, 390.01s/it]
                                                    
{'loss': -0.0076, 'grad_norm': 0.010462122038006783, 'learning_rate': 4.551908603018191e-06, 'num_tokens': 776085.0, 'completions/mean_length': 909.109375, 'completions/min_length': 242.0, 'completions/max_length': 1814.0, 'completions/clipped_ratio': 0.84375, 'completions/mean_terminated_length': 528.7999877929688, 'completions/min_terminated_length': 242.0, 'completions/max_terminated_length': 890.0, 'rollout/action_tokens_mean': 909.109375, 'rollout/episode_turns_mean': 28.0, 'rollout/truncated_ratio': 0.84375, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.10140625387430191, 'rewards/alfworld_rollout_reward_func/std': 0.38891327381134033, 'reward': 0.10140626132488251, 'reward_std': 0.36953580379486084, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.021498428657650948, 'sampling/sampling_logp_difference/max': 4.642125606536865, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.3665631115436554, 'sampling/importance_sampling_ratio/max': 2.0437681674957275, 'kl': 0.0016676104332873365, 'entropy': 0.35556588135659695, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 409.45475878100024, 'epoch': 0.12}

 23%|██▎       | 15/64 [1:36:15<5:18:30, 390.01s/it]
 25%|██▌       | 16/64 [1:42:00<5:01:10, 376.47s/it]
                                                    
{'loss': -0.002, 'grad_norm': 0.007739651948213577, 'learning_rate': 4.476939342344246e-06, 'num_tokens': 831028.0, 'completions/mean_length': 858.484375, 'completions/min_length': 128.0, 'completions/max_length': 2069.0, 'completions/clipped_ratio': 0.703125, 'completions/mean_terminated_length': 363.0, 'completions/min_terminated_length': 128.0, 'completions/max_terminated_length': 815.0, 'rollout/action_tokens_mean': 858.484375, 'rollout/episode_turns_mean': 25.015625, 'rollout/truncated_ratio': 0.703125, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.2515625059604645, 'rewards/alfworld_rollout_reward_func/std': 0.48498478531837463, 'reward': 0.2515625059604645, 'reward_std': 0.4047408103942871, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.021193569526076317, 'sampling/sampling_logp_difference/max': 1.3733601570129395, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.3564271330833435, 'sampling/importance_sampling_ratio/max': 2.197270631790161, 'kl': 0.0015719706807431066, 'entropy': 0.3426384939812124, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 341.3529842470016, 'epoch': 0.13}

 25%|██▌       | 16/64 [1:42:00<5:01:10, 376.47s/it]
 27%|██▋       | 17/64 [1:49:23<5:10:30, 396.40s/it]
                                                    
{'loss': 0.0005, 'grad_norm': 1.4890336990356445, 'learning_rate': 4.396895306731978e-06, 'num_tokens': 882285.0, 'completions/mean_length': 800.890625, 'completions/min_length': 199.0, 'completions/max_length': 1344.0, 'completions/clipped_ratio': 0.765625, 'completions/mean_terminated_length': 410.0000305175781, 'completions/min_terminated_length': 199.0, 'completions/max_terminated_length': 872.0, 'rollout/action_tokens_mean': 800.890625, 'rollout/episode_turns_mean': 26.265625, 'rollout/truncated_ratio': 0.765625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.18124999105930328, 'rewards/alfworld_rollout_reward_func/std': 0.4482647478580475, 'reward': 0.18125000596046448, 'reward_std': 0.24295468628406525, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020599324256181717, 'sampling/sampling_logp_difference/max': 10.240796089172363, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.37133079767227173, 'sampling/importance_sampling_ratio/max': 2.3350934982299805, 'kl': 0.8330444995917787, 'entropy': 0.3335589300841093, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 436.20165634099976, 'epoch': 0.14}

 27%|██▋       | 17/64 [1:49:23<5:10:30, 396.40s/it]
 28%|██▊       | 18/64 [1:56:46<5:14:51, 410.68s/it]
                                                    
{'loss': -0.0099, 'grad_norm': 0.007197697646915913, 'learning_rate': 4.3119819680728e-06, 'num_tokens': 937622.0, 'completions/mean_length': 864.640625, 'completions/min_length': 240.0, 'completions/max_length': 1359.0, 'completions/clipped_ratio': 0.890625, 'completions/mean_terminated_length': 638.7142944335938, 'completions/min_terminated_length': 240.0, 'completions/max_terminated_length': 872.0, 'rollout/action_tokens_mean': 864.640625, 'rollout/episode_turns_mean': 28.9375, 'rollout/truncated_ratio': 0.890625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.06968750059604645, 'rewards/alfworld_rollout_reward_func/std': 0.3224163353443146, 'reward': 0.06968750059604645, 'reward_std': 0.19933855533599854, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.019858187064528465, 'sampling/sampling_logp_difference/max': 2.0254077911376953, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5041627287864685, 'sampling/importance_sampling_ratio/max': 2.2717995643615723, 'kl': 0.0016915635187615408, 'entropy': 0.3253911091014743, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 437.1889309570033, 'epoch': 0.14}

 28%|██▊       | 18/64 [1:56:47<5:14:51, 410.68s/it]
 30%|██▉       | 19/64 [2:02:40<4:55:12, 393.60s/it]
                                                    
{'loss': 0.0213, 'grad_norm': 0.010224551893770695, 'learning_rate': 4.222417297689217e-06, 'num_tokens': 991670.0, 'completions/mean_length': 844.5, 'completions/min_length': 140.0, 'completions/max_length': 2201.0, 'completions/clipped_ratio': 0.703125, 'completions/mean_terminated_length': 540.631591796875, 'completions/min_terminated_length': 140.0, 'completions/max_terminated_length': 1115.0, 'rollout/action_tokens_mean': 844.5, 'rollout/episode_turns_mean': 26.171875, 'rollout/truncated_ratio': 0.703125, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.24437500536441803, 'rewards/alfworld_rollout_reward_func/std': 0.48038968443870544, 'reward': 0.24437499046325684, 'reward_std': 0.42425790429115295, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01941317319869995, 'sampling/sampling_logp_difference/max': 1.632045865058899, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5376668572425842, 'sampling/importance_sampling_ratio/max': 2.5866730213165283, 'kl': 0.001574573227117071, 'entropy': 0.33306339103728533, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 348.9997736020032, 'epoch': 0.15}

 30%|██▉       | 19/64 [2:02:40<4:55:12, 393.60s/it]
 31%|███▏      | 20/64 [2:08:39<4:40:53, 383.03s/it]
                                                    
{'loss': 0.0044, 'grad_norm': 0.010612000711262226, 'learning_rate': 4.128431206805556e-06, 'num_tokens': 1051679.0, 'completions/mean_length': 937.640625, 'completions/min_length': 201.0, 'completions/max_length': 1764.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 530.6875, 'completions/min_terminated_length': 201.0, 'completions/max_terminated_length': 1085.0, 'rollout/action_tokens_mean': 937.640625, 'rollout/episode_turns_mean': 26.6875, 'rollout/truncated_ratio': 0.75, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.1901562511920929, 'rewards/alfworld_rollout_reward_func/std': 0.4530303478240967, 'reward': 0.1901562511920929, 'reward_std': 0.39348262548446655, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.0198683924973011, 'sampling/sampling_logp_difference/max': 2.34731388092041, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4413934350013733, 'sampling/importance_sampling_ratio/max': 2.5002288818359375, 'kl': 0.001602184345756541, 'entropy': 0.3477782104164362, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 353.7044870629943, 'epoch': 0.16}

 31%|███▏      | 20/64 [2:08:39<4:40:53, 383.03s/it]
 33%|███▎      | 21/64 [2:16:17<4:50:40, 405.59s/it]
                                                    
{'loss': -0.0018, 'grad_norm': 0.010538731701672077, 'learning_rate': 4.030264956369158e-06, 'num_tokens': 1102908.0, 'completions/mean_length': 800.453125, 'completions/min_length': 99.0, 'completions/max_length': 1475.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 418.0909118652344, 'completions/min_terminated_length': 99.0, 'completions/max_terminated_length': 939.0, 'rollout/action_tokens_mean': 800.453125, 'rollout/episode_turns_mean': 24.4375, 'rollout/truncated_ratio': 0.65625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.2993749976158142, 'rewards/alfworld_rollout_reward_func/std': 0.4982314109802246, 'reward': 0.2993749976158142, 'reward_std': 0.40504124760627747, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022293493151664734, 'sampling/sampling_logp_difference/max': 1.2475838661193848, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.448123574256897, 'sampling/importance_sampling_ratio/max': 2.447692632675171, 'kl': 0.001873640143458033, 'entropy': 0.36278699059039354, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 452.2219315199982, 'epoch': 0.17}

 33%|███▎      | 21/64 [2:16:17<4:50:40, 405.59s/it]
 34%|███▍      | 22/64 [2:22:12<4:33:23, 390.56s/it]
                                                    
{'loss': 0.0059, 'grad_norm': 0.007643942255526781, 'learning_rate': 3.9281705377369814e-06, 'num_tokens': 1150505.0, 'completions/mean_length': 743.703125, 'completions/min_length': 137.0, 'completions/max_length': 1566.0, 'completions/clipped_ratio': 0.625, 'completions/mean_terminated_length': 460.91668701171875, 'completions/min_terminated_length': 137.0, 'completions/max_terminated_length': 929.0, 'rollout/action_tokens_mean': 743.703125, 'rollout/episode_turns_mean': 24.796875, 'rollout/truncated_ratio': 0.625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.32624998688697815, 'rewards/alfworld_rollout_reward_func/std': 0.5121089816093445, 'reward': 0.32625001668930054, 'reward_std': 0.4726526141166687, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020861441269516945, 'sampling/sampling_logp_difference/max': 2.366598129272461, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4393410086631775, 'sampling/importance_sampling_ratio/max': 2.206357479095459, 'kl': 0.0019213874111301266, 'entropy': 0.3329707160592079, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 350.210106904995, 'epoch': 0.18}

 34%|███▍      | 22/64 [2:22:12<4:33:23, 390.56s/it]
 36%|███▌      | 23/64 [2:27:16<4:09:03, 364.48s/it]
                                                    
{'loss': 0.0064, 'grad_norm': 0.010937543585896492, 'learning_rate': 3.8224100258174066e-06, 'num_tokens': 1201566.0, 'completions/mean_length': 797.828125, 'completions/min_length': 87.0, 'completions/max_length': 1494.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 504.04547119140625, 'completions/min_terminated_length': 87.0, 'completions/max_terminated_length': 1159.0, 'rollout/action_tokens_mean': 797.828125, 'rollout/episode_turns_mean': 25.328125, 'rollout/truncated_ratio': 0.65625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.3057812452316284, 'rewards/alfworld_rollout_reward_func/std': 0.49343568086624146, 'reward': 0.3057812452316284, 'reward_std': 0.36121153831481934, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022007007151842117, 'sampling/sampling_logp_difference/max': 1.2014861106872559, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.43328696489334106, 'sampling/importance_sampling_ratio/max': 2.8112552165985107, 'kl': 0.00202080328745069, 'entropy': 0.35823060059919953, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 299.4183257309978, 'epoch': 0.18}

 36%|███▌      | 23/64 [2:27:16<4:09:03, 364.48s/it]Warning: BPE mismatch at turn 20 (expected prefix 3325, got 3449 tokens). Attempting delta recovery.

 38%|███▊      | 24/64 [2:34:28<4:16:31, 384.78s/it]
                                                    
{'loss': -0.0022, 'grad_norm': 0.008087356574833393, 'learning_rate': 3.7132549063277033e-06, 'num_tokens': 1249556.0, 'completions/mean_length': 749.84375, 'completions/min_length': 135.0, 'completions/max_length': 1409.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 406.0, 'completions/min_terminated_length': 135.0, 'completions/max_terminated_length': 962.0, 'rollout/action_tokens_mean': 749.84375, 'rollout/episode_turns_mean': 25.53125, 'rollout/truncated_ratio': 0.75, 'rollout/mismatch_mean': 0.03125, 'rollout/mismatch_ratio': 0.03125, 'rewards/alfworld_rollout_reward_func/mean': 0.20937500894069672, 'rewards/alfworld_rollout_reward_func/std': 0.45049840211868286, 'reward': 0.20937500894069672, 'reward_std': 0.2735254764556885, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.021238919347524643, 'sampling/sampling_logp_difference/max': 2.2067627906799316, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4837469756603241, 'sampling/importance_sampling_ratio/max': 2.822704792022705, 'kl': 0.0017565299822308589, 'entropy': 0.314591265283525, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 423.74345705999986, 'epoch': 0.19}

 38%|███▊      | 24/64 [2:34:28<4:16:31, 384.78s/it]
 39%|███▉      | 25/64 [2:40:53<4:10:03, 384.72s/it]
                                                    
{'loss': -0.0029, 'grad_norm': 0.005950183607637882, 'learning_rate': 3.600985378894086e-06, 'num_tokens': 1298023.0, 'completions/mean_length': 757.296875, 'completions/min_length': 211.0, 'completions/max_length': 1469.0, 'completions/clipped_ratio': 0.828125, 'completions/mean_terminated_length': 445.2727355957031, 'completions/min_terminated_length': 211.0, 'completions/max_terminated_length': 854.0, 'rollout/action_tokens_mean': 757.296875, 'rollout/episode_turns_mean': 27.515625, 'rollout/truncated_ratio': 0.828125, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.12843750417232513, 'rewards/alfworld_rollout_reward_func/std': 0.39419689774513245, 'reward': 0.12843748927116394, 'reward_std': 0.3103031814098358, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.0199856199324131, 'sampling/sampling_logp_difference/max': 3.0986688137054443, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.30797556042671204, 'sampling/importance_sampling_ratio/max': 1.9042965173721313, 'kl': 0.0024588965916336747, 'entropy': 0.3146928045898676, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 379.6568018289963, 'epoch': 0.2}

 39%|███▉      | 25/64 [2:40:53<4:10:03, 384.72s/it]
 41%|████      | 26/64 [2:48:40<4:19:18, 409.43s/it]
                                                    
{'loss': 0.0009, 'grad_norm': 0.006396389566361904, 'learning_rate': 3.4858896377832966e-06, 'num_tokens': 1350535.0, 'completions/mean_length': 820.5, 'completions/min_length': 174.0, 'completions/max_length': 1607.0, 'completions/clipped_ratio': 0.796875, 'completions/mean_terminated_length': 396.923095703125, 'completions/min_terminated_length': 174.0, 'completions/max_terminated_length': 975.0, 'rollout/action_tokens_mean': 820.5, 'rollout/episode_turns_mean': 26.59375, 'rollout/truncated_ratio': 0.796875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.1639062464237213, 'rewards/alfworld_rollout_reward_func/std': 0.418832927942276, 'reward': 0.1639062464237213, 'reward_std': 0.32607388496398926, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.018608851358294487, 'sampling/sampling_logp_difference/max': 3.6588082313537598, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.46614933013916016, 'sampling/importance_sampling_ratio/max': 2.7253878116607666, 'kl': 0.001548040645502624, 'entropy': 0.301429130602628, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 458.5120370709956, 'epoch': 0.21}

 41%|████      | 26/64 [2:48:40<4:19:18, 409.43s/it]
 42%|████▏     | 27/64 [2:54:52<4:05:30, 398.13s/it]
                                                    
{'loss': -0.0109, 'grad_norm': 0.00987387914210558, 'learning_rate': 3.3682631321120507e-06, 'num_tokens': 1404517.0, 'completions/mean_length': 843.46875, 'completions/min_length': 123.0, 'completions/max_length': 1363.0, 'completions/clipped_ratio': 0.8125, 'completions/mean_terminated_length': 518.0833740234375, 'completions/min_terminated_length': 123.0, 'completions/max_terminated_length': 851.0, 'rollout/action_tokens_mean': 843.46875, 'rollout/episode_turns_mean': 27.703125, 'rollout/truncated_ratio': 0.8125, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.14921875298023224, 'rewards/alfworld_rollout_reward_func/std': 0.40770089626312256, 'reward': 0.14921875298023224, 'reward_std': 0.3126556873321533, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020594514906406403, 'sampling/sampling_logp_difference/max': 1.8558279275894165, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4162726402282715, 'sampling/importance_sampling_ratio/max': 2.323033332824707, 'kl': 0.001589072278875392, 'entropy': 0.3490098095498979, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 366.20077905700964, 'epoch': 0.22}

 42%|████▏     | 27/64 [2:54:52<4:05:30, 398.13s/it]
 44%|████▍     | 28/64 [3:00:15<3:45:23, 375.66s/it]
                                                    
{'loss': 0.012, 'grad_norm': 0.01545465737581253, 'learning_rate': 3.248407807433396e-06, 'num_tokens': 1451100.0, 'completions/mean_length': 727.859375, 'completions/min_length': 97.0, 'completions/max_length': 2369.0, 'completions/clipped_ratio': 0.5625, 'completions/mean_terminated_length': 404.3571472167969, 'completions/min_terminated_length': 97.0, 'completions/max_terminated_length': 1151.0, 'rollout/action_tokens_mean': 727.859375, 'rollout/episode_turns_mean': 22.984375, 'rollout/truncated_ratio': 0.5625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.3981249928474426, 'rewards/alfworld_rollout_reward_func/std': 0.5216788649559021, 'reward': 0.3981249928474426, 'reward_std': 0.5213440656661987, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.019816581159830093, 'sampling/sampling_logp_difference/max': 1.65842604637146, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.6839978694915771, 'sampling/importance_sampling_ratio/max': 2.7356457710266113, 'kl': 0.0016130298517964548, 'entropy': 0.30788369150832295, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 317.7198553449962, 'epoch': 0.22}

 44%|████▍     | 28/64 [3:00:15<3:45:23, 375.66s/it]Warning: BPE mismatch at turn 1 (expected prefix 644, got 903 tokens). Attempting delta recovery.

 45%|████▌     | 29/64 [3:06:31<3:39:17, 375.93s/it]
                                                    
{'loss': -0.0153, 'grad_norm': 0.010329142212867737, 'learning_rate': 3.1266313306468018e-06, 'num_tokens': 1505118.0, 'completions/mean_length': 844.03125, 'completions/min_length': 153.0, 'completions/max_length': 1781.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 524.75, 'completions/min_terminated_length': 153.0, 'completions/max_terminated_length': 980.0, 'rollout/action_tokens_mean': 844.03125, 'rollout/episode_turns_mean': 26.890625, 'rollout/truncated_ratio': 0.75, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.20656250417232513, 'rewards/alfworld_rollout_reward_func/std': 0.4480122923851013, 'reward': 0.20656250417232513, 'reward_std': 0.29942071437835693, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022610869258642197, 'sampling/sampling_logp_difference/max': 5.319100856781006, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.39373910427093506, 'sampling/importance_sampling_ratio/max': 2.6475582122802734, 'kl': 0.0034811663499567658, 'entropy': 0.3514281096868217, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 372.01922321400525, 'epoch': 0.23}

 45%|████▌     | 29/64 [3:06:31<3:39:17, 375.93s/it]
 47%|████▋     | 30/64 [3:13:00<3:35:10, 379.72s/it]
                                                    
{'loss': 0.0038, 'grad_norm': 0.006770053878426552, 'learning_rate': 3.0032463002216504e-06, 'num_tokens': 1555119.0, 'completions/mean_length': 781.265625, 'completions/min_length': 172.0, 'completions/max_length': 1465.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 451.8125, 'completions/min_terminated_length': 172.0, 'completions/max_terminated_length': 804.0, 'rollout/action_tokens_mean': 781.265625, 'rollout/episode_turns_mean': 26.21875, 'rollout/truncated_ratio': 0.75, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.21671876311302185, 'rewards/alfworld_rollout_reward_func/std': 0.44675213098526, 'reward': 0.21671874821186066, 'reward_std': 0.24240338802337646, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01828092336654663, 'sampling/sampling_logp_difference/max': 1.7825241088867188, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.49506521224975586, 'sampling/importance_sampling_ratio/max': 2.8479506969451904, 'kl': 0.0015057786895340541, 'entropy': 0.2927012937143445, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 382.84014304800075, 'epoch': 0.24}

 47%|████▋     | 30/64 [3:13:00<3:35:10, 379.72s/it]
 48%|████▊     | 31/64 [3:21:21<3:48:56, 416.24s/it]
                                                    
{'loss': -0.0078, 'grad_norm': 0.009322401136159897, 'learning_rate': 2.878569443761442e-06, 'num_tokens': 1606771.0, 'completions/mean_length': 807.0625, 'completions/min_length': 153.0, 'completions/max_length': 1591.0, 'completions/clipped_ratio': 0.8125, 'completions/mean_terminated_length': 503.8333435058594, 'completions/min_terminated_length': 153.0, 'completions/max_terminated_length': 899.0, 'rollout/action_tokens_mean': 807.0625, 'rollout/episode_turns_mean': 27.546875, 'rollout/truncated_ratio': 0.8125, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.14656248688697815, 'rewards/alfworld_rollout_reward_func/std': 0.4100453853607178, 'reward': 0.14656250178813934, 'reward_std': 0.337169349193573, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.021321609616279602, 'sampling/sampling_logp_difference/max': 2.855207920074463, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.42428821325302124, 'sampling/importance_sampling_ratio/max': 2.9021573066711426, 'kl': 0.0018803979564836482, 'entropy': 0.33275228925049305, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 494.6054373920015, 'epoch': 0.25}

 48%|████▊     | 31/64 [3:21:21<3:48:56, 416.24s/it]
 50%|█████     | 32/64 [3:26:32<3:25:01, 384.42s/it]
                                                    
{'loss': -0.0093, 'grad_norm': 0.010985196568071842, 'learning_rate': 2.752920804968581e-06, 'num_tokens': 1657442.0, 'completions/mean_length': 791.734375, 'completions/min_length': 125.0, 'completions/max_length': 1383.0, 'completions/clipped_ratio': 0.796875, 'completions/mean_terminated_length': 412.923095703125, 'completions/min_terminated_length': 125.0, 'completions/max_terminated_length': 1038.0, 'rollout/action_tokens_mean': 791.734375, 'rollout/episode_turns_mean': 26.5, 'rollout/truncated_ratio': 0.796875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.14656250178813934, 'rewards/alfworld_rollout_reward_func/std': 0.4263809323310852, 'reward': 0.14656248688697815, 'reward_std': 0.27167972922325134, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022872522473335266, 'sampling/sampling_logp_difference/max': 2.398648977279663, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.39966434240341187, 'sampling/importance_sampling_ratio/max': 2.170595169067383, 'kl': 0.0017831376208050642, 'entropy': 0.38143961410969496, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 305.8924537160019, 'epoch': 0.26}

 50%|█████     | 32/64 [3:26:32<3:25:01, 384.42s/it]
 52%|█████▏    | 33/64 [3:32:37<3:15:37, 378.62s/it]
                                                    
{'loss': -0.0045, 'grad_norm': 0.010168999433517456, 'learning_rate': 2.626622922096782e-06, 'num_tokens': 1715999.0, 'completions/mean_length': 914.953125, 'completions/min_length': 291.0, 'completions/max_length': 1564.0, 'completions/clipped_ratio': 0.734375, 'completions/mean_terminated_length': 616.4705810546875, 'completions/min_terminated_length': 291.0, 'completions/max_terminated_length': 1151.0, 'rollout/action_tokens_mean': 914.953125, 'rollout/episode_turns_mean': 27.140625, 'rollout/truncated_ratio': 0.734375, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.21843750774860382, 'rewards/alfworld_rollout_reward_func/std': 0.46011462807655334, 'reward': 0.21843750774860382, 'reward_std': 0.4097728729248047, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02191011980175972, 'sampling/sampling_logp_difference/max': 2.2772090435028076, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.386021226644516, 'sampling/importance_sampling_ratio/max': 2.6107707023620605, 'kl': 0.0017962153833650518, 'entropy': 0.39743291586637497, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 360.4347860389935, 'epoch': 0.26}

 52%|█████▏    | 33/64 [3:32:37<3:15:37, 378.62s/it]
 53%|█████▎    | 34/64 [3:36:55<2:51:18, 342.62s/it]
                                                    
{'loss': 0.016, 'grad_norm': 0.01134358998388052, 'learning_rate': 2.5e-06, 'num_tokens': 1771227.0, 'completions/mean_length': 862.9375, 'completions/min_length': 254.0, 'completions/max_length': 1527.0, 'completions/clipped_ratio': 0.796875, 'completions/mean_terminated_length': 506.3077087402344, 'completions/min_terminated_length': 254.0, 'completions/max_terminated_length': 743.0, 'rollout/action_tokens_mean': 862.9375, 'rollout/episode_turns_mean': 27.515625, 'rollout/truncated_ratio': 0.796875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.15781249105930328, 'rewards/alfworld_rollout_reward_func/std': 0.4202209413051605, 'reward': 0.15781250596046448, 'reward_std': 0.29862138628959656, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020839743316173553, 'sampling/sampling_logp_difference/max': 2.3659703731536865, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.41097065806388855, 'sampling/importance_sampling_ratio/max': 2.3946166038513184, 'kl': 0.001760420509526739, 'entropy': 0.35848523303866386, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 255.86244797499785, 'epoch': 0.27}

 53%|█████▎    | 34/64 [3:36:55<2:51:18, 342.62s/it]
 55%|█████▍    | 35/64 [3:43:10<2:50:19, 352.39s/it]
                                                    
{'loss': 0.0012, 'grad_norm': 0.007748698350042105, 'learning_rate': 2.3733770779032185e-06, 'num_tokens': 1822496.0, 'completions/mean_length': 801.078125, 'completions/min_length': 121.0, 'completions/max_length': 1503.0, 'completions/clipped_ratio': 0.640625, 'completions/mean_terminated_length': 446.0434875488281, 'completions/min_terminated_length': 121.0, 'completions/max_terminated_length': 1127.0, 'rollout/action_tokens_mean': 801.078125, 'rollout/episode_turns_mean': 24.203125, 'rollout/truncated_ratio': 0.640625, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.3109375238418579, 'rewards/alfworld_rollout_reward_func/std': 0.5012863874435425, 'reward': 0.3109375238418579, 'reward_std': 0.34154438972473145, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020977256819605827, 'sampling/sampling_logp_difference/max': 2.4488577842712402, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4648974537849426, 'sampling/importance_sampling_ratio/max': 2.6798601150512695, 'kl': 0.0017098757416533772, 'entropy': 0.33974334690719843, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 369.7442883429976, 'epoch': 0.28}

 55%|█████▍    | 35/64 [3:43:10<2:50:19, 352.39s/it]
 56%|█████▋    | 36/64 [3:50:54<2:59:59, 385.70s/it]
                                                    
{'loss': 0.0067, 'grad_norm': 0.012169571593403816, 'learning_rate': 2.24707919503142e-06, 'num_tokens': 1871333.0, 'completions/mean_length': 763.078125, 'completions/min_length': 90.0, 'completions/max_length': 1817.0, 'completions/clipped_ratio': 0.5625, 'completions/mean_terminated_length': 383.21429443359375, 'completions/min_terminated_length': 90.0, 'completions/max_terminated_length': 1098.0, 'rollout/action_tokens_mean': 763.078125, 'rollout/episode_turns_mean': 22.328125, 'rollout/truncated_ratio': 0.5625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.3814062476158142, 'rewards/alfworld_rollout_reward_func/std': 0.5243707299232483, 'reward': 0.3814062476158142, 'reward_std': 0.325703501701355, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022463176399469376, 'sampling/sampling_logp_difference/max': 3.9331345558166504, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4907624125480652, 'sampling/importance_sampling_ratio/max': 2.630308151245117, 'kl': 0.0024242461659014225, 'entropy': 0.37786182574927807, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 452.8674500970046, 'epoch': 0.29}

 56%|█████▋    | 36/64 [3:50:54<2:59:59, 385.70s/it]
 58%|█████▊    | 37/64 [3:57:27<2:54:29, 387.77s/it]
                                                    
{'loss': 0.0028, 'grad_norm': 0.009033096954226494, 'learning_rate': 2.1214305562385592e-06, 'num_tokens': 1909281.0, 'completions/mean_length': 592.9375, 'completions/min_length': 102.0, 'completions/max_length': 1560.0, 'completions/clipped_ratio': 0.453125, 'completions/mean_terminated_length': 322.4285583496094, 'completions/min_terminated_length': 102.0, 'completions/max_terminated_length': 1173.0, 'rollout/action_tokens_mean': 592.9375, 'rollout/episode_turns_mean': 19.015625, 'rollout/truncated_ratio': 0.453125, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.5143750309944153, 'rewards/alfworld_rollout_reward_func/std': 0.5290928483009338, 'reward': 0.5143750309944153, 'reward_std': 0.3860568404197693, 'frac_reward_zero_std': 0.125, 'sampling/sampling_logp_difference/mean': 0.01983683556318283, 'sampling/sampling_logp_difference/max': 2.304647445678711, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5531047582626343, 'sampling/importance_sampling_ratio/max': 2.8567276000976562, 'kl': 0.0017318885384156602, 'entropy': 0.2960165496915579, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 388.3879080370025, 'epoch': 0.3}

 58%|█████▊    | 37/64 [3:57:27<2:54:29, 387.77s/it]Warning: BPE mismatch at turn 1 (expected prefix 653, got 908 tokens). Attempting delta recovery.

 59%|█████▉    | 38/64 [4:04:47<2:54:56, 403.73s/it]
                                                    
{'loss': 0.0014, 'grad_norm': 0.014773277565836906, 'learning_rate': 1.9967536997783495e-06, 'num_tokens': 1951658.0, 'completions/mean_length': 662.140625, 'completions/min_length': 77.0, 'completions/max_length': 1343.0, 'completions/clipped_ratio': 0.53125, 'completions/mean_terminated_length': 343.8333435058594, 'completions/min_terminated_length': 77.0, 'completions/max_terminated_length': 792.0, 'rollout/action_tokens_mean': 662.140625, 'rollout/episode_turns_mean': 21.40625, 'rollout/truncated_ratio': 0.53125, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.4350000023841858, 'rewards/alfworld_rollout_reward_func/std': 0.5278016924858093, 'reward': 0.4350000023841858, 'reward_std': 0.3387959599494934, 'frac_reward_zero_std': 0.125, 'sampling/sampling_logp_difference/mean': 0.022423237562179565, 'sampling/sampling_logp_difference/max': 3.768939971923828, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.563366711139679, 'sampling/importance_sampling_ratio/max': 2.9260754585266113, 'kl': 0.0018760439170364407, 'entropy': 0.32609579246491194, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 436.1079989810023, 'epoch': 0.3}

 59%|█████▉    | 38/64 [4:04:47<2:54:56, 403.73s/it]
 61%|██████    | 39/64 [4:10:47<2:42:41, 390.44s/it]
                                                    
{'loss': -0.0025, 'grad_norm': 0.01313227228820324, 'learning_rate': 1.8733686693531986e-06, 'num_tokens': 2012036.0, 'completions/mean_length': 943.40625, 'completions/min_length': 237.0, 'completions/max_length': 1869.0, 'completions/clipped_ratio': 0.734375, 'completions/mean_terminated_length': 584.4117431640625, 'completions/min_terminated_length': 237.0, 'completions/max_terminated_length': 1018.0, 'rollout/action_tokens_mean': 943.40625, 'rollout/episode_turns_mean': 26.90625, 'rollout/truncated_ratio': 0.734375, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.21578124165534973, 'rewards/alfworld_rollout_reward_func/std': 0.46428921818733215, 'reward': 0.21578124165534973, 'reward_std': 0.4323212504386902, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.021743088960647583, 'sampling/sampling_logp_difference/max': 3.389880657196045, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.3670840859413147, 'sampling/importance_sampling_ratio/max': 2.5490753650665283, 'kl': 0.0022639515991613735, 'entropy': 0.3758079996332526, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 354.37862886100265, 'epoch': 0.31}

 61%|██████    | 39/64 [4:10:47<2:42:41, 390.44s/it]
 62%|██████▎   | 40/64 [4:16:48<2:32:40, 381.70s/it]
                                                    
{'loss': -0.0206, 'grad_norm': 0.009722444228827953, 'learning_rate': 1.7515921925666053e-06, 'num_tokens': 2056910.0, 'completions/mean_length': 701.15625, 'completions/min_length': 155.0, 'completions/max_length': 1334.0, 'completions/clipped_ratio': 0.671875, 'completions/mean_terminated_length': 308.1428527832031, 'completions/min_terminated_length': 155.0, 'completions/max_terminated_length': 1062.0, 'rollout/action_tokens_mean': 701.15625, 'rollout/episode_turns_mean': 23.625, 'rollout/truncated_ratio': 0.671875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.29640626907348633, 'rewards/alfworld_rollout_reward_func/std': 0.4918813109397888, 'reward': 0.29640626907348633, 'reward_std': 0.23870769143104553, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01931983232498169, 'sampling/sampling_logp_difference/max': 3.3793482780456543, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5806776881217957, 'sampling/importance_sampling_ratio/max': 2.825843572616577, 'kl': 0.0016241020402958384, 'entropy': 0.2994346688501537, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 355.1650720970065, 'epoch': 0.32}

 62%|██████▎   | 40/64 [4:16:48<2:32:40, 381.70s/it]
 64%|██████▍   | 41/64 [4:23:18<2:27:12, 384.01s/it]
                                                    
{'loss': 0.0013, 'grad_norm': 0.01665978506207466, 'learning_rate': 1.6317368678879497e-06, 'num_tokens': 2111989.0, 'completions/mean_length': 860.609375, 'completions/min_length': 117.0, 'completions/max_length': 1877.0, 'completions/clipped_ratio': 0.71875, 'completions/mean_terminated_length': 526.0, 'completions/min_terminated_length': 117.0, 'completions/max_terminated_length': 997.0, 'rollout/action_tokens_mean': 860.609375, 'rollout/episode_turns_mean': 26.859375, 'rollout/truncated_ratio': 0.71875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.23156249523162842, 'rewards/alfworld_rollout_reward_func/std': 0.4703366458415985, 'reward': 0.23156249523162842, 'reward_std': 0.42284566164016724, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.019619014114141464, 'sampling/sampling_logp_difference/max': 2.0510313510894775, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.47027722001075745, 'sampling/importance_sampling_ratio/max': 2.746248960494995, 'kl': 0.001694458733254578, 'entropy': 0.33433423191308975, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 383.9065069159915, 'epoch': 0.33}

 64%|██████▍   | 41/64 [4:23:18<2:27:12, 384.01s/it]
 66%|██████▌   | 42/64 [4:29:06<2:16:54, 373.39s/it]
                                                    
{'loss': 0.0075, 'grad_norm': 0.011248192749917507, 'learning_rate': 1.5141103622167042e-06, 'num_tokens': 2156732.0, 'completions/mean_length': 699.109375, 'completions/min_length': 178.0, 'completions/max_length': 1491.0, 'completions/clipped_ratio': 0.578125, 'completions/mean_terminated_length': 410.9259338378906, 'completions/min_terminated_length': 178.0, 'completions/max_terminated_length': 1206.0, 'rollout/action_tokens_mean': 699.109375, 'rollout/episode_turns_mean': 23.015625, 'rollout/truncated_ratio': 0.578125, 'rollout/mismatch_mean': 0.03125, 'rollout/mismatch_ratio': 0.03125, 'rewards/alfworld_rollout_reward_func/mean': 0.37812501192092896, 'rewards/alfworld_rollout_reward_func/std': 0.5151386857032776, 'reward': 0.37812501192092896, 'reward_std': 0.3792918920516968, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02142202854156494, 'sampling/sampling_logp_difference/max': 2.1700198650360107, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.49075883626937866, 'sampling/importance_sampling_ratio/max': 2.8479621410369873, 'kl': 0.0020856883202213794, 'entropy': 0.3300422574393451, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 342.9056702470225, 'epoch': 0.34}

 66%|██████▌   | 42/64 [4:29:06<2:16:54, 373.39s/it]
 67%|██████▋   | 43/64 [4:37:00<2:21:14, 403.55s/it]
                                                    
{'loss': -0.0022, 'grad_norm': 0.013282958418130875, 'learning_rate': 1.3990146211059141e-06, 'num_tokens': 2206684.0, 'completions/mean_length': 780.5, 'completions/min_length': 209.0, 'completions/max_length': 1746.0, 'completions/clipped_ratio': 0.71875, 'completions/mean_terminated_length': 496.6111145019531, 'completions/min_terminated_length': 209.0, 'completions/max_terminated_length': 864.0, 'rollout/action_tokens_mean': 780.5, 'rollout/episode_turns_mean': 26.484375, 'rollout/truncated_ratio': 0.71875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.2423437535762787, 'rewards/alfworld_rollout_reward_func/std': 0.47146087884902954, 'reward': 0.2423437535762787, 'reward_std': 0.44438114762306213, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020154234021902084, 'sampling/sampling_logp_difference/max': 2.89083194732666, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.548378586769104, 'sampling/importance_sampling_ratio/max': 2.7073540687561035, 'kl': 0.0016229442680923967, 'entropy': 0.31365567492321134, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 465.0683610840024, 'epoch': 0.34}

 67%|██████▋   | 43/64 [4:37:00<2:21:14, 403.55s/it]Warning: BPE mismatch at turn 29 (expected prefix 5593, got 5708 tokens). Attempting delta recovery.

 69%|██████▉   | 44/64 [4:43:06<2:10:43, 392.16s/it]
                                                    
{'loss': 0.0006, 'grad_norm': 0.011168000288307667, 'learning_rate': 1.286745093672298e-06, 'num_tokens': 2263136.0, 'completions/mean_length': 882.0625, 'completions/min_length': 161.0, 'completions/max_length': 2028.0, 'completions/clipped_ratio': 0.765625, 'completions/mean_terminated_length': 563.0, 'completions/min_terminated_length': 161.0, 'completions/max_terminated_length': 1179.0, 'rollout/action_tokens_mean': 882.0625, 'rollout/episode_turns_mean': 26.578125, 'rollout/truncated_ratio': 0.765625, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.18671876192092896, 'rewards/alfworld_rollout_reward_func/std': 0.4461868703365326, 'reward': 0.18671876192092896, 'reward_std': 0.23547054827213287, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.019173569977283478, 'sampling/sampling_logp_difference/max': 8.617661476135254, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4952065646648407, 'sampling/importance_sampling_ratio/max': 2.5664165019989014, 'kl': 0.001302105893046246, 'entropy': 0.30738420970737934, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 361.35297568800524, 'epoch': 0.35}

 69%|██████▉   | 44/64 [4:43:06<2:10:43, 392.16s/it]
 70%|███████   | 45/64 [4:51:14<2:13:21, 421.14s/it]
                                                    
{'loss': 0.0031, 'grad_norm': 0.005941484589129686, 'learning_rate': 1.1775899741825947e-06, 'num_tokens': 2314846.0, 'completions/mean_length': 807.96875, 'completions/min_length': 172.0, 'completions/max_length': 1390.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 565.9091186523438, 'completions/min_terminated_length': 172.0, 'completions/max_terminated_length': 1011.0, 'rollout/action_tokens_mean': 807.96875, 'rollout/episode_turns_mean': 25.65625, 'rollout/truncated_ratio': 0.65625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.2996875047683716, 'rewards/alfworld_rollout_reward_func/std': 0.48867806792259216, 'reward': 0.2996875047683716, 'reward_std': 0.383698046207428, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.021218668669462204, 'sampling/sampling_logp_difference/max': 1.8846027851104736, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4718725383281708, 'sampling/importance_sampling_ratio/max': 1.910278081893921, 'kl': 0.001596126568983891, 'entropy': 0.33469977881759405, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 477.9487361019965, 'epoch': 0.36}

 70%|███████   | 45/64 [4:51:14<2:13:21, 421.14s/it]
 72%|███████▏  | 46/64 [4:57:31<2:02:20, 407.81s/it]
                                                    
{'loss': -0.007, 'grad_norm': 0.01500615943223238, 'learning_rate': 1.0718294622630188e-06, 'num_tokens': 2357037.0, 'completions/mean_length': 659.234375, 'completions/min_length': 106.0, 'completions/max_length': 1381.0, 'completions/clipped_ratio': 0.609375, 'completions/mean_terminated_length': 271.7200012207031, 'completions/min_terminated_length': 106.0, 'completions/max_terminated_length': 827.0, 'rollout/action_tokens_mean': 659.234375, 'rollout/episode_turns_mean': 21.640625, 'rollout/truncated_ratio': 0.609375, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.3592187464237213, 'rewards/alfworld_rollout_reward_func/std': 0.5135136246681213, 'reward': 0.3592187464237213, 'reward_std': 0.3250337839126587, 'frac_reward_zero_std': 0.125, 'sampling/sampling_logp_difference/mean': 0.02017129212617874, 'sampling/sampling_logp_difference/max': 1.5571508407592773, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.6121923923492432, 'sampling/importance_sampling_ratio/max': 2.703761100769043, 'kl': 0.0017743182979756966, 'entropy': 0.3191372868604958, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 368.9170330479901, 'epoch': 0.37}

 72%|███████▏  | 46/64 [4:57:31<2:02:20, 407.81s/it]
 73%|███████▎  | 47/64 [5:01:54<1:43:14, 364.41s/it]
                                                    
{'loss': 0.0171, 'grad_norm': 0.014015787281095982, 'learning_rate': 9.697350436308428e-07, 'num_tokens': 2397368.0, 'completions/mean_length': 630.171875, 'completions/min_length': 75.0, 'completions/max_length': 1449.0, 'completions/clipped_ratio': 0.46875, 'completions/mean_terminated_length': 346.20587158203125, 'completions/min_terminated_length': 75.0, 'completions/max_terminated_length': 879.0, 'rollout/action_tokens_mean': 630.171875, 'rollout/episode_turns_mean': 19.6875, 'rollout/truncated_ratio': 0.46875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.4931250214576721, 'rewards/alfworld_rollout_reward_func/std': 0.5321918725967407, 'reward': 0.4931250214576721, 'reward_std': 0.29200682044029236, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02060038223862648, 'sampling/sampling_logp_difference/max': 1.4474594593048096, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.6627721190452576, 'sampling/importance_sampling_ratio/max': 2.7606446743011475, 'kl': 0.001809486606362043, 'entropy': 0.33235412929207087, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 259.2473690390034, 'epoch': 0.38}

 73%|███████▎  | 47/64 [5:01:54<1:43:14, 364.41s/it]
 75%|███████▌  | 48/64 [5:08:25<1:39:16, 372.29s/it]
                                                    
{'loss': 0.0326, 'grad_norm': 0.014235694892704487, 'learning_rate': 8.71568793194445e-07, 'num_tokens': 2457758.0, 'completions/mean_length': 943.59375, 'completions/min_length': 207.0, 'completions/max_length': 2126.0, 'completions/clipped_ratio': 0.875, 'completions/mean_terminated_length': 424.0, 'completions/min_terminated_length': 207.0, 'completions/max_terminated_length': 1218.0, 'rollout/action_tokens_mean': 943.59375, 'rollout/episode_turns_mean': 27.734375, 'rollout/truncated_ratio': 0.875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.06859375536441803, 'rewards/alfworld_rollout_reward_func/std': 0.3545883595943451, 'reward': 0.06859375536441803, 'reward_std': 0.24723710119724274, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020999200642108917, 'sampling/sampling_logp_difference/max': 1.8059403896331787, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5324603319168091, 'sampling/importance_sampling_ratio/max': 2.791482925415039, 'kl': 0.0015301553139579482, 'entropy': 0.3577259639278054, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 384.36283810400346, 'epoch': 0.38}

 75%|███████▌  | 48/64 [5:08:25<1:39:16, 372.29s/it]Warning: BPE mismatch at turn 1 (expected prefix 767, got 973 tokens). Attempting delta recovery.

 77%|███████▋  | 49/64 [5:15:09<1:35:25, 381.72s/it]
                                                    
{'loss': 0.0044, 'grad_norm': 0.006656537763774395, 'learning_rate': 7.775827023107835e-07, 'num_tokens': 2512875.0, 'completions/mean_length': 861.203125, 'completions/min_length': 144.0, 'completions/max_length': 1635.0, 'completions/clipped_ratio': 0.734375, 'completions/mean_terminated_length': 481.4705810546875, 'completions/min_terminated_length': 144.0, 'completions/max_terminated_length': 980.0, 'rollout/action_tokens_mean': 861.203125, 'rollout/episode_turns_mean': 26.59375, 'rollout/truncated_ratio': 0.734375, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.21015626192092896, 'rewards/alfworld_rollout_reward_func/std': 0.4713321030139923, 'reward': 0.21015624701976776, 'reward_std': 0.3197978734970093, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02258184552192688, 'sampling/sampling_logp_difference/max': 4.545639991760254, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4099380373954773, 'sampling/importance_sampling_ratio/max': 2.697384834289551, 'kl': 0.0018834842448995914, 'entropy': 0.36975698871538043, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 397.8871677600073, 'epoch': 0.39}

 77%|███████▋  | 49/64 [5:15:09<1:35:25, 381.72s/it]
 78%|███████▊  | 50/64 [5:21:18<1:28:11, 377.96s/it]
                                                    
{'loss': 0.0071, 'grad_norm': 0.013544962741434574, 'learning_rate': 6.880180319272006e-07, 'num_tokens': 2569676.0, 'completions/mean_length': 887.515625, 'completions/min_length': 240.0, 'completions/max_length': 1624.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 580.0625, 'completions/min_terminated_length': 240.0, 'completions/max_terminated_length': 910.0, 'rollout/action_tokens_mean': 887.515625, 'rollout/episode_turns_mean': 26.828125, 'rollout/truncated_ratio': 0.75, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.1901562511920929, 'rewards/alfworld_rollout_reward_func/std': 0.4570210576057434, 'reward': 0.1901562511920929, 'reward_std': 0.45852425694465637, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02207544818520546, 'sampling/sampling_logp_difference/max': 4.044807434082031, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5466522574424744, 'sampling/importance_sampling_ratio/max': 2.99810791015625, 'kl': 0.0015946346466080286, 'entropy': 0.3667739760130644, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 363.5887358939908, 'epoch': 0.4}

 78%|███████▊  | 50/64 [5:21:18<1:28:11, 377.96s/it]
 80%|███████▉  | 51/64 [5:27:46<1:22:33, 381.04s/it]
                                                    
{'loss': 0.0148, 'grad_norm': 0.01083005964756012, 'learning_rate': 6.031046932680229e-07, 'num_tokens': 2619156.0, 'completions/mean_length': 773.125, 'completions/min_length': 110.0, 'completions/max_length': 1596.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 392.4090881347656, 'completions/min_terminated_length': 110.0, 'completions/max_terminated_length': 1015.0, 'rollout/action_tokens_mean': 773.125, 'rollout/episode_turns_mean': 24.046875, 'rollout/truncated_ratio': 0.65625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.30000001192092896, 'rewards/alfworld_rollout_reward_func/std': 0.500076174736023, 'reward': 0.30000001192092896, 'reward_std': 0.43087929487228394, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022034121677279472, 'sampling/sampling_logp_difference/max': 1.6584203243255615, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4208452105522156, 'sampling/importance_sampling_ratio/max': 2.189059257507324, 'kl': 0.002072367888104054, 'entropy': 0.3592249136418104, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 381.66528784099137, 'epoch': 0.41}

 80%|███████▉  | 51/64 [5:27:46<1:22:33, 381.04s/it]Warning: BPE mismatch at turn 20 (expected prefix 9514, got 9915 tokens). Attempting delta recovery.

 81%|████████▏ | 52/64 [5:35:58<1:22:51, 414.28s/it]
                                                    
{'loss': -0.0079, 'grad_norm': 0.005050386302173138, 'learning_rate': 5.23060657655754e-07, 'num_tokens': 2669096.0, 'completions/mean_length': 780.3125, 'completions/min_length': 98.0, 'completions/max_length': 1738.0, 'completions/clipped_ratio': 0.59375, 'completions/mean_terminated_length': 445.5769348144531, 'completions/min_terminated_length': 98.0, 'completions/max_terminated_length': 1079.0, 'rollout/action_tokens_mean': 780.3125, 'rollout/episode_turns_mean': 23.90625, 'rollout/truncated_ratio': 0.59375, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.3726562559604645, 'rewards/alfworld_rollout_reward_func/std': 0.5065045356750488, 'reward': 0.3726562559604645, 'reward_std': 0.36754581332206726, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022033972665667534, 'sampling/sampling_logp_difference/max': 2.5774283409118652, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.4066748023033142, 'sampling/importance_sampling_ratio/max': 2.1458444595336914, 'kl': 0.0017878458802442765, 'entropy': 0.3415462737902999, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 482.9236067970087, 'epoch': 0.42}

 81%|████████▏ | 52/64 [5:35:58<1:22:51, 414.28s/it]
 83%|████████▎ | 53/64 [5:43:26<1:17:49, 424.48s/it]
                                                    
{'loss': -0.0017, 'grad_norm': 0.010500618256628513, 'learning_rate': 4.480913969818099e-07, 'num_tokens': 2721532.0, 'completions/mean_length': 819.3125, 'completions/min_length': 158.0, 'completions/max_length': 1387.0, 'completions/clipped_ratio': 0.734375, 'completions/mean_terminated_length': 579.8823852539062, 'completions/min_terminated_length': 158.0, 'completions/max_terminated_length': 1161.0, 'rollout/action_tokens_mean': 819.3125, 'rollout/episode_turns_mean': 27.015625, 'rollout/truncated_ratio': 0.734375, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.21718750894069672, 'rewards/alfworld_rollout_reward_func/std': 0.4590102434158325, 'reward': 0.21718750894069672, 'reward_std': 0.38691049814224243, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020567741245031357, 'sampling/sampling_logp_difference/max': 1.6585437059402466, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.38772356510162354, 'sampling/importance_sampling_ratio/max': 2.4449539184570312, 'kl': 0.0019351580704096705, 'entropy': 0.33714828826487064, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 440.7185651119944, 'epoch': 0.42}

 83%|████████▎ | 53/64 [5:43:26<1:17:49, 424.48s/it]
 84%|████████▍ | 54/64 [5:50:49<1:11:38, 429.81s/it]
                                                    
{'loss': -0.003, 'grad_norm': 0.00624613743275404, 'learning_rate': 3.7838935626312246e-07, 'num_tokens': 2776470.0, 'completions/mean_length': 858.40625, 'completions/min_length': 196.0, 'completions/max_length': 1538.0, 'completions/clipped_ratio': 0.765625, 'completions/mean_terminated_length': 563.4666748046875, 'completions/min_terminated_length': 196.0, 'completions/max_terminated_length': 1077.0, 'rollout/action_tokens_mean': 858.40625, 'rollout/episode_turns_mean': 26.546875, 'rollout/truncated_ratio': 0.765625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.18359375, 'rewards/alfworld_rollout_reward_func/std': 0.44527989625930786, 'reward': 0.18359375, 'reward_std': 0.32859885692596436, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022003933787345886, 'sampling/sampling_logp_difference/max': 1.6584253311157227, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.2892346978187561, 'sampling/importance_sampling_ratio/max': 2.0156307220458984, 'kl': 0.0019235094769101124, 'entropy': 0.3832448711618781, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 433.99726870600716, 'epoch': 0.43}

 84%|████████▍ | 54/64 [5:50:49<1:11:38, 429.81s/it]
 86%|████████▌ | 55/64 [5:57:05<1:02:03, 413.71s/it]
                                                    
{'loss': 0.0162, 'grad_norm': 0.019602373242378235, 'learning_rate': 3.141334596385448e-07, 'num_tokens': 2826022.0, 'completions/mean_length': 774.25, 'completions/min_length': 103.0, 'completions/max_length': 1862.0, 'completions/clipped_ratio': 0.640625, 'completions/mean_terminated_length': 319.9565124511719, 'completions/min_terminated_length': 103.0, 'completions/max_terminated_length': 1001.0, 'rollout/action_tokens_mean': 774.25, 'rollout/episode_turns_mean': 22.890625, 'rollout/truncated_ratio': 0.640625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.30687499046325684, 'rewards/alfworld_rollout_reward_func/std': 0.5158684253692627, 'reward': 0.30687499046325684, 'reward_std': 0.2975793182849884, 'frac_reward_zero_std': 0.125, 'sampling/sampling_logp_difference/mean': 0.021400175988674164, 'sampling/sampling_logp_difference/max': 1.6167523860931396, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5007343292236328, 'sampling/importance_sampling_ratio/max': 2.4803109169006348, 'kl': 0.0015943678226904012, 'entropy': 0.3484980883076787, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 371.2160142980065, 'epoch': 0.44}

 86%|████████▌ | 55/64 [5:57:05<1:02:03, 413.71s/it]
 88%|████████▊ | 56/64 [6:04:15<55:50, 418.84s/it]  
                                                  
{'loss': 0.0093, 'grad_norm': 0.01010950654745102, 'learning_rate': 2.5548865107314606e-07, 'num_tokens': 2876933.0, 'completions/mean_length': 795.484375, 'completions/min_length': 136.0, 'completions/max_length': 1620.0, 'completions/clipped_ratio': 0.734375, 'completions/mean_terminated_length': 376.058837890625, 'completions/min_terminated_length': 136.0, 'completions/max_terminated_length': 957.0, 'rollout/action_tokens_mean': 795.484375, 'rollout/episode_turns_mean': 25.328125, 'rollout/truncated_ratio': 0.734375, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.22109374403953552, 'rewards/alfworld_rollout_reward_func/std': 0.4678628444671631, 'reward': 0.22109374403953552, 'reward_std': 0.31529495120048523, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020041368901729584, 'sampling/sampling_logp_difference/max': 5.368804454803467, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5188072919845581, 'sampling/importance_sampling_ratio/max': 2.8859243392944336, 'kl': 0.0015955448616296053, 'entropy': 0.32310054916888475, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 424.94623591700656, 'epoch': 0.45}

 88%|████████▊ | 56/64 [6:04:15<55:50, 418.84s/it]
 89%|████████▉ | 57/64 [6:11:43<49:53, 427.57s/it]
                                                  
{'loss': -0.0097, 'grad_norm': 0.010526562109589577, 'learning_rate': 2.026054709494235e-07, 'num_tokens': 2929345.0, 'completions/mean_length': 818.9375, 'completions/min_length': 213.0, 'completions/max_length': 1384.0, 'completions/clipped_ratio': 0.71875, 'completions/mean_terminated_length': 486.0, 'completions/min_terminated_length': 213.0, 'completions/max_terminated_length': 1021.0, 'rollout/action_tokens_mean': 818.9375, 'rollout/episode_turns_mean': 25.953125, 'rollout/truncated_ratio': 0.71875, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.23046875, 'rewards/alfworld_rollout_reward_func/std': 0.475816547870636, 'reward': 0.23046875, 'reward_std': 0.4045883119106293, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022473536431789398, 'sampling/sampling_logp_difference/max': 1.8799049854278564, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5196441411972046, 'sampling/importance_sampling_ratio/max': 2.9720585346221924, 'kl': 0.0017983552861551289, 'entropy': 0.35508036240935326, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 439.1710239440072, 'epoch': 0.46}

 89%|████████▉ | 57/64 [6:11:43<49:53, 427.57s/it]
 91%|█████████ | 58/64 [6:17:49<40:54, 409.09s/it]
                                                  
{'loss': -0.0058, 'grad_norm': 0.00702791940420866, 'learning_rate': 1.5561966963229925e-07, 'num_tokens': 2976605.0, 'completions/mean_length': 738.4375, 'completions/min_length': 137.0, 'completions/max_length': 1419.0, 'completions/clipped_ratio': 0.5625, 'completions/mean_terminated_length': 387.2500305175781, 'completions/min_terminated_length': 137.0, 'completions/max_terminated_length': 765.0, 'rollout/action_tokens_mean': 738.4375, 'rollout/episode_turns_mean': 22.078125, 'rollout/truncated_ratio': 0.5625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.3982812464237213, 'rewards/alfworld_rollout_reward_func/std': 0.5275837779045105, 'reward': 0.3982812762260437, 'reward_std': 0.37868061661720276, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02082432433962822, 'sampling/sampling_logp_difference/max': 2.1239027976989746, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.42546379566192627, 'sampling/importance_sampling_ratio/max': 1.7666839361190796, 'kl': 0.0016746247110859258, 'entropy': 0.3470684285275638, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 360.1903216189894, 'epoch': 0.46}

 91%|█████████ | 58/64 [6:17:49<40:54, 409.09s/it]
 92%|█████████▏| 59/64 [6:23:20<32:07, 385.45s/it]
                                                  
{'loss': 0.0023, 'grad_norm': 0.016277866438031197, 'learning_rate': 1.1465185899987797e-07, 'num_tokens': 3018708.0, 'completions/mean_length': 657.859375, 'completions/min_length': 81.0, 'completions/max_length': 1488.0, 'completions/clipped_ratio': 0.4375, 'completions/mean_terminated_length': 380.02777099609375, 'completions/min_terminated_length': 81.0, 'completions/max_terminated_length': 1112.0, 'rollout/action_tokens_mean': 657.859375, 'rollout/episode_turns_mean': 19.984375, 'rollout/truncated_ratio': 0.4375, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.5278124809265137, 'rewards/alfworld_rollout_reward_func/std': 0.5249382853507996, 'reward': 0.5278124809265137, 'reward_std': 0.386292964220047, 'frac_reward_zero_std': 0.125, 'sampling/sampling_logp_difference/mean': 0.01990351639688015, 'sampling/sampling_logp_difference/max': 1.4242944717407227, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5997130274772644, 'sampling/importance_sampling_ratio/max': 2.904142379760742, 'kl': 0.0016064580941019813, 'entropy': 0.3138961121439934, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 324.923169317015, 'epoch': 0.47}

 92%|█████████▏| 59/64 [6:23:20<32:07, 385.45s/it]Warning: BPE mismatch at turn 2 (expected prefix 770, got 910 tokens). Attempting delta recovery.

 94%|█████████▍| 60/64 [6:29:43<25:38, 384.74s/it]
                                                  
{'loss': 0.0022, 'grad_norm': 0.01379719190299511, 'learning_rate': 7.980720283448957e-08, 'num_tokens': 3071557.0, 'completions/mean_length': 825.765625, 'completions/min_length': 132.0, 'completions/max_length': 1532.0, 'completions/clipped_ratio': 0.65625, 'completions/mean_terminated_length': 551.6818237304688, 'completions/min_terminated_length': 132.0, 'completions/max_terminated_length': 1118.0, 'rollout/action_tokens_mean': 825.765625, 'rollout/episode_turns_mean': 25.703125, 'rollout/truncated_ratio': 0.65625, 'rollout/mismatch_mean': 0.015625, 'rollout/mismatch_ratio': 0.015625, 'rewards/alfworld_rollout_reward_func/mean': 0.28171873092651367, 'rewards/alfworld_rollout_reward_func/std': 0.50815749168396, 'reward': 0.28171876072883606, 'reward_std': 0.49070847034454346, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02150684781372547, 'sampling/sampling_logp_difference/max': 4.668699741363525, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.47809475660324097, 'sampling/importance_sampling_ratio/max': 2.6620919704437256, 'kl': 0.001881336524093058, 'entropy': 0.3388841552659869, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 377.0457431760042, 'epoch': 0.48}

 94%|█████████▍| 60/64 [6:29:43<25:38, 384.74s/it]
 95%|█████████▌| 61/64 [6:36:27<19:32, 390.73s/it]
                                                  
{'loss': 0.0038, 'grad_norm': 0.005873848684132099, 'learning_rate': 5.117514686876379e-08, 'num_tokens': 3122278.0, 'completions/mean_length': 792.515625, 'completions/min_length': 165.0, 'completions/max_length': 1838.0, 'completions/clipped_ratio': 0.703125, 'completions/mean_terminated_length': 347.631591796875, 'completions/min_terminated_length': 165.0, 'completions/max_terminated_length': 806.0, 'rollout/action_tokens_mean': 792.515625, 'rollout/episode_turns_mean': 24.234375, 'rollout/truncated_ratio': 0.703125, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.2498437464237213, 'rewards/alfworld_rollout_reward_func/std': 0.4883873760700226, 'reward': 0.2498437464237213, 'reward_std': 0.3333018124103546, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02179795503616333, 'sampling/sampling_logp_difference/max': 4.1571760177612305, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.33059564232826233, 'sampling/importance_sampling_ratio/max': 1.6227197647094727, 'kl': 0.002195166100136703, 'entropy': 0.36354396492242813, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 399.4235413080096, 'epoch': 0.49}

 95%|█████████▌| 61/64 [6:36:27<19:32, 390.73s/it]
 97%|█████████▋| 62/64 [6:42:40<12:50, 385.19s/it]
                                                  
{'loss': 0.0094, 'grad_norm': 0.014743300154805183, 'learning_rate': 2.8829189179721552e-08, 'num_tokens': 3171695.0, 'completions/mean_length': 772.140625, 'completions/min_length': 127.0, 'completions/max_length': 1345.0, 'completions/clipped_ratio': 0.6875, 'completions/mean_terminated_length': 398.20001220703125, 'completions/min_terminated_length': 127.0, 'completions/max_terminated_length': 1183.0, 'rollout/action_tokens_mean': 772.140625, 'rollout/episode_turns_mean': 24.515625, 'rollout/truncated_ratio': 0.6875, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.2721875011920929, 'rewards/alfworld_rollout_reward_func/std': 0.4889914393424988, 'reward': 0.2721875011920929, 'reward_std': 0.40832602977752686, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.020675554871559143, 'sampling/sampling_logp_difference/max': 3.382229804992676, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5585420727729797, 'sampling/importance_sampling_ratio/max': 2.4185616970062256, 'kl': 0.0017252494653803296, 'entropy': 0.3363960413262248, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 366.09222524699, 'epoch': 0.5}

 97%|█████████▋| 62/64 [6:42:40<12:50, 385.19s/it]
 98%|█████████▊| 63/64 [6:49:08<06:26, 386.08s/it]
                                                  
{'loss': 0.0236, 'grad_norm': 0.009038163349032402, 'learning_rate': 1.2826691520262114e-08, 'num_tokens': 3223152.0, 'completions/mean_length': 804.015625, 'completions/min_length': 116.0, 'completions/max_length': 1560.0, 'completions/clipped_ratio': 0.640625, 'completions/mean_terminated_length': 414.3913269042969, 'completions/min_terminated_length': 116.0, 'completions/max_terminated_length': 1246.0, 'rollout/action_tokens_mean': 804.015625, 'rollout/episode_turns_mean': 23.90625, 'rollout/truncated_ratio': 0.640625, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.30265623331069946, 'rewards/alfworld_rollout_reward_func/std': 0.5124087333679199, 'reward': 0.30265626311302185, 'reward_std': 0.3771285116672516, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02032957598567009, 'sampling/sampling_logp_difference/max': 2.050037384033203, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.5557758808135986, 'sampling/importance_sampling_ratio/max': 2.994922161102295, 'kl': 0.0018702544839470647, 'entropy': 0.3338092272169888, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 381.51745763700455, 'epoch': 0.5}

 98%|█████████▊| 63/64 [6:49:08<06:26, 386.08s/it]
100%|██████████| 64/64 [6:55:42<00:00, 388.43s/it]
                                                  
{'loss': 0.0198, 'grad_norm': 0.011489775963127613, 'learning_rate': 3.208732072368104e-09, 'num_tokens': 3276816.0, 'completions/mean_length': 838.5, 'completions/min_length': 124.0, 'completions/max_length': 1427.0, 'completions/clipped_ratio': 0.75, 'completions/mean_terminated_length': 443.9375, 'completions/min_terminated_length': 124.0, 'completions/max_terminated_length': 964.0, 'rollout/action_tokens_mean': 838.5, 'rollout/episode_turns_mean': 26.109375, 'rollout/truncated_ratio': 0.75, 'rollout/mismatch_mean': 0.0, 'rollout/mismatch_ratio': 0.0, 'rewards/alfworld_rollout_reward_func/mean': 0.20734375715255737, 'rewards/alfworld_rollout_reward_func/std': 0.4573083817958832, 'reward': 0.20734374225139618, 'reward_std': 0.32793596386909485, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.02185032144188881, 'sampling/sampling_logp_difference/max': 1.943666696548462, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.423524409532547, 'sampling/importance_sampling_ratio/max': 2.7888708114624023, 'kl': 0.0017125457925430965, 'entropy': 0.3799225674010813, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 388.11920543999804, 'epoch': 0.51}

100%|██████████| 64/64 [6:55:42<00:00, 388.43s/it]
                                                  
{'train_runtime': 24946.4114, 'train_samples_per_second': 0.164, 'train_steps_per_second': 0.003, 'train_loss': 0.003138344107355806, 'epoch': 0.51}

100%|██████████| 64/64 [6:55:42<00:00, 388.43s/it]
100%|██████████| 64/64 [6:55:43<00:00, 389.75s/it]
[2026-01-22 00:01:11,431] [INFO] [axolotl.train.save_trained_model:233] [PID:251] Training completed! Saving trained model to /app/checkpoints/1/environment_test.
[2026-01-22 00:01:12,028] [INFO] [axolotl.train.save_trained_model:351] [PID:251] Model successfully saved to /app/checkpoints/1/environment_test
Initializing AlfWorld environment on rank 1 at http://environment-server-1:8000...
Environment initialized. ID: 0
Warning: BPE mismatch at turn 4 (expected prefix 1217, got 1374 tokens). Attempting delta recovery.
Warning: BPE mismatch at turn 26 (expected prefix 7688, got 7920 tokens). Attempting delta recovery.
Warning: BPE mismatch at turn 1 (expected prefix 700, got 897 tokens). Attempting delta recovery.
Warning: BPE mismatch at turn 2 (expected prefix 1276, got 1593 tokens). Attempting delta recovery.
Warning: BPE mismatch at turn 17 (expected prefix 5550, got 5790 tokens). Attempting delta recovery.
Warning: BPE mismatch at turn 14 (expected prefix 5476, got 5921 tokens). Attempting delta recovery.
Warning: BPE mismatch at turn 23 (expected prefix 6418, got 6646 tokens). Attempting delta recovery.
Warning: BPE mismatch at turn 26 (expected prefix 9335, got 9636 tokens). Attempting delta recovery.
Warning: BPE mismatch at turn 1 (expected prefix 670, got 861 tokens). Attempting delta recovery.