Instructions to use bimabk/environment_test with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries
PEFT
How to use bimabk/environment_test with PEFT:
```
Base model is not found.
```

How to use bimabk/environment_test with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="bimabk/environment_test")
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("bimabk/environment_test")
model = AutoModelForCausalLM.from_pretrained("bimabk/environment_test", device_map="auto")
messages = [
    {"role": "user", "content": "Who are you?"},
]
inputs = tokenizer.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=40)
print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use bimabk/environment_test with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "bimabk/environment_test"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/bimabk/environment_test

SGLang

How to use bimabk/environment_test with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "bimabk/environment_test" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "bimabk/environment_test" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use bimabk/environment_test with Docker Model Runner:
```
docker model run hf.co/bimabk/environment_test
```

bimabk commited on Mar 13

Commit

2910877

verified ·

1 Parent(s): f1dc933

Upload task output 1

Browse files

Files changed (6) hide show

README.md +1 -1
adapter_config.json +8 -10
adapter_model.safetensors +2 -2
loss.txt +1 -1
trainer_state.json +1794 -145
training_args.bin +2 -2

README.md CHANGED Viewed

@@ -206,4 +206,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 [More Information Needed]
 ### Framework versions
-- PEFT 0.18.2.dev0

 [More Information Needed]
 ### Framework versions
+- PEFT 0.18.1

adapter_config.json CHANGED Viewed

@@ -16,32 +16,30 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 256,
   "lora_bias": false,
   "lora_dropout": 0.05,
-  "lora_ga_config": null,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "peft_version": "0.18.2.dev0@1d08144b855d3a58442a4f17187c5a8ebcf3b237",
   "qalora_group_size": 16,
-  "r": 128,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "k_proj",
-    "down_proj",
     "v_proj",
     "up_proj",
-    "gate_proj",
-    "o_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,
-  "use_bdlora": null,
   "use_dora": false,
   "use_qalora": false,
   "use_rslora": false

   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
+  "lora_alpha": 64,
   "lora_bias": false,
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
+  "peft_version": "0.18.1",
   "qalora_group_size": 16,
+  "r": 32,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "o_proj",
     "v_proj",
+    "k_proj",
     "up_proj",
+    "down_proj",
+    "q_proj",
+    "gate_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,
   "use_dora": false,
   "use_qalora": false,
   "use_rslora": false

adapter_model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e162c64c34c562057fc3e8070abf86b086e58c5b39b382aad71e2ba9f36a793
-size 957942768

 version https://git-lfs.github.com/spec/v1
+oid sha256:aaa76d7b523dc6d384a6dd98ee0f30ffe7d9ec8f762da429b933d8e16e7594db
+size 239536272

loss.txt CHANGED Viewed

	@@ -1 +1 @@
1	- 4,~~-0.010877630673348904~~


1	+ 75,no_eval

trainer_state.json CHANGED Viewed

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 4,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -16,30 +16,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 52.15178680419922,
-      "completions/mean_terminated_length": 52.15178680419922,
-      "completions/min_length": 30.0,
-      "completions/min_terminated_length": 30.0,
-      "entropy": 0.5559625327587128,
-      "epoch": 0.25,
-      "frac_reward_zero_std": 0.0714285746216774,
-      "grad_norm": 0.3681248426437378,
       "learning_rate": 0.0,
-      "loss": -0.0119,
-      "num_tokens": 217107.0,
-      "reward": 0.3636607229709625,
-      "reward_std": 0.5160845518112183,
-      "rewards/env_alfworld_reward/mean": 0.3636607229709625,
-      "rewards/env_alfworld_reward/std": 0.516084611415863,
-      "sampling/importance_sampling_ratio/max": 2.9794723987579346,
-      "sampling/importance_sampling_ratio/mean": 0.9032093286514282,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 1.4227659702301025,
-      "sampling/sampling_logp_difference/mean": 0.031906239688396454,
       "step": 1,
-      "step_time": 1695.3378897999974
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -47,133 +48,1781 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 50.145835876464844,
-      "completions/mean_terminated_length": 50.145835876464844,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 0.5740033835172653,
-      "epoch": 0.5,
-      "frac_reward_zero_std": 0.0714285746216774,
-      "grad_norm": 0.44471269845962524,
-      "learning_rate": 2.84304e-07,
-      "loss": -0.019,
-      "num_tokens": 458068.0,
-      "reward": 0.2202083021402359,
-      "reward_std": 0.4740038514137268,
-      "rewards/env_alfworld_reward/mean": 0.2202083021402359,
-      "rewards/env_alfworld_reward/std": 0.4740039110183716,
-      "sampling/importance_sampling_ratio/max": 2.9810025691986084,
-      "sampling/importance_sampling_ratio/mean": 1.0009801387786865,
       "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 4.155694007873535,
-      "sampling/sampling_logp_difference/mean": 0.032881855964660645,
-      "step": 2,
-      "step_time": 1649.8384637760028
-    },
-    {
-      "clip_ratio/high_max": 0.007568976026959717,
-      "clip_ratio/high_mean": 0.006711035966873169,
-      "clip_ratio/low_mean": 0.003108485194388777,
-      "clip_ratio/low_min": 0.0022171867603901774,
-      "clip_ratio/region_mean": 0.009819521103054285,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 104.0,
-      "completions/max_terminated_length": 104.0,
-      "completions/mean_length": 50.770835876464844,
-      "completions/mean_terminated_length": 50.770835876464844,
-      "completions/min_length": 25.0,
-      "completions/min_terminated_length": 25.0,
-      "entropy": 0.6086474657058716,
-      "epoch": 0.75,
-      "frac_reward_zero_std": 0.095238097012043,
-      "grad_norm": 0.35934266448020935,
-      "learning_rate": 5.68608e-07,
-      "loss": -0.0351,
-      "num_tokens": 693527.0,
-      "reward": 0.33044642210006714,
-      "reward_std": 0.5141857266426086,
-      "rewards/env_alfworld_reward/mean": 0.33044642210006714,
-      "rewards/env_alfworld_reward/std": 0.5141857266426086,
-      "sampling/importance_sampling_ratio/max": 2.783240795135498,
-      "sampling/importance_sampling_ratio/mean": 0.9537404775619507,
       "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 1.1146756410598755,
-      "sampling/sampling_logp_difference/mean": 0.03519035875797272,
-      "step": 3,
-      "step_time": 1672.7712960310018
-    },
-    {
-      "clip_ratio/high_max": 0.007020744145847857,
-      "clip_ratio/high_mean": 0.006485109799541533,
-      "clip_ratio/low_mean": 0.004376257362309843,
-      "clip_ratio/low_min": 0.003922754956874996,
-      "clip_ratio/region_mean": 0.010861367220059037,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 51.60714340209961,
-      "completions/mean_terminated_length": 51.60714340209961,
-      "completions/min_length": 28.0,
-      "completions/min_terminated_length": 28.0,
-      "entropy": 0.5597293674945831,
-      "epoch": 1.0,
-      "frac_reward_zero_std": 0.0535714291036129,
-      "grad_norm": 0.409810870885849,
-      "learning_rate": 8.529119999999999e-07,
-      "loss": 0.0229,
-      "num_tokens": 942371.0,
-      "reward": 0.03812500089406967,
-      "reward_std": 0.29200857877731323,
-      "rewards/env_alfworld_reward/mean": 0.03812500089406967,
-      "rewards/env_alfworld_reward/std": 0.2920086085796356,
-      "sampling/importance_sampling_ratio/max": 2.8310158252716064,
-      "sampling/importance_sampling_ratio/mean": 0.9631035327911377,
       "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 1.3009870052337646,
-      "sampling/sampling_logp_difference/mean": 0.03364055976271629,
-      "step": 4,
-      "step_time": 1871.100103906003
-    },
-    {
-      "epoch": 1.0,
-      "eval_clip_ratio/high_max": 0.0,
-      "eval_clip_ratio/high_mean": 0.0,
-      "eval_clip_ratio/low_mean": 0.0,
-      "eval_clip_ratio/low_min": 0.0,
-      "eval_clip_ratio/region_mean": 0.0,
-      "eval_completions/clipped_ratio": 0.0,
-      "eval_completions/max_length": 68.96,
-      "eval_completions/max_terminated_length": 68.96,
-      "eval_completions/mean_length": 51.315,
-      "eval_completions/mean_terminated_length": 51.315,
-      "eval_completions/min_length": 37.8,
-      "eval_completions/min_terminated_length": 37.8,
-      "eval_entropy": 0.6129815554618836,
-      "eval_frac_reward_zero_std": 0.09,
-      "eval_loss": -0.010877630673348904,
-      "eval_num_tokens": 942371.0,
-      "eval_reward": 0.19167500026524067,
-      "eval_reward_std": 0.38457470636814833,
-      "eval_rewards/env_alfworld_reward/mean": 0.19167500026524067,
-      "eval_rewards/env_alfworld_reward/std": 0.38457471802830695,
-      "eval_runtime": 2099.7568,
-      "eval_samples_per_second": 0.095,
-      "eval_sampling/importance_sampling_ratio/max": 1.5984654092788697,
-      "eval_sampling/importance_sampling_ratio/mean": 0.8831864225864411,
-      "eval_sampling/importance_sampling_ratio/min": 0.3859718208014965,
-      "eval_sampling/sampling_logp_difference/max": 0.6316441667079925,
-      "eval_sampling/sampling_logp_difference/mean": 0.03440669998526573,
-      "eval_steps_per_second": 0.012,
-      "step": 4
     }
   ],
   "logging_steps": 1.0,
-  "max_steps": 8,
-  "num_input_tokens_seen": 942371,
-  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -188,7 +1837,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 42,
   "trial_name": null,
   "trial_params": null
 }

   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
+  "epoch": 0.00075,
   "eval_steps": 500,
+  "global_step": 75,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1178.0,
+      "completions/max_terminated_length": 1178.0,
+      "completions/mean_length": 280.09375,
+      "completions/mean_terminated_length": 280.09375,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.3659224039874971,
+      "epoch": 1e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8075896501541138,
+      "kl": 0.0,
       "learning_rate": 0.0,
+      "loss": -0.0378,
+      "num_tokens": 37691.0,
+      "reward": -1.0071403980255127,
+      "reward_std": 0.036310724914073944,
+      "rewards/rollout_reward_func/mean": -1.0071403980255127,
+      "rewards/rollout_reward_func/std": 0.0389554463326931,
+      "sampling/importance_sampling_ratio/max": 1.6058107614517212,
+      "sampling/importance_sampling_ratio/mean": 1.0605039596557617,
+      "sampling/importance_sampling_ratio/min": 0.6887973546981812,
+      "sampling/sampling_logp_difference/max": 0.4979521632194519,
+      "sampling/sampling_logp_difference/mean": 0.04375916346907616,
       "step": 1,
+      "step_time": 11.142912297999601
     },
     {
       "clip_ratio/high_max": 0.0,
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
+      "entropy": 0.3659224039874971,
+      "epoch": 2e-05,
+      "grad_norm": 0.8133119344711304,
+      "kl": 0.0,
+      "learning_rate": 2.8571428571428573e-06,
+      "loss": -0.0378,
+      "step": 2,
+      "step_time": 6.929950346999931
+    },
+    {
+      "clip_ratio/high_max": 0.05902777845039964,
+      "clip_ratio/high_mean": 0.02951388922519982,
+      "clip_ratio/low_mean": 0.031250000931322575,
+      "clip_ratio/low_min": 0.02083333395421505,
+      "clip_ratio/region_mean": 0.06076389015652239,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1232.0,
+      "completions/max_terminated_length": 1232.0,
+      "completions/mean_length": 310.90625,
+      "completions/mean_terminated_length": 310.90625,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.4082131152972579,
+      "epoch": 3e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.716623067855835,
+      "kl": 0.0072153940433281605,
+      "learning_rate": 5.7142857142857145e-06,
+      "loss": -0.0034,
+      "num_tokens": 79271.0,
+      "reward": -0.9292834997177124,
+      "reward_std": 0.20812131464481354,
+      "rewards/rollout_reward_func/mean": -0.9292834997177124,
+      "rewards/rollout_reward_func/std": 0.3781917691230774,
+      "sampling/importance_sampling_ratio/max": 1.392927646636963,
+      "sampling/importance_sampling_ratio/mean": 1.0064890384674072,
+      "sampling/importance_sampling_ratio/min": 0.6378100514411926,
+      "sampling/sampling_logp_difference/max": 0.53961181640625,
+      "sampling/sampling_logp_difference/mean": 0.039233192801475525,
+      "step": 3,
+      "step_time": 10.315485310000213
+    },
+    {
+      "clip_ratio/high_max": 0.0069444444961845875,
+      "clip_ratio/high_mean": 0.0034722222480922937,
+      "clip_ratio/low_mean": 0.027083334047347307,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0305555562954396,
+      "entropy": 0.40274658892303705,
+      "epoch": 4e-05,
+      "grad_norm": 0.5002186298370361,
+      "kl": 0.008539682462242126,
+      "learning_rate": 8.571428571428573e-06,
+      "loss": -0.0029,
+      "step": 4,
+      "step_time": 6.540185272000144
+    },
+    {
+      "clip_ratio/high_max": 0.012500000186264515,
+      "clip_ratio/high_mean": 0.0062500000931322575,
+      "clip_ratio/low_mean": 0.015625000465661287,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.021875000558793545,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1022.0,
+      "completions/max_terminated_length": 1022.0,
+      "completions/mean_length": 225.15625,
+      "completions/mean_terminated_length": 225.15625,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.2807091409340501,
+      "epoch": 5e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7028033137321472,
+      "kl": 0.047651219653289445,
+      "learning_rate": 1.1428571428571429e-05,
+      "loss": -0.017,
+      "num_tokens": 115228.0,
+      "reward": -0.8011959791183472,
+      "reward_std": 0.4463964104652405,
+      "rewards/rollout_reward_func/mean": -0.8011959791183472,
+      "rewards/rollout_reward_func/std": 0.6310392618179321,
+      "sampling/importance_sampling_ratio/max": 1.9444010257720947,
+      "sampling/importance_sampling_ratio/mean": 0.9612840414047241,
+      "sampling/importance_sampling_ratio/min": 0.2667683959007263,
+      "sampling/sampling_logp_difference/max": 0.8543260097503662,
+      "sampling/sampling_logp_difference/mean": 0.042739734053611755,
+      "step": 5,
+      "step_time": 9.484377300000233
+    },
+    {
+      "clip_ratio/high_max": 0.018750000279396772,
+      "clip_ratio/high_mean": 0.009375000139698386,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009375000139698386,
+      "entropy": 0.28512229211628437,
+      "epoch": 6e-05,
+      "grad_norm": 0.8833194971084595,
+      "kl": 0.011992669685696455,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": -0.0202,
+      "step": 6,
+      "step_time": 6.462289121999902
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.031250000931322575,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.031250000931322575,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1142.0,
+      "completions/max_terminated_length": 1142.0,
+      "completions/mean_length": 308.25,
+      "completions/mean_terminated_length": 308.25,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.3624303378164768,
+      "epoch": 7e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.885384738445282,
+      "kl": 0.005326485250407131,
+      "learning_rate": 1.7142857142857145e-05,
+      "loss": -0.008,
+      "num_tokens": 155454.0,
+      "reward": -0.8657017350196838,
+      "reward_std": 0.3935210704803467,
+      "rewards/rollout_reward_func/mean": -0.8657017350196838,
+      "rewards/rollout_reward_func/std": 0.5249258875846863,
+      "sampling/importance_sampling_ratio/max": 1.14151930809021,
+      "sampling/importance_sampling_ratio/mean": 0.9441938400268555,
+      "sampling/importance_sampling_ratio/min": 0.6393771767616272,
+      "sampling/sampling_logp_difference/max": 0.4876275062561035,
+      "sampling/sampling_logp_difference/mean": 0.037204425781965256,
+      "step": 7,
+      "step_time": 10.03836135199981
+    },
+    {
+      "clip_ratio/high_max": 0.013194444589316845,
+      "clip_ratio/high_mean": 0.0065972222946584225,
+      "clip_ratio/low_mean": 0.036805556854233146,
+      "clip_ratio/low_min": 0.012500000186264515,
+      "clip_ratio/region_mean": 0.04340277914889157,
+      "entropy": 0.3480250327847898,
+      "epoch": 8e-05,
+      "grad_norm": 0.6112654805183411,
+      "kl": 0.007768443218083121,
+      "learning_rate": 2e-05,
+      "loss": -0.0087,
+      "step": 8,
+      "step_time": 6.732798993000642
+    },
+    {
+      "clip_ratio/high_max": 0.005434782709926367,
+      "clip_ratio/high_mean": 0.0027173913549631834,
+      "clip_ratio/low_mean": 0.0062500000931322575,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00896739144809544,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1163.0,
+      "completions/max_terminated_length": 1163.0,
+      "completions/mean_length": 258.75,
+      "completions/mean_terminated_length": 258.75,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.27044630981981754,
+      "epoch": 9e-05,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7710390090942383,
+      "kl": 0.02225630160091896,
+      "learning_rate": 2.2857142857142858e-05,
+      "loss": -0.01,
+      "num_tokens": 193934.0,
+      "reward": -0.9323001503944397,
+      "reward_std": 0.20817707479000092,
+      "rewards/rollout_reward_func/mean": -0.9323001503944397,
+      "rewards/rollout_reward_func/std": 0.37744954228401184,
+      "sampling/importance_sampling_ratio/max": 1.3011233806610107,
+      "sampling/importance_sampling_ratio/mean": 0.9557175636291504,
+      "sampling/importance_sampling_ratio/min": 0.5675007700920105,
+      "sampling/sampling_logp_difference/max": 0.6186580657958984,
+      "sampling/sampling_logp_difference/mean": 0.04273971542716026,
+      "step": 9,
+      "step_time": 10.003510364000476
+    },
+    {
+      "clip_ratio/high_max": 0.014363354537636042,
+      "clip_ratio/high_mean": 0.007181677268818021,
+      "clip_ratio/low_mean": 0.06041666818782687,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.06759834592230618,
+      "entropy": 0.24492651503533125,
+      "epoch": 0.0001,
+      "grad_norm": 0.3038092851638794,
+      "kl": 0.061313093479839154,
+      "learning_rate": 2.5714285714285714e-05,
+      "loss": -0.0121,
+      "step": 10,
+      "step_time": 6.334576341999991
+    },
+    {
+      "clip_ratio/high_max": 0.010416666977107525,
+      "clip_ratio/high_mean": 0.0052083334885537624,
+      "clip_ratio/low_mean": 0.01666666753590107,
+      "clip_ratio/low_min": 0.012500000186264515,
+      "clip_ratio/region_mean": 0.021875001024454832,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1808.0,
+      "completions/max_terminated_length": 1808.0,
+      "completions/mean_length": 366.3125,
+      "completions/mean_terminated_length": 366.3125,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.32991205714643,
+      "epoch": 0.00011,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.708510160446167,
+      "kl": 0.1540648011941812,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": -0.0489,
+      "num_tokens": 236260.0,
+      "reward": -0.9286847114562988,
+      "reward_std": 0.211622416973114,
+      "rewards/rollout_reward_func/mean": -0.9286847114562988,
+      "rewards/rollout_reward_func/std": 0.3866735100746155,
+      "sampling/importance_sampling_ratio/max": 1.9185539484024048,
+      "sampling/importance_sampling_ratio/mean": 0.9692364931106567,
       "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.5063650608062744,
+      "sampling/sampling_logp_difference/mean": 0.07806695252656937,
+      "step": 11,
+      "step_time": 12.247949407000306
+    },
+    {
+      "clip_ratio/high_max": 0.05850694561377168,
+      "clip_ratio/high_mean": 0.032725695287808776,
+      "clip_ratio/low_mean": 0.06458333553746343,
+      "clip_ratio/low_min": 0.033333334140479565,
+      "clip_ratio/region_mean": 0.0973090308252722,
+      "entropy": 0.3409234285354614,
+      "epoch": 0.00012,
+      "grad_norm": 0.527890682220459,
+      "kl": 0.5000243990944,
+      "learning_rate": 3.142857142857143e-05,
+      "loss": -0.0513,
+      "step": 12,
+      "step_time": 7.004255187000126
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.018978387233801186,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.018978387233801186,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1584.0,
+      "completions/max_terminated_length": 1584.0,
+      "completions/mean_length": 398.375,
+      "completions/mean_terminated_length": 398.375,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.28245622059330344,
+      "epoch": 0.00013,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.366127371788025,
+      "kl": 0.26600568945286795,
+      "learning_rate": 3.428571428571429e-05,
+      "loss": 0.0152,
+      "num_tokens": 278805.0,
+      "reward": -0.6331639289855957,
+      "reward_std": 0.7617669105529785,
+      "rewards/rollout_reward_func/mean": -0.6331639289855957,
+      "rewards/rollout_reward_func/std": 0.7844160199165344,
+      "sampling/importance_sampling_ratio/max": 1.327277660369873,
+      "sampling/importance_sampling_ratio/mean": 0.8719787001609802,
       "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2040467262268066,
+      "sampling/sampling_logp_difference/mean": 0.07095030695199966,
+      "step": 13,
+      "step_time": 12.302737500999683
+    },
+    {
+      "clip_ratio/high_max": 0.0416666679084301,
+      "clip_ratio/high_mean": 0.02083333395421505,
+      "clip_ratio/low_mean": 0.13363321172073483,
+      "clip_ratio/low_min": 0.04375000111758709,
+      "clip_ratio/region_mean": 0.15446654567494988,
+      "entropy": 0.2645200351253152,
+      "epoch": 0.00014,
+      "grad_norm": 0.6336849927902222,
+      "kl": 1.7774635917339765,
+      "learning_rate": 3.7142857142857143e-05,
+      "loss": 0.0159,
+      "step": 14,
+      "step_time": 6.674852292999958
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.05131448491010815,
+      "clip_ratio/low_min": 0.010416666977107525,
+      "clip_ratio/region_mean": 0.05131448491010815,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2135.0,
+      "completions/max_terminated_length": 2135.0,
+      "completions/mean_length": 362.1875,
+      "completions/mean_terminated_length": 362.1875,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.33170084515586495,
+      "epoch": 0.00015,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5385106801986694,
+      "kl": 0.5612458096584305,
+      "learning_rate": 4e-05,
+      "loss": 0.0529,
+      "num_tokens": 321363.0,
+      "reward": -0.22063349187374115,
+      "reward_std": 1.032537579536438,
+      "rewards/rollout_reward_func/mean": -0.22063349187374115,
+      "rewards/rollout_reward_func/std": 1.0019752979278564,
+      "sampling/importance_sampling_ratio/max": 2.0951614379882812,
+      "sampling/importance_sampling_ratio/mean": 1.0781480073928833,
+      "sampling/importance_sampling_ratio/min": 0.40334585309028625,
+      "sampling/sampling_logp_difference/max": 0.7475135326385498,
+      "sampling/sampling_logp_difference/mean": 0.05335211753845215,
+      "step": 15,
+      "step_time": 13.901658290000341
+    },
+    {
+      "clip_ratio/high_max": 0.04583333432674408,
+      "clip_ratio/high_mean": 0.02291666716337204,
+      "clip_ratio/low_mean": 0.07991071639116853,
+      "clip_ratio/low_min": 0.02083333395421505,
+      "clip_ratio/region_mean": 0.10282738448586315,
+      "entropy": 0.33263772330246866,
+      "epoch": 0.00016,
+      "grad_norm": 0.3616006076335907,
+      "kl": 0.5915351018775254,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 0.0492,
+      "step": 16,
+      "step_time": 7.827889540000115
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1066.0,
+      "completions/max_terminated_length": 1066.0,
+      "completions/mean_length": 146.75,
+      "completions/mean_terminated_length": 146.75,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.21940526392427273,
+      "epoch": 0.00017,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7156424522399902,
+      "kl": 0.9828777723014355,
+      "learning_rate": 4.5714285714285716e-05,
+      "loss": 0.0274,
+      "num_tokens": 354064.0,
+      "reward": 0.18792903423309326,
+      "reward_std": 1.0650506019592285,
+      "rewards/rollout_reward_func/mean": 0.18792903423309326,
+      "rewards/rollout_reward_func/std": 1.025183081626892,
+      "sampling/importance_sampling_ratio/max": 1.465952754020691,
+      "sampling/importance_sampling_ratio/mean": 0.9368042945861816,
+      "sampling/importance_sampling_ratio/min": 0.2505682706832886,
+      "sampling/sampling_logp_difference/max": 1.1606903076171875,
+      "sampling/sampling_logp_difference/mean": 0.06353487074375153,
+      "step": 17,
+      "step_time": 10.211389272000588
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.11508049350231886,
+      "clip_ratio/low_min": 0.033333334140479565,
+      "clip_ratio/region_mean": 0.11508049350231886,
+      "entropy": 0.19974320210167207,
+      "epoch": 0.00018,
+      "grad_norm": 2.1809680461883545,
+      "kl": 2.973201504908502,
+      "learning_rate": 4.8571428571428576e-05,
+      "loss": 0.0255,
+      "step": 18,
+      "step_time": 5.537876568999991
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.02291666716337204,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.02291666716337204,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1190.0,
+      "completions/max_terminated_length": 1190.0,
+      "completions/mean_length": 183.71875,
+      "completions/mean_terminated_length": 183.71875,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.2391725802444853,
+      "epoch": 0.00019,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5152996182441711,
+      "kl": 0.628770228009671,
+      "learning_rate": 5.142857142857143e-05,
+      "loss": 0.0056,
+      "num_tokens": 389262.0,
+      "reward": 0.09779167175292969,
+      "reward_std": 0.9968031048774719,
+      "rewards/rollout_reward_func/mean": 0.09779167175292969,
+      "rewards/rollout_reward_func/std": 1.0246973037719727,
+      "sampling/importance_sampling_ratio/max": 1.6519438028335571,
+      "sampling/importance_sampling_ratio/mean": 0.9000259637832642,
+      "sampling/importance_sampling_ratio/min": 0.026913145557045937,
+      "sampling/sampling_logp_difference/max": 3.1608994007110596,
+      "sampling/sampling_logp_difference/mean": 0.10347369313240051,
+      "step": 19,
+      "step_time": 11.03900628299948
+    },
+    {
+      "clip_ratio/high_max": 0.012500000186264515,
+      "clip_ratio/high_mean": 0.0062500000931322575,
+      "clip_ratio/low_mean": 0.14107143180444837,
+      "clip_ratio/low_min": 0.051190477795898914,
+      "clip_ratio/region_mean": 0.14732143189758062,
+      "entropy": 0.25456089232466184,
+      "epoch": 0.0002,
+      "grad_norm": 0.3765561580657959,
+      "kl": 1.1224204124882817,
+      "learning_rate": 5.428571428571428e-05,
+      "loss": 0.0047,
+      "step": 20,
+      "step_time": 6.163490915999773
+    },
+    {
+      "clip_ratio/high_max": 0.010416666977107525,
+      "clip_ratio/high_mean": 0.0052083334885537624,
+      "clip_ratio/low_mean": 0.007812500232830644,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.013020833721384406,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2228.0,
+      "completions/max_terminated_length": 2228.0,
+      "completions/mean_length": 374.4375,
+      "completions/mean_terminated_length": 374.4375,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.20231306218192913,
+      "epoch": 0.00021,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5450725555419922,
+      "kl": 1.0656099561601877,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": -0.0062,
+      "num_tokens": 432157.0,
+      "reward": 0.3385208547115326,
+      "reward_std": 0.9284825921058655,
+      "rewards/rollout_reward_func/mean": 0.3385208547115326,
+      "rewards/rollout_reward_func/std": 0.9307616949081421,
+      "sampling/importance_sampling_ratio/max": 1.735904335975647,
+      "sampling/importance_sampling_ratio/mean": 0.8049801588058472,
       "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.504056453704834,
+      "sampling/sampling_logp_difference/mean": 0.09936025738716125,
+      "step": 21,
+      "step_time": 14.117857149999963
+    },
+    {
+      "clip_ratio/high_max": 0.017361111473292112,
+      "clip_ratio/high_mean": 0.011805556016042829,
+      "clip_ratio/low_mean": 0.07820799155160785,
+      "clip_ratio/low_min": 0.027472528629004955,
+      "clip_ratio/region_mean": 0.09001354733482003,
+      "entropy": 0.20029474921466317,
+      "epoch": 0.00022,
+      "grad_norm": 0.38578858971595764,
+      "kl": 1.4086124375462532,
+      "learning_rate": 6e-05,
+      "loss": -0.008,
+      "step": 22,
+      "step_time": 7.929054158000099
+    },
+    {
+      "clip_ratio/high_max": 0.02500000037252903,
+      "clip_ratio/high_mean": 0.012500000186264515,
+      "clip_ratio/low_mean": 0.010156250093132257,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.022656250279396772,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 896.0,
+      "completions/max_terminated_length": 896.0,
+      "completions/mean_length": 165.5,
+      "completions/mean_terminated_length": 165.5,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.12490838440135121,
+      "epoch": 0.00023,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5326600074768066,
+      "kl": 2.6836317628622055,
+      "learning_rate": 6.285714285714286e-05,
+      "loss": -0.0076,
+      "num_tokens": 466199.0,
+      "reward": 0.6161667108535767,
+      "reward_std": 0.6345345377922058,
+      "rewards/rollout_reward_func/mean": 0.6161667108535767,
+      "rewards/rollout_reward_func/std": 0.7305893301963806,
+      "sampling/importance_sampling_ratio/max": 1.6318753957748413,
+      "sampling/importance_sampling_ratio/mean": 0.8681706190109253,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.4805762767791748,
+      "sampling/sampling_logp_difference/mean": 0.07480701804161072,
+      "step": 23,
+      "step_time": 9.858622221000587
+    },
+    {
+      "clip_ratio/high_max": 0.026785715483129025,
+      "clip_ratio/high_mean": 0.013392857741564512,
+      "clip_ratio/low_mean": 0.01927083358168602,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.03266369132325053,
+      "entropy": 0.10265930875902995,
+      "epoch": 0.00024,
+      "grad_norm": 2.3974478244781494,
+      "kl": 3.5795856304466724,
+      "learning_rate": 6.571428571428571e-05,
+      "loss": -0.0019,
+      "step": 24,
+      "step_time": 5.317549161999295
+    },
+    {
+      "clip_ratio/high_max": 0.02500000037252903,
+      "clip_ratio/high_mean": 0.012500000186264515,
+      "clip_ratio/low_mean": 0.016964286100119352,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.029464286286383867,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1176.0,
+      "completions/max_terminated_length": 1176.0,
+      "completions/mean_length": 240.1875,
+      "completions/mean_terminated_length": 240.1875,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.137992134608794,
+      "epoch": 0.00025,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6068210601806641,
+      "kl": 1.5051679126918316,
+      "learning_rate": 6.857142857142858e-05,
+      "loss": -0.002,
+      "num_tokens": 503259.0,
+      "reward": 0.6975833177566528,
+      "reward_std": 0.7012296319007874,
+      "rewards/rollout_reward_func/mean": 0.6975833177566528,
+      "rewards/rollout_reward_func/std": 0.7207034826278687,
+      "sampling/importance_sampling_ratio/max": 1.254854679107666,
+      "sampling/importance_sampling_ratio/mean": 0.838233470916748,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 2.63065767288208,
+      "sampling/sampling_logp_difference/mean": 0.094658762216568,
+      "step": 25,
+      "step_time": 10.299735158999283
+    },
+    {
+      "clip_ratio/high_max": 0.04642857238650322,
+      "clip_ratio/high_mean": 0.029464286286383867,
+      "clip_ratio/low_mean": 0.041778274811804295,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.07124256109818816,
+      "entropy": 0.16137297594104894,
+      "epoch": 0.00026,
+      "grad_norm": 5.5549468994140625,
+      "kl": 11.136886082589626,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 0.0342,
+      "step": 26,
+      "step_time": 5.75506067900028
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 734.0,
+      "completions/max_terminated_length": 734.0,
+      "completions/mean_length": 146.28125,
+      "completions/mean_terminated_length": 146.28125,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.15740215915138833,
+      "epoch": 0.00027,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7164216041564941,
+      "kl": 1.4110768623650074,
+      "learning_rate": 7.428571428571429e-05,
+      "loss": -0.0054,
+      "num_tokens": 536415.0,
+      "reward": 0.613898515701294,
+      "reward_std": 0.7614923715591431,
+      "rewards/rollout_reward_func/mean": 0.613898515701294,
+      "rewards/rollout_reward_func/std": 0.7739964723587036,
+      "sampling/importance_sampling_ratio/max": 1.3575035333633423,
+      "sampling/importance_sampling_ratio/mean": 0.9464373588562012,
+      "sampling/importance_sampling_ratio/min": 0.18349826335906982,
+      "sampling/sampling_logp_difference/max": 1.3662632703781128,
+      "sampling/sampling_logp_difference/mean": 0.043687544763088226,
+      "step": 27,
+      "step_time": 9.54420504699965
+    },
+    {
+      "clip_ratio/high_max": 0.0062500000931322575,
+      "clip_ratio/high_mean": 0.0031250000465661287,
+      "clip_ratio/low_mean": 0.05654762079939246,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.05967262154445052,
+      "entropy": 0.14962338455370627,
+      "epoch": 0.00028,
+      "grad_norm": 0.2844441533088684,
+      "kl": 1.628791993483901,
+      "learning_rate": 7.714285714285715e-05,
+      "loss": -0.0084,
+      "step": 28,
+      "step_time": 5.213027034000561
+    },
+    {
+      "clip_ratio/high_max": 0.03281250037252903,
+      "clip_ratio/high_mean": 0.016406250186264515,
+      "clip_ratio/low_mean": 0.001953125,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.018359375186264515,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1858.0,
+      "completions/max_terminated_length": 1858.0,
+      "completions/mean_length": 245.40625,
+      "completions/mean_terminated_length": 245.40625,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.1823743417917285,
+      "epoch": 0.00029,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5472207069396973,
+      "kl": 1.2950569400563836,
+      "learning_rate": 8e-05,
+      "loss": 0.0192,
+      "num_tokens": 572908.0,
+      "reward": 0.7323958873748779,
+      "reward_std": 0.75600266456604,
+      "rewards/rollout_reward_func/mean": 0.7323958873748779,
+      "rewards/rollout_reward_func/std": 0.7951676249504089,
+      "sampling/importance_sampling_ratio/max": 1.7107934951782227,
+      "sampling/importance_sampling_ratio/mean": 1.0277063846588135,
+      "sampling/importance_sampling_ratio/min": 0.6659436225891113,
+      "sampling/sampling_logp_difference/max": 0.7972254753112793,
+      "sampling/sampling_logp_difference/mean": 0.0571967251598835,
+      "step": 29,
+      "step_time": 12.326968075999275
+    },
+    {
+      "clip_ratio/high_max": 0.07599431928247213,
+      "clip_ratio/high_mean": 0.03799715964123607,
+      "clip_ratio/low_mean": 0.029296875,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0672940332442522,
+      "entropy": 0.18252644933818374,
+      "epoch": 0.0003,
+      "grad_norm": 0.42486661672592163,
+      "kl": 1.2426311783492565,
+      "learning_rate": 8.285714285714287e-05,
+      "loss": 0.0204,
+      "step": 30,
+      "step_time": 7.674688885000478
+    },
+    {
+      "clip_ratio/high_max": 0.02083333395421505,
+      "clip_ratio/high_mean": 0.010416666977107525,
+      "clip_ratio/low_mean": 0.010156250093132257,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.020572917070239782,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1072.0,
+      "completions/max_terminated_length": 1072.0,
+      "completions/mean_length": 203.8125,
+      "completions/mean_terminated_length": 203.8125,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.18441346354666166,
+      "epoch": 0.00031,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.8369215130805969,
+      "kl": 2.210440620779991,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 0.0219,
+      "num_tokens": 609478.0,
+      "reward": 0.6868749856948853,
+      "reward_std": 0.723414421081543,
+      "rewards/rollout_reward_func/mean": 0.6868749856948853,
+      "rewards/rollout_reward_func/std": 0.7201352715492249,
+      "sampling/importance_sampling_ratio/max": 2.42289662361145,
+      "sampling/importance_sampling_ratio/mean": 1.0437120199203491,
+      "sampling/importance_sampling_ratio/min": 0.0,
+      "sampling/sampling_logp_difference/max": 1.2931618690490723,
+      "sampling/sampling_logp_difference/mean": 0.06052036210894585,
+      "step": 31,
+      "step_time": 9.324151609999944
+    },
+    {
+      "clip_ratio/high_max": 0.0476190485060215,
+      "clip_ratio/high_mean": 0.026650432962924242,
+      "clip_ratio/low_mean": 0.03255208348855376,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.059202516451478004,
+      "entropy": 0.18722298718057573,
+      "epoch": 0.00032,
+      "grad_norm": 0.1640804558992386,
+      "kl": 2.2689803540706635,
+      "learning_rate": 8.857142857142857e-05,
+      "loss": 0.0193,
+      "step": 32,
+      "step_time": 6.513984409999921
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0052083334885537624,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0052083334885537624,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 941.0,
+      "completions/max_terminated_length": 941.0,
+      "completions/mean_length": 260.71875,
+      "completions/mean_terminated_length": 260.71875,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.3189265597611666,
+      "epoch": 0.00033,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0061767101287842,
+      "kl": 1.4462116956710815,
+      "learning_rate": 9.142857142857143e-05,
+      "loss": 0.0183,
+      "num_tokens": 648894.0,
+      "reward": 1.0101041793823242,
+      "reward_std": 0.3137112855911255,
+      "rewards/rollout_reward_func/mean": 1.0101041793823242,
+      "rewards/rollout_reward_func/std": 0.4550347924232483,
+      "sampling/importance_sampling_ratio/max": 2.6488871574401855,
+      "sampling/importance_sampling_ratio/mean": 1.0141404867172241,
+      "sampling/importance_sampling_ratio/min": 0.8440296649932861,
+      "sampling/sampling_logp_difference/max": 0.6730160713195801,
+      "sampling/sampling_logp_difference/mean": 0.037406109273433685,
+      "step": 33,
+      "step_time": 9.245252041999493
+    },
+    {
+      "clip_ratio/high_max": 0.03219697065651417,
+      "clip_ratio/high_mean": 0.016098485328257084,
+      "clip_ratio/low_mean": 0.03906250186264515,
+      "clip_ratio/low_min": 0.015625,
+      "clip_ratio/region_mean": 0.05516098719090223,
+      "entropy": 0.2540598356135888,
+      "epoch": 0.00034,
+      "grad_norm": 3.8695592880249023,
+      "kl": 5.638480260968208,
+      "learning_rate": 9.428571428571429e-05,
+      "loss": 0.0256,
+      "step": 34,
+      "step_time": 6.270834967000383
+    },
+    {
+      "clip_ratio/high_max": 0.008928571827709675,
+      "clip_ratio/high_mean": 0.004464285913854837,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004464285913854837,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 996.0,
+      "completions/max_terminated_length": 996.0,
+      "completions/mean_length": 313.28125,
+      "completions/mean_terminated_length": 313.28125,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.1834048866876401,
+      "epoch": 0.00035,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.25010761618614197,
+      "kl": 2.1367476396262646,
+      "learning_rate": 9.714285714285715e-05,
+      "loss": 0.0022,
+      "num_tokens": 688634.0,
+      "reward": 0.755062460899353,
+      "reward_std": 0.5053232312202454,
+      "rewards/rollout_reward_func/mean": 0.755062460899353,
+      "rewards/rollout_reward_func/std": 0.5986501574516296,
+      "sampling/importance_sampling_ratio/max": 1.6370441913604736,
+      "sampling/importance_sampling_ratio/mean": 0.9969434142112732,
+      "sampling/importance_sampling_ratio/min": 0.5411856174468994,
+      "sampling/sampling_logp_difference/max": 0.6061768531799316,
+      "sampling/sampling_logp_difference/mean": 0.024847794324159622,
+      "step": 35,
+      "step_time": 9.242927256999792
+    },
+    {
+      "clip_ratio/high_max": 0.034598215483129025,
+      "clip_ratio/high_mean": 0.017299107741564512,
+      "clip_ratio/low_mean": 0.00937500037252903,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.026674108114093542,
+      "entropy": 0.1706730479199905,
+      "epoch": 0.00036,
+      "grad_norm": 0.17176270484924316,
+      "kl": 2.141612984240055,
+      "learning_rate": 0.0001,
+      "loss": 0.0007,
+      "step": 36,
+      "step_time": 6.4294925179997335
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1151.0,
+      "completions/max_terminated_length": 1151.0,
+      "completions/mean_length": 213.9375,
+      "completions/mean_terminated_length": 213.9375,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.17484768042049836,
+      "epoch": 0.00037,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.26422277092933655,
+      "kl": 1.4826668202877045,
+      "learning_rate": 9.999736485702831e-05,
+      "loss": -0.005,
+      "num_tokens": 724933.0,
+      "reward": 0.7631042003631592,
+      "reward_std": 0.6272491216659546,
+      "rewards/rollout_reward_func/mean": 0.7631042003631592,
+      "rewards/rollout_reward_func/std": 0.6655160188674927,
+      "sampling/importance_sampling_ratio/max": 1.4438666105270386,
+      "sampling/importance_sampling_ratio/mean": 0.9936277270317078,
+      "sampling/importance_sampling_ratio/min": 0.41797348856925964,
+      "sampling/sampling_logp_difference/max": 0.7619318962097168,
+      "sampling/sampling_logp_difference/mean": 0.03127425163984299,
+      "step": 37,
+      "step_time": 9.695621070000243
+    },
+    {
+      "clip_ratio/high_max": 0.033928572200238705,
+      "clip_ratio/high_mean": 0.016964286100119352,
+      "clip_ratio/low_mean": 0.02187500079162419,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.03883928689174354,
+      "entropy": 0.15785920085181715,
+      "epoch": 0.00038,
+      "grad_norm": 0.16681881248950958,
+      "kl": 1.522850140929222,
+      "learning_rate": 9.998945979845876e-05,
+      "loss": -0.0062,
+      "step": 38,
+      "step_time": 6.750669727000513
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1154.0,
+      "completions/max_terminated_length": 1154.0,
+      "completions/mean_length": 231.53125,
+      "completions/mean_terminated_length": 231.53125,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.11877059687685687,
+      "epoch": 0.00039,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.24095091223716736,
+      "kl": 1.5396657940000296,
+      "learning_rate": 9.997628593527586e-05,
+      "loss": -0.0042,
+      "num_tokens": 762128.0,
+      "reward": 0.7163541316986084,
+      "reward_std": 0.6136023998260498,
+      "rewards/rollout_reward_func/mean": 0.7163541316986084,
+      "rewards/rollout_reward_func/std": 0.610213577747345,
+      "sampling/importance_sampling_ratio/max": 1.3014845848083496,
+      "sampling/importance_sampling_ratio/mean": 1.0244951248168945,
+      "sampling/importance_sampling_ratio/min": 0.889356791973114,
+      "sampling/sampling_logp_difference/max": 0.2697678804397583,
+      "sampling/sampling_logp_difference/mean": 0.01579492911696434,
+      "step": 39,
+      "step_time": 9.640163768999855
+    },
+    {
+      "clip_ratio/high_max": 0.01785714365541935,
+      "clip_ratio/high_mean": 0.008928571827709675,
+      "clip_ratio/low_mean": 0.018750000279396772,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.027678572107106447,
+      "entropy": 0.10194598157977453,
+      "epoch": 0.0004,
+      "grad_norm": 0.06580612063407898,
+      "kl": 1.5702023096382618,
+      "learning_rate": 9.995784511894694e-05,
+      "loss": -0.006,
+      "step": 40,
+      "step_time": 6.767393307999555
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.004464285913854837,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004464285913854837,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1234.0,
+      "completions/max_terminated_length": 1234.0,
+      "completions/mean_length": 262.125,
+      "completions/mean_terminated_length": 262.125,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.11627826229232596,
+      "epoch": 0.00041,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1414710283279419,
+      "kl": 1.8666857853531837,
+      "learning_rate": 9.993413994116206e-05,
+      "loss": -0.0112,
+      "num_tokens": 800926.0,
+      "reward": 0.6219791769981384,
+      "reward_std": 0.6369723677635193,
+      "rewards/rollout_reward_func/mean": 0.6219791769981384,
+      "rewards/rollout_reward_func/std": 0.7292099595069885,
+      "sampling/importance_sampling_ratio/max": 1.110335350036621,
+      "sampling/importance_sampling_ratio/mean": 0.9453052282333374,
+      "sampling/importance_sampling_ratio/min": 0.4698891043663025,
+      "sampling/sampling_logp_difference/max": 0.7473087310791016,
+      "sampling/sampling_logp_difference/mean": 0.026049025356769562,
+      "step": 41,
+      "step_time": 10.046313624000959
+    },
+    {
+      "clip_ratio/high_max": 0.012500000186264515,
+      "clip_ratio/high_mean": 0.0062500000931322575,
+      "clip_ratio/low_mean": 0.0062500000931322575,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.012500000186264515,
+      "entropy": 0.11148970657086466,
+      "epoch": 0.00042,
+      "grad_norm": 0.18441708385944366,
+      "kl": 1.8012165799736977,
+      "learning_rate": 9.990517373346957e-05,
+      "loss": -0.0113,
+      "step": 42,
+      "step_time": 6.999212858999272
+    },
+    {
+      "clip_ratio/high_max": 0.008928571827709675,
+      "clip_ratio/high_mean": 0.004464285913854837,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004464285913854837,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1055.0,
+      "completions/max_terminated_length": 1055.0,
+      "completions/mean_length": 225.75,
+      "completions/mean_terminated_length": 225.75,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.07133681878985954,
+      "epoch": 0.00043,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13256920874118805,
+      "kl": 1.7951918244361877,
+      "learning_rate": 9.98709505668081e-05,
+      "loss": -0.0056,
+      "num_tokens": 838155.0,
+      "reward": 0.7849375009536743,
+      "reward_std": 0.504135251045227,
+      "rewards/rollout_reward_func/mean": 0.7849375009536743,
+      "rewards/rollout_reward_func/std": 0.5278446674346924,
+      "sampling/importance_sampling_ratio/max": 1.3283132314682007,
+      "sampling/importance_sampling_ratio/mean": 1.0156052112579346,
+      "sampling/importance_sampling_ratio/min": 0.6983067393302917,
+      "sampling/sampling_logp_difference/max": 0.3479170799255371,
+      "sampling/sampling_logp_difference/mean": 0.013128189370036125,
+      "step": 43,
+      "step_time": 9.243569922999086
+    },
+    {
+      "clip_ratio/high_max": 0.037500000558793545,
+      "clip_ratio/high_mean": 0.018750000279396772,
+      "clip_ratio/low_mean": 0.018750000279396772,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.037500000558793545,
+      "entropy": 0.0734066000659368,
+      "epoch": 0.00044,
+      "grad_norm": 0.13717395067214966,
+      "kl": 1.8053691610693932,
+      "learning_rate": 9.983147525093428e-05,
+      "loss": -0.0058,
+      "step": 44,
+      "step_time": 6.4607698830004665
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1415.0,
+      "completions/max_terminated_length": 1415.0,
+      "completions/mean_length": 238.96875,
+      "completions/mean_terminated_length": 238.96875,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.09801992701250128,
+      "epoch": 0.00045,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1328868269920349,
+      "kl": 1.706927303224802,
+      "learning_rate": 9.978675333374685e-05,
+      "loss": 0.0016,
+      "num_tokens": 876303.0,
+      "reward": 0.7598124742507935,
+      "reward_std": 0.5679430365562439,
+      "rewards/rollout_reward_func/mean": 0.7598124742507935,
+      "rewards/rollout_reward_func/std": 0.5967590808868408,
+      "sampling/importance_sampling_ratio/max": 1.169681429862976,
+      "sampling/importance_sampling_ratio/mean": 1.0279597043991089,
+      "sampling/importance_sampling_ratio/min": 0.8635076880455017,
+      "sampling/sampling_logp_difference/max": 0.1565859317779541,
+      "sampling/sampling_logp_difference/mean": 0.012495058588683605,
+      "step": 45,
+      "step_time": 10.574091848000535
+    },
+    {
+      "clip_ratio/high_max": 0.05000000074505806,
+      "clip_ratio/high_mean": 0.02500000037252903,
+      "clip_ratio/low_mean": 0.004464285913854837,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.029464286286383867,
+      "entropy": 0.10025507608952466,
+      "epoch": 0.00046,
+      "grad_norm": 0.06521525233983994,
+      "kl": 1.6916628777980804,
+      "learning_rate": 9.973679110050689e-05,
+      "loss": 0.001,
+      "step": 46,
+      "step_time": 6.771763879000218
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1134.0,
+      "completions/max_terminated_length": 1134.0,
+      "completions/mean_length": 202.84375,
+      "completions/mean_terminated_length": 202.84375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.0900510510073218,
+      "epoch": 0.00047,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14434900879859924,
+      "kl": 1.4058816023170948,
+      "learning_rate": 9.968159557295458e-05,
+      "loss": -0.006,
+      "num_tokens": 911967.0,
+      "reward": 0.8627499938011169,
+      "reward_std": 0.5587868690490723,
+      "rewards/rollout_reward_func/mean": 0.8627499938011169,
+      "rewards/rollout_reward_func/std": 0.5566409230232239,
+      "sampling/importance_sampling_ratio/max": 1.1530331373214722,
+      "sampling/importance_sampling_ratio/mean": 0.9977372288703918,
+      "sampling/importance_sampling_ratio/min": 0.5316535234451294,
+      "sampling/sampling_logp_difference/max": 0.6474550366401672,
+      "sampling/sampling_logp_difference/mean": 0.016066744923591614,
+      "step": 47,
+      "step_time": 9.447806593999758
+    },
+    {
+      "clip_ratio/high_max": 0.02142857201397419,
+      "clip_ratio/high_mean": 0.010714286006987095,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.010714286006987095,
+      "entropy": 0.09766281770862406,
+      "epoch": 0.00048,
+      "grad_norm": 0.06667943298816681,
+      "kl": 1.405099406838417,
+      "learning_rate": 9.962117450832225e-05,
+      "loss": -0.0064,
+      "step": 48,
+      "step_time": 6.173912093000354
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 658.0,
+      "completions/max_terminated_length": 658.0,
+      "completions/mean_length": 215.1875,
+      "completions/mean_terminated_length": 215.1875,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.09605835549155017,
+      "epoch": 0.00049,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.06864096969366074,
+      "kl": 1.3723388984799385,
+      "learning_rate": 9.955553639824423e-05,
+      "loss": -0.0005,
+      "num_tokens": 948707.0,
+      "reward": 0.893958330154419,
+      "reward_std": 0.47342604398727417,
+      "rewards/rollout_reward_func/mean": 0.893958330154419,
+      "rewards/rollout_reward_func/std": 0.47255393862724304,
+      "sampling/importance_sampling_ratio/max": 1.1744683980941772,
+      "sampling/importance_sampling_ratio/mean": 0.9887692928314209,
+      "sampling/importance_sampling_ratio/min": 0.89034104347229,
+      "sampling/sampling_logp_difference/max": 0.16428709030151367,
+      "sampling/sampling_logp_difference/mean": 0.010529636405408382,
+      "step": 49,
+      "step_time": 9.043114574000356
+    },
+    {
+      "clip_ratio/high_max": 0.02500000037252903,
+      "clip_ratio/high_mean": 0.012500000186264515,
+      "clip_ratio/low_mean": 0.016666667070239782,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.029166667256504297,
+      "entropy": 0.1075448725168826,
+      "epoch": 0.0005,
+      "grad_norm": 0.0960911437869072,
+      "kl": 1.4063957203179598,
+      "learning_rate": 9.948469046756344e-05,
+      "loss": -0.0009,
+      "step": 50,
+      "step_time": 5.642086785000174
+    },
+    {
+      "clip_ratio/high_max": 0.008928571827709675,
+      "clip_ratio/high_mean": 0.004464285913854837,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004464285913854837,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1071.0,
+      "completions/max_terminated_length": 1071.0,
+      "completions/mean_length": 316.09375,
+      "completions/mean_terminated_length": 316.09375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.17863493447657675,
+      "epoch": 0.00051,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6474497318267822,
+      "kl": 2.192594848573208,
+      "learning_rate": 9.940864667303489e-05,
+      "loss": 0.0143,
+      "num_tokens": 989035.0,
+      "reward": 0.5325833559036255,
+      "reward_std": 0.8769392967224121,
+      "rewards/rollout_reward_func/mean": 0.5325833559036255,
+      "rewards/rollout_reward_func/std": 0.853682279586792,
+      "sampling/importance_sampling_ratio/max": 1.9499620199203491,
+      "sampling/importance_sampling_ratio/mean": 0.9530384540557861,
+      "sampling/importance_sampling_ratio/min": 0.4134676158428192,
+      "sampling/sampling_logp_difference/max": 0.9089558124542236,
+      "sampling/sampling_logp_difference/mean": 0.05595976859331131,
+      "step": 51,
+      "step_time": 9.98290348799992
+    },
+    {
+      "clip_ratio/high_max": 0.04642857238650322,
+      "clip_ratio/high_mean": 0.02321428619325161,
+      "clip_ratio/low_mean": 0.0703125,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.09352678572759032,
+      "entropy": 0.14376915276807267,
+      "epoch": 0.00052,
+      "grad_norm": 10.807016372680664,
+      "kl": 35.954969733953476,
+      "learning_rate": 9.932741570192633e-05,
+      "loss": 0.0328,
+      "step": 52,
+      "step_time": 5.674436524999692
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1646.0,
+      "completions/max_terminated_length": 1646.0,
+      "completions/mean_length": 292.21875,
+      "completions/mean_terminated_length": 292.21875,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.11845442898629699,
+      "epoch": 0.00053,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2344297617673874,
+      "kl": 1.753089528530836,
+      "learning_rate": 9.924100897051629e-05,
+      "loss": -0.011,
+      "num_tokens": 1028265.0,
+      "reward": 0.9336875081062317,
+      "reward_std": 0.40108171105384827,
+      "rewards/rollout_reward_func/mean": 0.9336875081062317,
+      "rewards/rollout_reward_func/std": 0.44726231694221497,
+      "sampling/importance_sampling_ratio/max": 1.4263041019439697,
+      "sampling/importance_sampling_ratio/mean": 1.0030502080917358,
+      "sampling/importance_sampling_ratio/min": 0.7242881655693054,
+      "sampling/sampling_logp_difference/max": 0.3650219440460205,
+      "sampling/sampling_logp_difference/mean": 0.015114575624465942,
+      "step": 53,
+      "step_time": 11.967371391000597
+    },
+    {
+      "clip_ratio/high_max": 0.028373016510158777,
+      "clip_ratio/high_mean": 0.014186508255079389,
+      "clip_ratio/low_mean": 0.05000000121071935,
+      "clip_ratio/low_min": 0.012500000186264515,
+      "clip_ratio/region_mean": 0.06418650993146002,
+      "entropy": 0.18176122102886438,
+      "epoch": 0.00054,
+      "grad_norm": 0.0920347049832344,
+      "kl": 1.7955459654331207,
+      "learning_rate": 9.914943862248966e-05,
+      "loss": -0.0119,
+      "step": 54,
+      "step_time": 6.745521994000001
+    },
+    {
+      "clip_ratio/high_max": 0.029761905781924725,
+      "clip_ratio/high_mean": 0.014880952890962362,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.014880952890962362,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1255.0,
+      "completions/max_terminated_length": 1255.0,
+      "completions/mean_length": 414.40625,
+      "completions/mean_terminated_length": 414.40625,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.21612564753741026,
+      "epoch": 0.00055,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6154258847236633,
+      "kl": 2.4827291294932365,
+      "learning_rate": 9.905271752723088e-05,
+      "loss": -0.0024,
+      "num_tokens": 1070717.0,
+      "reward": 1.012783169746399,
+      "reward_std": 0.3933926522731781,
+      "rewards/rollout_reward_func/mean": 1.012783169746399,
+      "rewards/rollout_reward_func/std": 0.45623427629470825,
+      "sampling/importance_sampling_ratio/max": 1.7324354648590088,
+      "sampling/importance_sampling_ratio/mean": 0.9738206267356873,
+      "sampling/importance_sampling_ratio/min": 0.5882648825645447,
+      "sampling/sampling_logp_difference/max": 0.5556355714797974,
+      "sampling/sampling_logp_difference/mean": 0.04333949089050293,
+      "step": 55,
+      "step_time": 11.013223535000407
+    },
+    {
+      "clip_ratio/high_max": 0.1761904815211892,
+      "clip_ratio/high_mean": 0.11964286165311933,
+      "clip_ratio/low_mean": 0.014062500093132257,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.13370536174625158,
+      "entropy": 0.17086607962846756,
+      "epoch": 0.00056,
+      "grad_norm": 0.2525999844074249,
+      "kl": 2.5824186354875565,
+      "learning_rate": 9.895085927801542e-05,
+      "loss": -0.0014,
+      "step": 56,
+      "step_time": 6.016107872999555
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1386.0,
+      "completions/max_terminated_length": 1386.0,
+      "completions/mean_length": 284.8125,
+      "completions/mean_terminated_length": 284.8125,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.18887321837246418,
+      "epoch": 0.00057,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2054257094860077,
+      "kl": 1.618610456585884,
+      "learning_rate": 9.884387819009922e-05,
+      "loss": -0.0089,
+      "num_tokens": 1110225.0,
+      "reward": 0.6539791822433472,
+      "reward_std": 0.5930131673812866,
+      "rewards/rollout_reward_func/mean": 0.6539791822433472,
+      "rewards/rollout_reward_func/std": 0.6302005052566528,
+      "sampling/importance_sampling_ratio/max": 1.8165205717086792,
+      "sampling/importance_sampling_ratio/mean": 1.0121564865112305,
+      "sampling/importance_sampling_ratio/min": 0.865464985370636,
+      "sampling/sampling_logp_difference/max": 0.5969423055648804,
+      "sampling/sampling_logp_difference/mean": 0.021497800946235657,
+      "step": 57,
+      "step_time": 11.156232571999908
+    },
+    {
+      "clip_ratio/high_max": 0.058333334513008595,
+      "clip_ratio/high_mean": 0.029166667256504297,
+      "clip_ratio/low_mean": 0.021875000093132257,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.051041667349636555,
+      "entropy": 0.18203126941807568,
+      "epoch": 0.00058,
+      "grad_norm": 0.21828053891658783,
+      "kl": 1.6002055294811726,
+      "learning_rate": 9.873178929870695e-05,
+      "loss": -0.0086,
+      "step": 58,
+      "step_time": 6.244077477000019
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0078125,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0078125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1694.0,
+      "completions/max_terminated_length": 1694.0,
+      "completions/mean_length": 245.75,
+      "completions/mean_terminated_length": 245.75,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.20505834033247083,
+      "epoch": 0.00059,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.239760160446167,
+      "kl": 2.8409063071012497,
+      "learning_rate": 9.86146083569188e-05,
+      "loss": -0.0085,
+      "num_tokens": 1147717.0,
+      "reward": 0.6759583353996277,
+      "reward_std": 0.7167876958847046,
+      "rewards/rollout_reward_func/mean": 0.6759583353996277,
+      "rewards/rollout_reward_func/std": 0.8124518990516663,
+      "sampling/importance_sampling_ratio/max": 2.0188424587249756,
+      "sampling/importance_sampling_ratio/mean": 1.0246764421463013,
+      "sampling/importance_sampling_ratio/min": 0.8060900568962097,
+      "sampling/sampling_logp_difference/max": 0.4889869689941406,
+      "sampling/sampling_logp_difference/mean": 0.015371443703770638,
+      "step": 59,
+      "step_time": 12.236902825000243
+    },
+    {
+      "clip_ratio/high_max": 0.015873016323894262,
+      "clip_ratio/high_mean": 0.007936508161947131,
+      "clip_ratio/low_mean": 0.024479167070239782,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.032415675232186913,
+      "entropy": 0.2697412967681885,
+      "epoch": 0.0006,
+      "grad_norm": 1.4479070901870728,
+      "kl": 1.8084059804677963,
+      "learning_rate": 9.84923518334567e-05,
+      "loss": 0.0064,
+      "step": 60,
+      "step_time": 6.916969224000468
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1141.0,
+      "completions/max_terminated_length": 1141.0,
+      "completions/mean_length": 306.1875,
+      "completions/mean_terminated_length": 306.1875,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.2432498256675899,
+      "epoch": 0.00061,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15688766539096832,
+      "kl": 1.928163781762123,
+      "learning_rate": 9.83650369103696e-05,
+      "loss": -0.0154,
+      "num_tokens": 1187828.0,
+      "reward": 0.7319375276565552,
+      "reward_std": 0.5858461856842041,
+      "rewards/rollout_reward_func/mean": 0.7319375276565552,
+      "rewards/rollout_reward_func/std": 0.6664270162582397,
+      "sampling/importance_sampling_ratio/max": 1.3562289476394653,
+      "sampling/importance_sampling_ratio/mean": 0.9950809478759766,
+      "sampling/importance_sampling_ratio/min": 0.8087555766105652,
+      "sampling/sampling_logp_difference/max": 0.3163447380065918,
+      "sampling/sampling_logp_difference/mean": 0.0226910300552845,
+      "step": 61,
+      "step_time": 10.608102406999706
+    },
+    {
+      "clip_ratio/high_max": 0.030357143841683865,
+      "clip_ratio/high_mean": 0.015178571920841932,
+      "clip_ratio/low_mean": 0.010416666977107525,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.025595238897949457,
+      "entropy": 0.22057522786781192,
+      "epoch": 0.00062,
+      "grad_norm": 0.11316227912902832,
+      "kl": 1.9541875012218952,
+      "learning_rate": 9.823268148061883e-05,
+      "loss": -0.0145,
+      "step": 62,
+      "step_time": 5.81027040199956
+    },
+    {
+      "clip_ratio/high_max": 0.008928571827709675,
+      "clip_ratio/high_mean": 0.004464285913854837,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004464285913854837,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 850.0,
+      "completions/max_terminated_length": 850.0,
+      "completions/mean_length": 260.40625,
+      "completions/mean_terminated_length": 260.40625,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "entropy": 0.175975039601326,
+      "epoch": 0.00063,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.23777881264686584,
+      "kl": 1.8976656533777714,
+      "learning_rate": 9.809530414556335e-05,
+      "loss": -0.0073,
+      "num_tokens": 1226242.0,
+      "reward": 0.9314790964126587,
+      "reward_std": 0.4598495364189148,
+      "rewards/rollout_reward_func/mean": 0.9314790964126587,
+      "rewards/rollout_reward_func/std": 0.4464755654335022,
+      "sampling/importance_sampling_ratio/max": 1.409760594367981,
+      "sampling/importance_sampling_ratio/mean": 1.0067412853240967,
+      "sampling/importance_sampling_ratio/min": 0.8432673811912537,
+      "sampling/sampling_logp_difference/max": 0.34805774688720703,
+      "sampling/sampling_logp_difference/mean": 0.015504386276006699,
+      "step": 63,
+      "step_time": 9.866925459000186
+    },
+    {
+      "clip_ratio/high_max": 0.058928572572767735,
+      "clip_ratio/high_mean": 0.029464286286383867,
+      "clip_ratio/low_mean": 0.031250000931322575,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.06071428768336773,
+      "entropy": 0.21677518263459206,
+      "epoch": 0.00064,
+      "grad_norm": 0.10355119407176971,
+      "kl": 1.9398510716855526,
+      "learning_rate": 9.79529242123455e-05,
+      "loss": -0.0093,
+      "step": 64,
+      "step_time": 5.271362073999853
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1041.0,
+      "completions/max_terminated_length": 1041.0,
+      "completions/mean_length": 201.59375,
+      "completions/mean_terminated_length": 201.59375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.21963271405547857,
+      "epoch": 0.00065,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5803110599517822,
+      "kl": 3.1230736933648586,
+      "learning_rate": 9.780556169117757e-05,
+      "loss": -0.0078,
+      "num_tokens": 1260829.0,
+      "reward": 0.6616458296775818,
+      "reward_std": 0.7855631113052368,
+      "rewards/rollout_reward_func/mean": 0.6616458296775818,
+      "rewards/rollout_reward_func/std": 0.8183318972587585,
+      "sampling/importance_sampling_ratio/max": 1.4000798463821411,
+      "sampling/importance_sampling_ratio/mean": 1.0056772232055664,
+      "sampling/importance_sampling_ratio/min": 0.71177077293396,
+      "sampling/sampling_logp_difference/max": 0.34459519386291504,
+      "sampling/sampling_logp_difference/mean": 0.03253237158060074,
+      "step": 65,
+      "step_time": 10.17715724000027
+    },
+    {
+      "clip_ratio/high_max": 0.11666666995733976,
+      "clip_ratio/high_mean": 0.062797621358186,
+      "clip_ratio/low_mean": 0.078125,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.14092262228950858,
+      "entropy": 0.1336969246622175,
+      "epoch": 0.00066,
+      "grad_norm": 0.13620424270629883,
+      "kl": 2.933967538177967,
+      "learning_rate": 9.765323729252955e-05,
+      "loss": -0.0085,
+      "step": 66,
+      "step_time": 5.5793444789999285
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 668.0,
+      "completions/max_terminated_length": 668.0,
+      "completions/mean_length": 276.25,
+      "completions/mean_terminated_length": 276.25,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.16010587196797132,
+      "epoch": 0.00067,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12310951948165894,
+      "kl": 2.098513074219227,
+      "learning_rate": 9.749597242421838e-05,
+      "loss": -0.0122,
+      "num_tokens": 1300882.0,
+      "reward": 0.8329999446868896,
+      "reward_std": 0.5346106290817261,
+      "rewards/rollout_reward_func/mean": 0.8329999446868896,
+      "rewards/rollout_reward_func/std": 0.6208124756813049,
+      "sampling/importance_sampling_ratio/max": 1.1149344444274902,
+      "sampling/importance_sampling_ratio/mean": 0.9865162372589111,
+      "sampling/importance_sampling_ratio/min": 0.8121993541717529,
+      "sampling/sampling_logp_difference/max": 0.21269893646240234,
+      "sampling/sampling_logp_difference/mean": 0.015977365896105766,
+      "step": 67,
+      "step_time": 9.141569508000657
+    },
+    {
+      "clip_ratio/high_max": 0.012500000186264515,
+      "clip_ratio/high_mean": 0.0062500000931322575,
+      "clip_ratio/low_mean": 0.012500000186264515,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.018750000279396772,
+      "entropy": 0.1498410563217476,
+      "epoch": 0.00068,
+      "grad_norm": 0.1016835868358612,
+      "kl": 2.0821792110800743,
+      "learning_rate": 9.733378918839942e-05,
+      "loss": -0.0124,
+      "step": 68,
+      "step_time": 5.649315163999745
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1055.0,
+      "completions/max_terminated_length": 1055.0,
+      "completions/mean_length": 288.59375,
+      "completions/mean_terminated_length": 288.59375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.14323098736349493,
+      "epoch": 0.00069,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11429130285978317,
+      "kl": 2.039843175560236,
+      "learning_rate": 9.716671037846007e-05,
+      "loss": -0.0046,
+      "num_tokens": 1340730.0,
+      "reward": 0.7588750123977661,
+      "reward_std": 0.48252782225608826,
+      "rewards/rollout_reward_func/mean": 0.7588750123977661,
+      "rewards/rollout_reward_func/std": 0.5914071798324585,
+      "sampling/importance_sampling_ratio/max": 1.140119194984436,
+      "sampling/importance_sampling_ratio/mean": 0.9898408651351929,
+      "sampling/importance_sampling_ratio/min": 0.8956636190414429,
+      "sampling/sampling_logp_difference/max": 0.13106560707092285,
+      "sampling/sampling_logp_difference/mean": 0.01080230250954628,
+      "step": 69,
+      "step_time": 9.683018002999916
+    },
+    {
+      "clip_ratio/high_max": 0.07500000111758709,
+      "clip_ratio/high_mean": 0.04821428656578064,
+      "clip_ratio/low_mean": 0.021875000093132257,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0700892866589129,
+      "entropy": 0.1461901287548244,
+      "epoch": 0.0007,
+      "grad_norm": 0.1435869336128235,
+      "kl": 1.9891464561223984,
+      "learning_rate": 9.699475947581644e-05,
+      "loss": -0.0043,
+      "step": 70,
+      "step_time": 6.030160345999548
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1151.0,
+      "completions/max_terminated_length": 1151.0,
+      "completions/mean_length": 270.71875,
+      "completions/mean_terminated_length": 270.71875,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.15293755863967817,
+      "epoch": 0.00071,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3050353527069092,
+      "kl": 1.9519358221441507,
+      "learning_rate": 9.681796064661319e-05,
+      "loss": -0.0008,
+      "num_tokens": 1379557.0,
+      "reward": 0.61760413646698,
+      "reward_std": 0.6345539093017578,
+      "rewards/rollout_reward_func/mean": 0.61760413646698,
+      "rewards/rollout_reward_func/std": 0.6240441203117371,
+      "sampling/importance_sampling_ratio/max": 1.1850391626358032,
+      "sampling/importance_sampling_ratio/mean": 1.007583737373352,
+      "sampling/importance_sampling_ratio/min": 0.869886577129364,
+      "sampling/sampling_logp_difference/max": 0.16973721981048584,
+      "sampling/sampling_logp_difference/mean": 0.008598901331424713,
+      "step": 71,
+      "step_time": 10.008160938000401
+    },
+    {
+      "clip_ratio/high_max": 0.029761905781924725,
+      "clip_ratio/high_mean": 0.014880952890962362,
+      "clip_ratio/low_mean": 0.021875000093132257,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.036755953915417194,
+      "entropy": 0.13349127455148846,
+      "epoch": 0.00072,
+      "grad_norm": 0.11685528606176376,
+      "kl": 1.9348566476255655,
+      "learning_rate": 9.663633873832725e-05,
+      "loss": -0.0011,
+      "step": 72,
+      "step_time": 6.208596755999906
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 723.0,
+      "completions/max_terminated_length": 723.0,
+      "completions/mean_length": 207.84375,
+      "completions/mean_terminated_length": 207.84375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.14575498150952626,
+      "epoch": 0.00073,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1255907416343689,
+      "kl": 1.1826815884560347,
+      "learning_rate": 9.644991927627566e-05,
+      "loss": -0.006,
+      "num_tokens": 1416475.0,
+      "reward": 0.7622500061988831,
+      "reward_std": 0.5831079483032227,
+      "rewards/rollout_reward_func/mean": 0.7622500061988831,
+      "rewards/rollout_reward_func/std": 0.6012318730354309,
+      "sampling/importance_sampling_ratio/max": 1.1591823101043701,
+      "sampling/importance_sampling_ratio/mean": 0.98829185962677,
+      "sampling/importance_sampling_ratio/min": 0.9088780879974365,
+      "sampling/sampling_logp_difference/max": 0.14767801761627197,
+      "sampling/sampling_logp_difference/mean": 0.007826489396393299,
+      "step": 73,
+      "step_time": 9.11005223400025
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "entropy": 0.1513779065135168,
+      "epoch": 0.00074,
+      "grad_norm": 0.12437106668949127,
+      "kl": 1.1780924815684557,
+      "learning_rate": 9.625872846002834e-05,
+      "loss": -0.0066,
+      "step": 74,
+      "step_time": 5.63687633499967
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 705.0,
+      "completions/max_terminated_length": 705.0,
+      "completions/mean_length": 281.375,
+      "completions/mean_terminated_length": 281.375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "entropy": 0.18086719792336226,
+      "epoch": 0.00075,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15126265585422516,
+      "kl": 1.6214838400483131,
+      "learning_rate": 9.606279315972582e-05,
+      "loss": 0.002,
+      "num_tokens": 1455741.0,
+      "reward": 0.6960416436195374,
+      "reward_std": 0.6535277366638184,
+      "rewards/rollout_reward_func/mean": 0.6960416436195374,
+      "rewards/rollout_reward_func/std": 0.6603111624717712,
+      "sampling/importance_sampling_ratio/max": 1.1913020610809326,
+      "sampling/importance_sampling_ratio/mean": 0.9901882410049438,
+      "sampling/importance_sampling_ratio/min": 0.8861557245254517,
+      "sampling/sampling_logp_difference/max": 0.17504119873046875,
+      "sampling/sampling_logp_difference/mean": 0.008757364936172962,
+      "step": 75,
+      "step_time": 8.987358285000255
     }
   ],
   "logging_steps": 1.0,
+  "max_steps": 300,
+  "num_input_tokens_seen": 1455741,
+  "num_train_epochs": 1,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
     }
   },
   "total_flos": 0.0,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ed4e467527db6458333d49b7f960c0fc9f43d07312ed047c641931255b615a8
-size 7825

 version https://git-lfs.github.com/spec/v1
+oid sha256:d9a8ff53bf4dd13387a7f6f06286ba4c2179cd03bdae768c161a96c9eecce8b1
+size 8145