Instructions to use bimabk/environment_test_affine with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries
PEFT
How to use bimabk/environment_test_affine with PEFT:
```
Base model is not found.
```

How to use bimabk/environment_test_affine with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="bimabk/environment_test_affine")
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("bimabk/environment_test_affine")
model = AutoModelForCausalLM.from_pretrained("bimabk/environment_test_affine", device_map="auto")
messages = [
    {"role": "user", "content": "Who are you?"},
]
inputs = tokenizer.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=40)
print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use bimabk/environment_test_affine with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "bimabk/environment_test_affine"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test_affine",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/bimabk/environment_test_affine

SGLang

How to use bimabk/environment_test_affine with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "bimabk/environment_test_affine" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test_affine",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "bimabk/environment_test_affine" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test_affine",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use bimabk/environment_test_affine with Docker Model Runner:
```
docker model run hf.co/bimabk/environment_test_affine
```

bimabk commited on Feb 9

Commit

4940b17

verified ·

1 Parent(s): afe908f

Upload task output 1

Browse files

Files changed (5) hide show

adapter_config.json +4 -4
adapter_model.safetensors +1 -1
loss.txt +1 -1
trainer_state.json +461 -362
training_args.bin +1 -1

adapter_config.json CHANGED Viewed

@@ -29,13 +29,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "gate_proj",
-    "k_proj",
-    "v_proj",
     "o_proj",
-    "q_proj",
     "up_proj",
-    "down_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "q_proj",
     "gate_proj",
     "o_proj",
+    "down_proj",
+    "k_proj",
     "up_proj",
+    "v_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:377dc67b8d9ac5d2e476c15f9cf0609b27690828cfdd9f4491dba511d66d3ae0
 size 957942768

 version https://git-lfs.github.com/spec/v1
+oid sha256:c9bc32fac7f25bcede80af9be64db4efa53407d9dbd6b511856089e76298e8f7
 size 957942768

loss.txt CHANGED Viewed

	@@ -1 +1 @@
1	- 68,~~no_eval~~


1	+ 75,-0.5700000166893006

trainer_state.json CHANGED Viewed

@@ -2,446 +2,545 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.0102010201020102,
   "eval_steps": 500,
-  "global_step": 68,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 374.4,
-      "completions/max_terminated_length": 374.4,
-      "completions/mean_length": 291.50001220703126,
-      "completions/mean_terminated_length": 291.50001220703126,
-      "completions/min_length": 174.8,
-      "completions/min_terminated_length": 174.8,
-      "entropy": 0.737431001663208,
-      "epoch": 0.00075007500750075,
-      "frac_reward_zero_std": 0.6533333778381347,
-      "grad_norm": 0.369140625,
-      "kl": 0.00885026976466179,
       "learning_rate": 1.137216e-06,
-      "loss": 0.00034544954542070627,
-      "num_tokens": 126703.0,
-      "reward": 0.0383333370089531,
-      "reward_std": 0.05835988484323025,
-      "rewards/env_goofspiel_reward/mean": 0.03833333514630795,
-      "rewards/env_goofspiel_reward/std": 0.15159520953893663,
-      "sampling/importance_sampling_ratio/max": 1.7607571125030517,
-      "sampling/importance_sampling_ratio/mean": 0.9916059970855713,
-      "sampling/importance_sampling_ratio/min": 0.5116463124752044,
-      "sampling/sampling_logp_difference/max": 0.6887424349784851,
-      "sampling/sampling_logp_difference/mean": 0.06558751240372658,
       "step": 5,
-      "step_time": 3.541577606199735
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.6,
-      "completions/max_terminated_length": 373.6,
-      "completions/mean_length": 292.0866760253906,
-      "completions/mean_terminated_length": 292.0866760253906,
-      "completions/min_length": 179.8,
-      "completions/min_terminated_length": 179.8,
-      "entropy": 0.7154561956723531,
-      "epoch": 0.0015001500150015,
-      "frac_reward_zero_std": 0.6666666865348816,
-      "grad_norm": 0.462890625,
-      "kl": 0.009678552253171801,
       "learning_rate": 2.5587359999999995e-06,
-      "loss": -3.925009514205158e-05,
-      "num_tokens": 254525.0,
-      "reward": 0.03886667159385979,
-      "reward_std": 0.05817132312804461,
-      "rewards/env_goofspiel_reward/mean": 0.03886666861362755,
-      "rewards/env_goofspiel_reward/std": 0.14201362472958862,
-      "sampling/importance_sampling_ratio/max": 1.9776098012924195,
-      "sampling/importance_sampling_ratio/mean": 1.0280600190162659,
-      "sampling/importance_sampling_ratio/min": 0.4622887670993805,
-      "sampling/sampling_logp_difference/max": 0.8463176250457763,
-      "sampling/sampling_logp_difference/mean": 0.06613281443715095,
       "step": 10,
-      "step_time": 3.1668822380001074
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.6,
-      "completions/max_terminated_length": 373.6,
-      "completions/mean_length": 288.72001953125,
-      "completions/mean_terminated_length": 288.72001953125,
-      "completions/min_length": 187.2,
-      "completions/min_terminated_length": 187.2,
-      "entropy": 0.6402347763379415,
-      "epoch": 0.0022502250225022503,
-      "frac_reward_zero_std": 0.8000000476837158,
-      "grad_norm": 0.412109375,
-      "kl": 0.00951577623685201,
       "learning_rate": 3.9802559999999995e-06,
-      "loss": 0.00047838701866567137,
-      "num_tokens": 380635.0,
-      "reward": 0.07146667279303073,
-      "reward_std": 0.09088679552078247,
-      "rewards/env_goofspiel_reward/mean": 0.07146666683256626,
-      "rewards/env_goofspiel_reward/std": 0.22424434274435043,
-      "sampling/importance_sampling_ratio/max": 1.5242333889007569,
-      "sampling/importance_sampling_ratio/mean": 0.9860278725624084,
-      "sampling/importance_sampling_ratio/min": 0.5668014168739319,
-      "sampling/sampling_logp_difference/max": 0.5752804994583129,
-      "sampling/sampling_logp_difference/mean": 0.05473804771900177,
       "step": 15,
-      "step_time": 3.1726541418000123
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.2,
-      "completions/max_terminated_length": 373.2,
-      "completions/mean_length": 273.82001037597655,
-      "completions/mean_terminated_length": 273.82001037597655,
-      "completions/min_length": 200.2,
-      "completions/min_terminated_length": 200.2,
-      "entropy": 0.5869409064451854,
-      "epoch": 0.003000300030003,
-      "frac_reward_zero_std": 0.7600000500679016,
-      "grad_norm": 0.158203125,
-      "kl": 0.029618356159577766,
       "learning_rate": 5.401775999999999e-06,
-      "loss": 0.00020813762675970794,
-      "num_tokens": 501743.0,
-      "reward": 0.06360000669956208,
-      "reward_std": 0.07976165302097797,
-      "rewards/env_goofspiel_reward/mean": 0.06360000558197498,
-      "rewards/env_goofspiel_reward/std": 0.20161283165216445,
-      "sampling/importance_sampling_ratio/max": 1.7165520668029786,
-      "sampling/importance_sampling_ratio/mean": 0.9985299944877625,
-      "sampling/importance_sampling_ratio/min": 0.6464852690696716,
-      "sampling/sampling_logp_difference/max": 0.6010382175445557,
-      "sampling/sampling_logp_difference/mean": 0.05117494091391563,
       "step": 20,
-      "step_time": 3.1184157538001274
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 374.2,
-      "completions/max_terminated_length": 374.2,
-      "completions/mean_length": 291.16668090820315,
-      "completions/mean_terminated_length": 291.16668090820315,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
-      "entropy": 0.5825250327587128,
-      "epoch": 0.0037503750375037503,
-      "frac_reward_zero_std": 0.8133333921432495,
-      "grad_norm": 0.30859375,
-      "kl": 0.03789825042088826,
       "learning_rate": 6.8232959999999994e-06,
-      "loss": -5.795806646347046e-05,
-      "num_tokens": 629207.0,
-      "reward": 0.05593333579599857,
-      "reward_std": 0.0791016798466444,
-      "rewards/env_goofspiel_reward/mean": 0.055933335050940516,
-      "rewards/env_goofspiel_reward/std": 0.16461323350667953,
-      "sampling/importance_sampling_ratio/max": 1.6597614526748656,
-      "sampling/importance_sampling_ratio/mean": 0.9899526715278626,
-      "sampling/importance_sampling_ratio/min": 0.5461978197097779,
-      "sampling/sampling_logp_difference/max": 0.5474630713462829,
-      "sampling/sampling_logp_difference/mean": 0.056329603493213656,
       "step": 25,
-      "step_time": 3.1455567411999255
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.8,
-      "completions/max_terminated_length": 373.8,
-      "completions/mean_length": 288.2733459472656,
-      "completions/mean_terminated_length": 288.2733459472656,
-      "completions/min_length": 205.4,
-      "completions/min_terminated_length": 205.4,
-      "entropy": 0.6367802540461223,
-      "epoch": 0.004500450045004501,
-      "frac_reward_zero_std": 0.8133333921432495,
-      "grad_norm": 0.3203125,
-      "kl": 0.15099084513882796,
       "learning_rate": 8.244816e-06,
-      "loss": 2.9896479099988936e-05,
-      "num_tokens": 755141.0,
-      "reward": 0.04773333668708801,
-      "reward_std": 0.056945668533444405,
-      "rewards/env_goofspiel_reward/mean": 0.04773333407938481,
-      "rewards/env_goofspiel_reward/std": 0.1535952940583229,
-      "sampling/importance_sampling_ratio/max": 1.4472879409790038,
-      "sampling/importance_sampling_ratio/mean": 1.000359261035919,
-      "sampling/importance_sampling_ratio/min": 0.6299581527709961,
-      "sampling/sampling_logp_difference/max": 0.49155421257019044,
-      "sampling/sampling_logp_difference/mean": 0.04638729840517044,
       "step": 30,
-      "step_time": 3.104708982399825
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.4,
-      "completions/max_terminated_length": 373.4,
-      "completions/mean_length": 294.12001953125,
-      "completions/mean_terminated_length": 294.12001953125,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
-      "entropy": 0.6307838877042135,
-      "epoch": 0.005250525052505251,
-      "frac_reward_zero_std": 0.8133333921432495,
-      "grad_norm": 0.2041015625,
-      "kl": 0.21180881708860397,
       "learning_rate": 9.666336e-06,
-      "loss": 0.0003116762964054942,
-      "num_tokens": 881937.0,
-      "reward": 0.04366667197318748,
-      "reward_std": 0.0626968042459339,
-      "rewards/env_goofspiel_reward/mean": 0.04366666899295524,
-      "rewards/env_goofspiel_reward/std": 0.14476882207673042,
-      "sampling/importance_sampling_ratio/max": 1.540164875984192,
-      "sampling/importance_sampling_ratio/mean": 1.007632350921631,
-      "sampling/importance_sampling_ratio/min": 0.6116935849189759,
-      "sampling/sampling_logp_difference/max": 0.5826021313667298,
-      "sampling/sampling_logp_difference/mean": 0.054141230136156085,
       "step": 35,
-      "step_time": 3.152498085000025
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 365.8,
-      "completions/max_terminated_length": 365.8,
-      "completions/mean_length": 281.62001953125,
-      "completions/mean_terminated_length": 281.62001953125,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
-      "entropy": 0.553737320502599,
-      "epoch": 0.006000600060006,
-      "frac_reward_zero_std": 0.8266667246818542,
-      "grad_norm": 0.0888671875,
-      "kl": 0.5656270523866017,
-      "learning_rate": 9.950639260700543e-06,
-      "loss": 0.00017667778301984073,
-      "num_tokens": 1005294.0,
-      "reward": 0.056000004336237905,
-      "reward_std": 0.07919596247375012,
-      "rewards/env_goofspiel_reward/mean": 0.05600000284612179,
-      "rewards/env_goofspiel_reward/std": 0.18038542717695236,
-      "sampling/importance_sampling_ratio/max": 1.5142346620559692,
-      "sampling/importance_sampling_ratio/mean": 0.9889694690704346,
-      "sampling/importance_sampling_ratio/min": 0.6695701956748963,
-      "sampling/sampling_logp_difference/max": 0.45929052233695983,
-      "sampling/sampling_logp_difference/mean": 0.05039779171347618,
       "step": 40,
-      "step_time": 3.061584199799836
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 374.0,
-      "completions/max_terminated_length": 374.0,
-      "completions/mean_length": 303.58001708984375,
-      "completions/mean_terminated_length": 303.58001708984375,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
-      "entropy": 0.48466593225797017,
-      "epoch": 0.0067506750675067504,
-      "frac_reward_zero_std": 0.8266667246818542,
-      "grad_norm": 0.1748046875,
-      "kl": 1.0889011422793071,
-      "learning_rate": 9.950636257297004e-06,
-      "loss": 0.00040990984998643396,
-      "num_tokens": 1135969.0,
-      "reward": 0.04786667115986347,
-      "reward_std": 0.06807081587612629,
-      "rewards/env_goofspiel_reward/mean": 0.047866668179631235,
-      "rewards/env_goofspiel_reward/std": 0.1721431568264961,
-      "sampling/importance_sampling_ratio/max": 1.4374094486236573,
-      "sampling/importance_sampling_ratio/mean": 0.9694242954254151,
-      "sampling/importance_sampling_ratio/min": 0.6029698491096497,
-      "sampling/sampling_logp_difference/max": 0.5160395622253418,
-      "sampling/sampling_logp_difference/mean": 0.04300674088299274,
       "step": 45,
-      "step_time": 3.19344236120005
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 375.6,
-      "completions/max_terminated_length": 375.6,
-      "completions/mean_length": 296.2133483886719,
-      "completions/mean_terminated_length": 296.2133483886719,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
-      "entropy": 0.3623567740122477,
-      "epoch": 0.007500750075007501,
-      "frac_reward_zero_std": 0.8800000429153443,
-      "grad_norm": 0.134765625,
-      "kl": 0.9536839803059896,
-      "learning_rate": 9.950630943585028e-06,
-      "loss": 0.00046116397716104983,
-      "num_tokens": 1262891.0,
-      "reward": 0.040000003576278684,
-      "reward_std": 0.05656854659318924,
-      "rewards/env_goofspiel_reward/mean": 0.04000000208616257,
-      "rewards/env_goofspiel_reward/std": 0.13237088024616242,
-      "sampling/importance_sampling_ratio/max": 1.4614337921142577,
-      "sampling/importance_sampling_ratio/mean": 0.9917921781539917,
-      "sampling/importance_sampling_ratio/min": 0.7945461988449096,
-      "sampling/sampling_logp_difference/max": 0.34459147453308103,
-      "sampling/sampling_logp_difference/mean": 0.023883017525076867,
       "step": 50,
-      "step_time": 3.080125956599841
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.8,
-      "completions/max_terminated_length": 373.8,
-      "completions/mean_length": 297.3133483886719,
-      "completions/mean_terminated_length": 297.3133483886719,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
-      "entropy": 0.412084698677063,
-      "epoch": 0.00825082508250825,
-      "frac_reward_zero_std": 0.840000057220459,
-      "grad_norm": 0.11279296875,
-      "kl": 0.8989204486211141,
-      "learning_rate": 9.950623319567901e-06,
-      "loss": 0.00017252418911084533,
-      "num_tokens": 1391274.0,
-      "reward": 0.059933338314294815,
-      "reward_std": 0.07363338991999627,
-      "rewards/env_goofspiel_reward/mean": 0.059933335334062574,
-      "rewards/env_goofspiel_reward/std": 0.19352754354476928,
-      "sampling/importance_sampling_ratio/max": 1.510833191871643,
-      "sampling/importance_sampling_ratio/mean": 1.0025275230407715,
-      "sampling/importance_sampling_ratio/min": 0.7753824949264526,
-      "sampling/sampling_logp_difference/max": 0.37447161674499513,
-      "sampling/sampling_logp_difference/mean": 0.025511124357581138,
       "step": 55,
-      "step_time": 3.1612697836000736
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 374.0,
       "completions/max_terminated_length": 374.0,
-      "completions/mean_length": 293.233349609375,
-      "completions/mean_terminated_length": 293.233349609375,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
-      "entropy": 0.3592885712782542,
-      "epoch": 0.009000900090009001,
-      "frac_reward_zero_std": 0.8533333778381348,
-      "grad_norm": 0.0164794921875,
-      "kl": 0.9290923396746318,
-      "learning_rate": 9.950613385250344e-06,
-      "loss": 0.00017089147586375476,
-      "num_tokens": 1517875.0,
-      "reward": 0.06400000602006913,
-      "reward_std": 0.07919596470892429,
-      "rewards/env_goofspiel_reward/mean": 0.06400000229477883,
-      "rewards/env_goofspiel_reward/std": 0.192934051156044,
-      "sampling/importance_sampling_ratio/max": 1.3080760478973388,
-      "sampling/importance_sampling_ratio/mean": 1.00157231092453,
-      "sampling/importance_sampling_ratio/min": 0.809451448917389,
-      "sampling/sampling_logp_difference/max": 0.3210816144943237,
-      "sampling/sampling_logp_difference/mean": 0.021189583651721477,
-      "step": 60,
-      "step_time": 3.1453409107998596
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 374.0,
       "completions/max_terminated_length": 374.0,
-      "completions/mean_length": 280.706689453125,
-      "completions/mean_terminated_length": 280.706689453125,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
-      "entropy": 0.46913262009620665,
-      "epoch": 0.00975097509750975,
-      "frac_reward_zero_std": 0.840000057220459,
-      "grad_norm": 0.04541015625,
-      "kl": 1.9159570078055064,
-      "learning_rate": 9.95060114063851e-06,
-      "loss": 0.00043630152940750123,
-      "num_tokens": 1640903.0,
-      "reward": 0.07593334019184113,
-      "reward_std": 0.07363339141011238,
-      "rewards/env_goofspiel_reward/mean": 0.07593333572149277,
-      "rewards/env_goofspiel_reward/std": 0.22238647043704987,
-      "sampling/importance_sampling_ratio/max": 1.42401442527771,
-      "sampling/importance_sampling_ratio/mean": 1.001141333580017,
-      "sampling/importance_sampling_ratio/min": 0.7154132127761841,
-      "sampling/sampling_logp_difference/max": 0.3389713287353516,
-      "sampling/sampling_logp_difference/mean": 0.026769372820854186,
-      "step": 65,
-      "step_time": 3.0763218426000094
     }
   ],
   "logging_steps": 5,
-  "max_steps": 19998,
-  "num_input_tokens_seen": 1713916,
   "num_train_epochs": 3,
   "save_steps": 500,
   "stateful_callbacks": {
@@ -451,13 +550,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
       },
       "attributes": {}
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null
 }

   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
+  "epoch": 0.012,
   "eval_steps": 500,
+  "global_step": 75,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
+      "clip_ratio/high_max": 0.014437134563922881,
+      "clip_ratio/high_mean": 0.007218567281961441,
+      "clip_ratio/low_mean": 0.005763888917863369,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01298245619982481,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.0,
+      "completions/max_terminated_length": 374.0,
+      "completions/mean_length": 294.78125,
+      "completions/mean_terminated_length": 294.78125,
+      "completions/min_length": 189.6,
+      "completions/min_terminated_length": 189.6,
+      "entropy": 0.35714133381843566,
+      "epoch": 0.0008,
+      "frac_reward_zero_std": 0.475,
+      "grad_norm": 0.14499153196811676,
+      "kl": 0.006699140788987279,
       "learning_rate": 1.137216e-06,
+      "loss": 0.0004814713727682829,
+      "num_tokens": 136090.0,
+      "reward": 0.292125004529953,
+      "reward_std": 0.2656953722238541,
+      "rewards/env_goofspiel_reward/mean": 0.292125004529953,
+      "rewards/env_goofspiel_reward/std": 0.41643730401992796,
+      "sampling/importance_sampling_ratio/max": 1.8895051956176758,
+      "sampling/importance_sampling_ratio/mean": 0.9195514798164368,
+      "sampling/importance_sampling_ratio/min": 0.2350650832056999,
+      "sampling/sampling_logp_difference/max": 1.6994480609893798,
+      "sampling/sampling_logp_difference/mean": 0.09322866201400756,
       "step": 5,
+      "step_time": 5.709857220399681
     },
     {
+      "clip_ratio/high_max": 0.02671568635851145,
+      "clip_ratio/high_mean": 0.013357843179255724,
+      "clip_ratio/low_mean": 0.016053921636193992,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.029411764815449715,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.2,
+      "completions/max_terminated_length": 374.2,
+      "completions/mean_length": 290.54375,
+      "completions/mean_terminated_length": 290.54375,
+      "completions/min_length": 194.4,
+      "completions/min_terminated_length": 194.4,
+      "entropy": 0.36814531981945037,
+      "epoch": 0.0016,
+      "frac_reward_zero_std": 0.55,
+      "grad_norm": 0.15409794449806213,
+      "kl": 0.024915735074318945,
       "learning_rate": 2.5587359999999995e-06,
+      "loss": 0.00025723695289343597,
+      "num_tokens": 271136.0,
+      "reward": 0.30362500846385954,
+      "reward_std": 0.21761211454868318,
+      "rewards/env_goofspiel_reward/mean": 0.30362500846385954,
+      "rewards/env_goofspiel_reward/std": 0.40538435578346255,
+      "sampling/importance_sampling_ratio/max": 2.3090060234069822,
+      "sampling/importance_sampling_ratio/mean": 0.9982686519622803,
+      "sampling/importance_sampling_ratio/min": 0.11310269832611083,
+      "sampling/sampling_logp_difference/max": 1.6586394786834717,
+      "sampling/sampling_logp_difference/mean": 0.09253094047307968,
       "step": 10,
+      "step_time": 5.373984831200687
     },
     {
+      "clip_ratio/high_max": 0.030514705926179886,
+      "clip_ratio/high_mean": 0.01672794120386243,
+      "clip_ratio/low_mean": 0.01482843142002821,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.03155637262389064,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 375.6,
+      "completions/max_terminated_length": 375.6,
+      "completions/mean_length": 283.45625,
+      "completions/mean_terminated_length": 283.45625,
+      "completions/min_length": 194.6,
+      "completions/min_terminated_length": 194.6,
+      "entropy": 0.38065551668405534,
+      "epoch": 0.0024,
+      "frac_reward_zero_std": 0.4375,
+      "grad_norm": 0.0815606489777565,
+      "kl": 0.02344995441380888,
       "learning_rate": 3.9802559999999995e-06,
+      "loss": 0.000516003929078579,
+      "num_tokens": 403825.0,
+      "reward": 0.35987500548362733,
+      "reward_std": 0.2653418242931366,
+      "rewards/env_goofspiel_reward/mean": 0.35987500548362733,
+      "rewards/env_goofspiel_reward/std": 0.4154684245586395,
+      "sampling/importance_sampling_ratio/max": 1.841845488548279,
+      "sampling/importance_sampling_ratio/mean": 0.9473352670669556,
+      "sampling/importance_sampling_ratio/min": 0.2071388103067875,
+      "sampling/sampling_logp_difference/max": 1.491676390171051,
+      "sampling/sampling_logp_difference/mean": 0.09024534374475479,
       "step": 15,
+      "step_time": 5.279587671000627
     },
     {
+      "clip_ratio/high_max": 0.023333333432674408,
+      "clip_ratio/high_mean": 0.011666666716337204,
+      "clip_ratio/low_mean": 0.012774122878909111,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.024440789688378574,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 373.8,
+      "completions/max_terminated_length": 373.8,
+      "completions/mean_length": 279.5875,
+      "completions/mean_terminated_length": 279.5875,
+      "completions/min_length": 206.8,
+      "completions/min_terminated_length": 206.8,
+      "entropy": 0.3506437622010708,
+      "epoch": 0.0032,
+      "frac_reward_zero_std": 0.4625,
+      "grad_norm": 0.12220246344804764,
+      "kl": 0.23682632837444545,
       "learning_rate": 5.401775999999999e-06,
+      "loss": -0.0002838193904608488,
+      "num_tokens": 535747.0,
+      "reward": 0.374812513589859,
+      "reward_std": 0.24421700537204744,
+      "rewards/env_goofspiel_reward/mean": 0.374812513589859,
+      "rewards/env_goofspiel_reward/std": 0.39649735689163207,
+      "sampling/importance_sampling_ratio/max": 2.3718762159347535,
+      "sampling/importance_sampling_ratio/mean": 0.9897154331207275,
+      "sampling/importance_sampling_ratio/min": 0.2213693767786026,
+      "sampling/sampling_logp_difference/max": 2.0126638174057008,
+      "sampling/sampling_logp_difference/mean": 0.10554229319095612,
       "step": 20,
+      "step_time": 5.332370807199913
     },
     {
+      "clip_ratio/high_max": 0.029443860985338688,
+      "clip_ratio/high_mean": 0.016110819298774004,
+      "clip_ratio/low_mean": 0.027831450570374727,
+      "clip_ratio/low_min": 0.011572128906846047,
+      "clip_ratio/region_mean": 0.04394227024167776,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.4,
+      "completions/max_terminated_length": 374.4,
+      "completions/mean_length": 301.0375,
+      "completions/mean_terminated_length": 301.0375,
+      "completions/min_length": 218.8,
+      "completions/min_terminated_length": 218.8,
+      "entropy": 0.3545067012310028,
+      "epoch": 0.004,
+      "frac_reward_zero_std": 0.35,
+      "grad_norm": 0.09991537779569626,
+      "kl": 0.6746378809213638,
       "learning_rate": 6.8232959999999994e-06,
+      "loss": -0.0003991848789155483,
+      "num_tokens": 673746.0,
+      "reward": 0.34875001609325407,
+      "reward_std": 0.3128947615623474,
+      "rewards/env_goofspiel_reward/mean": 0.34875001609325407,
+      "rewards/env_goofspiel_reward/std": 0.3975376784801483,
+      "sampling/importance_sampling_ratio/max": 2.3523300170898436,
+      "sampling/importance_sampling_ratio/mean": 0.9350853443145752,
+      "sampling/importance_sampling_ratio/min": 0.03211224116384983,
+      "sampling/sampling_logp_difference/max": 2.597071409225464,
+      "sampling/sampling_logp_difference/mean": 0.14846422374248505,
       "step": 25,
+      "step_time": 5.470841645199835
     },
     {
+      "clip_ratio/high_max": 0.015093954280018806,
+      "clip_ratio/high_mean": 0.007546977140009403,
+      "clip_ratio/low_mean": 0.01916505442932248,
+      "clip_ratio/low_min": 0.005625000037252903,
+      "clip_ratio/region_mean": 0.026712031569331884,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.0,
+      "completions/max_terminated_length": 374.0,
+      "completions/mean_length": 283.44375,
+      "completions/mean_terminated_length": 283.44375,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
+      "entropy": 0.3386394247412682,
+      "epoch": 0.0048,
+      "frac_reward_zero_std": 0.5625,
+      "grad_norm": 0.05208470672369003,
+      "kl": 3.1543088920414446,
       "learning_rate": 8.244816e-06,
+      "loss": 3.84216895326972e-05,
+      "num_tokens": 805457.0,
+      "reward": 0.41250001192092894,
+      "reward_std": 0.2121320277452469,
+      "rewards/env_goofspiel_reward/mean": 0.41250001192092894,
+      "rewards/env_goofspiel_reward/std": 0.39523468613624574,
+      "sampling/importance_sampling_ratio/max": 2.1884907484054565,
+      "sampling/importance_sampling_ratio/mean": 0.9641352295875549,
+      "sampling/importance_sampling_ratio/min": 0.17558300793170928,
+      "sampling/sampling_logp_difference/max": 1.910474991798401,
+      "sampling/sampling_logp_difference/mean": 0.11390596330165863,
       "step": 30,
+      "step_time": 5.248301958399679
     },
     {
+      "clip_ratio/high_max": 0.021911457646638155,
+      "clip_ratio/high_mean": 0.010955728823319077,
+      "clip_ratio/low_mean": 0.016263545025140047,
+      "clip_ratio/low_min": 0.002631578966975212,
+      "clip_ratio/region_mean": 0.02721927403472364,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 366.0,
+      "completions/max_terminated_length": 366.0,
+      "completions/mean_length": 290.6625,
+      "completions/mean_terminated_length": 290.6625,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
+      "entropy": 0.4240268304944038,
+      "epoch": 0.0056,
+      "frac_reward_zero_std": 0.4,
+      "grad_norm": 0.08057750761508942,
+      "kl": 1.8736468333750964,
       "learning_rate": 9.666336e-06,
+      "loss": -2.150831278413534e-05,
+      "num_tokens": 940033.0,
+      "reward": 0.4274375081062317,
+      "reward_std": 0.2758600294589996,
+      "rewards/env_goofspiel_reward/mean": 0.4274375081062317,
+      "rewards/env_goofspiel_reward/std": 0.4116846978664398,
+      "sampling/importance_sampling_ratio/max": 2.5006643772125243,
+      "sampling/importance_sampling_ratio/mean": 0.966198992729187,
+      "sampling/importance_sampling_ratio/min": 0.07496144040487707,
+      "sampling/sampling_logp_difference/max": 2.63707594871521,
+      "sampling/sampling_logp_difference/mean": 0.1406691253185272,
       "step": 35,
+      "step_time": 5.299385342999813
     },
     {
+      "clip_ratio/high_max": 0.020319487527012826,
+      "clip_ratio/high_mean": 0.010159743763506413,
+      "clip_ratio/low_mean": 0.007234477158635855,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.017394221015274526,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.2,
+      "completions/max_terminated_length": 374.2,
+      "completions/mean_length": 289.475,
+      "completions/mean_terminated_length": 289.475,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
+      "entropy": 0.6166644155979156,
+      "epoch": 0.0064,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.051475733518600464,
+      "kl": 0.6312531501054763,
+      "learning_rate": 9.95063915881342e-06,
+      "loss": 0.0008587016724050045,
+      "num_tokens": 1074989.0,
+      "reward": 0.2586875051259995,
+      "reward_std": 0.2599501311779022,
+      "rewards/env_goofspiel_reward/mean": 0.2586875051259995,
+      "rewards/env_goofspiel_reward/std": 0.38087824583053587,
+      "sampling/importance_sampling_ratio/max": 2.293054127693176,
+      "sampling/importance_sampling_ratio/mean": 0.9133782982826233,
+      "sampling/importance_sampling_ratio/min": 0.09586721286177635,
+      "sampling/sampling_logp_difference/max": 1.7645570278167724,
+      "sampling/sampling_logp_difference/mean": 0.1391677066683769,
       "step": 40,
+      "step_time": 5.214609204999943
     },
     {
+      "clip_ratio/high_max": 0.01441670972853899,
+      "clip_ratio/high_mean": 0.007208354864269495,
+      "clip_ratio/low_mean": 0.005737766716629266,
+      "clip_ratio/low_min": 0.002631578966975212,
+      "clip_ratio/region_mean": 0.012946121580898761,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.2,
+      "completions/max_terminated_length": 374.2,
+      "completions/mean_length": 294.13125,
+      "completions/mean_terminated_length": 294.13125,
+      "completions/min_length": 184.6,
+      "completions/min_terminated_length": 184.6,
+      "entropy": 0.7186032980680466,
+      "epoch": 0.0072,
+      "frac_reward_zero_std": 0.5125,
+      "grad_norm": 0.10908176004886627,
+      "kl": 0.9199723824858665,
+      "learning_rate": 9.950635741493589e-06,
+      "loss": 0.0010974571108818055,
+      "num_tokens": 1211700.0,
+      "reward": 0.20568750202655792,
+      "reward_std": 0.21823083460330964,
+      "rewards/env_goofspiel_reward/mean": 0.20568750202655792,
+      "rewards/env_goofspiel_reward/std": 0.3572053849697113,
+      "sampling/importance_sampling_ratio/max": 2.1740296363830565,
+      "sampling/importance_sampling_ratio/mean": 0.8587659239768982,
+      "sampling/importance_sampling_ratio/min": 0.19039739817380905,
+      "sampling/sampling_logp_difference/max": 1.302869963645935,
+      "sampling/sampling_logp_difference/mean": 0.159588959813118,
       "step": 45,
+      "step_time": 5.3400724014001755
     },
     {
+      "clip_ratio/high_max": 0.014580108411610126,
+      "clip_ratio/high_mean": 0.008760642446577548,
+      "clip_ratio/low_mean": 0.005718954280018807,
+      "clip_ratio/low_min": 0.002777777798473835,
+      "clip_ratio/region_mean": 0.014479596912860871,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.0,
+      "completions/max_terminated_length": 374.0,
+      "completions/mean_length": 298.4875,
+      "completions/mean_terminated_length": 298.4875,
+      "completions/min_length": 194.6,
+      "completions/min_terminated_length": 194.6,
+      "entropy": 0.7520321547985077,
+      "epoch": 0.008,
+      "frac_reward_zero_std": 0.55,
+      "grad_norm": 0.07008689641952515,
+      "kl": 1.4948164954781533,
+      "learning_rate": 9.950629695468755e-06,
+      "loss": 0.000722643407061696,
+      "num_tokens": 1348707.0,
+      "reward": 0.19093750715255736,
+      "reward_std": 0.18605746924877167,
+      "rewards/env_goofspiel_reward/mean": 0.19093750715255736,
+      "rewards/env_goofspiel_reward/std": 0.32617470622062683,
+      "sampling/importance_sampling_ratio/max": 2.441471576690674,
+      "sampling/importance_sampling_ratio/mean": 0.8815865159034729,
+      "sampling/importance_sampling_ratio/min": 0.047414033114910124,
+      "sampling/sampling_logp_difference/max": 1.4149149417877198,
+      "sampling/sampling_logp_difference/mean": 0.17731134295463563,
       "step": 50,
+      "step_time": 5.3226749666000615
     },
     {
+      "clip_ratio/high_max": 0.017320261523127555,
+      "clip_ratio/high_mean": 0.008660130761563778,
+      "clip_ratio/low_mean": 0.0015625,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.010222630761563777,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 373.6,
+      "completions/max_terminated_length": 373.6,
+      "completions/mean_length": 284.1125,
+      "completions/mean_terminated_length": 284.1125,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
+      "entropy": 0.7574372291564941,
+      "epoch": 0.0088,
+      "frac_reward_zero_std": 0.6125,
+      "grad_norm": 0.05677078291773796,
+      "kl": 0.9213189110159874,
+      "learning_rate": 9.950621020743173e-06,
+      "loss": 0.00019418969750404358,
+      "num_tokens": 1481168.0,
+      "reward": 0.16468750387430192,
+      "reward_std": 0.15954096913337706,
+      "rewards/env_goofspiel_reward/mean": 0.16468750387430192,
+      "rewards/env_goofspiel_reward/std": 0.3025706380605698,
+      "sampling/importance_sampling_ratio/max": 2.087395262718201,
+      "sampling/importance_sampling_ratio/mean": 0.9323906660079956,
+      "sampling/importance_sampling_ratio/min": 0.12477300018072128,
+      "sampling/sampling_logp_difference/max": 1.5357290506362915,
+      "sampling/sampling_logp_difference/mean": 0.16227305233478545,
       "step": 55,
+      "step_time": 5.239984228400317
     },
     {
+      "clip_ratio/high_max": 0.011312134563922882,
+      "clip_ratio/high_mean": 0.005656067281961441,
+      "clip_ratio/low_mean": 0.008540054224431515,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.014196121599525213,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.8,
+      "completions/max_terminated_length": 374.8,
+      "completions/mean_length": 294.125,
+      "completions/mean_terminated_length": 294.125,
+      "completions/min_length": 218.4,
+      "completions/min_terminated_length": 218.4,
+      "entropy": 0.7390435010194778,
+      "epoch": 0.0096,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.059227459132671356,
+      "kl": 0.8731714501976967,
+      "learning_rate": 9.950609717322956e-06,
+      "loss": 0.00016935726162046195,
+      "num_tokens": 1616969.0,
+      "reward": 0.22868750393390655,
+      "reward_std": 0.23873692452907563,
+      "rewards/env_goofspiel_reward/mean": 0.22868750393390655,
+      "rewards/env_goofspiel_reward/std": 0.3541231632232666,
+      "sampling/importance_sampling_ratio/max": 2.4580262422561647,
+      "sampling/importance_sampling_ratio/mean": 1.027009415626526,
+      "sampling/importance_sampling_ratio/min": 0.08994593024253845,
+      "sampling/sampling_logp_difference/max": 1.2262043237686158,
+      "sampling/sampling_logp_difference/mean": 0.15936054587364196,
+      "step": 60,
+      "step_time": 5.197383608799828
+    },
+    {
+      "clip_ratio/high_max": 0.003125,
+      "clip_ratio/high_mean": 0.0015625,
+      "clip_ratio/low_mean": 0.004093567281961441,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005656067281961441,
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 374.0,
       "completions/max_terminated_length": 374.0,
+      "completions/mean_length": 272.7125,
+      "completions/mean_terminated_length": 272.7125,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
+      "entropy": 0.641479243338108,
+      "epoch": 0.0104,
+      "frac_reward_zero_std": 0.5125,
+      "grad_norm": 0.052345700562000275,
+      "kl": 0.8409833669662475,
+      "learning_rate": 9.950595785216067e-06,
+      "loss": -0.0004354896955192089,
+      "num_tokens": 1745735.0,
+      "reward": 0.30350000858306886,
+      "reward_std": 0.24943190813064575,
+      "rewards/env_goofspiel_reward/mean": 0.30350000858306886,
+      "rewards/env_goofspiel_reward/std": 0.43262303471565244,
+      "sampling/importance_sampling_ratio/max": 2.308809924125671,
+      "sampling/importance_sampling_ratio/mean": 0.9729154109954834,
+      "sampling/importance_sampling_ratio/min": 0.2105877071619034,
+      "sampling/sampling_logp_difference/max": 1.216427493095398,
+      "sampling/sampling_logp_difference/mean": 0.14146182239055632,
+      "step": 65,
+      "step_time": 5.165435432399863
     },
     {
+      "clip_ratio/high_max": 0.008823529444634914,
+      "clip_ratio/high_mean": 0.004411764722317457,
+      "clip_ratio/low_mean": 0.004421977140009403,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00883374186232686,
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 374.0,
       "completions/max_terminated_length": 374.0,
+      "completions/mean_length": 279.70625,
+      "completions/mean_terminated_length": 279.70625,
+      "completions/min_length": 194.6,
+      "completions/min_terminated_length": 194.6,
+      "entropy": 0.5012152880430222,
+      "epoch": 0.0112,
+      "frac_reward_zero_std": 0.325,
+      "grad_norm": 0.08534521609544754,
+      "kl": 0.8132658362388611,
+      "learning_rate": 9.950579224432321e-06,
+      "loss": -0.0004163400735706091,
+      "num_tokens": 1877877.0,
+      "reward": 0.4497500121593475,
+      "reward_std": 0.32915821075439455,
+      "rewards/env_goofspiel_reward/mean": 0.4497500121593475,
+      "rewards/env_goofspiel_reward/std": 0.4289704501628876,
+      "sampling/importance_sampling_ratio/max": 2.469445323944092,
+      "sampling/importance_sampling_ratio/mean": 0.9713802933692932,
+      "sampling/importance_sampling_ratio/min": 0.1933848649263382,
+      "sampling/sampling_logp_difference/max": 1.2900412559509278,
+      "sampling/sampling_logp_difference/mean": 0.12238389104604722,
+      "step": 70,
+      "step_time": 5.296977608999805
+    },
+    {
+      "clip_ratio/high_max": 0.005409356765449047,
+      "clip_ratio/high_mean": 0.0027046783827245234,
+      "clip_ratio/low_mean": 0.011513618659228087,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01421829704195261,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.4,
+      "completions/max_terminated_length": 374.4,
+      "completions/mean_length": 292.43125,
+      "completions/mean_terminated_length": 292.43125,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
+      "entropy": 0.39477833807468415,
+      "epoch": 0.012,
+      "frac_reward_zero_std": 0.45,
+      "grad_norm": 0.08385973423719406,
+      "kl": 0.9824723288416862,
+      "learning_rate": 9.950560034983382e-06,
+      "loss": -0.0006953636649996043,
+      "num_tokens": 2012929.0,
+      "reward": 0.5398125350475311,
+      "reward_std": 0.2548236042261124,
+      "rewards/env_goofspiel_reward/mean": 0.5398125350475311,
+      "rewards/env_goofspiel_reward/std": 0.4067754566669464,
+      "sampling/importance_sampling_ratio/max": 2.476490020751953,
+      "sampling/importance_sampling_ratio/mean": 0.9818659067153931,
+      "sampling/importance_sampling_ratio/min": 0.04677087515592575,
+      "sampling/sampling_logp_difference/max": 1.8360216617584229,
+      "sampling/sampling_logp_difference/mean": 0.1339985266327858,
+      "step": 75,
+      "step_time": 5.25195668339984
+    },
+    {
+      "epoch": 0.012,
+      "eval_clip_ratio/high_max": 0.0,
+      "eval_clip_ratio/high_mean": 0.0,
+      "eval_clip_ratio/low_mean": 0.0,
+      "eval_clip_ratio/low_min": 0.0,
+      "eval_clip_ratio/region_mean": 0.0,
+      "eval_completions/clipped_ratio": 0.0,
+      "eval_completions/max_length": 310.0,
+      "eval_completions/max_terminated_length": 310.0,
+      "eval_completions/mean_length": 274.9,
+      "eval_completions/mean_terminated_length": 274.9,
+      "eval_completions/min_length": 241.2,
+      "eval_completions/min_terminated_length": 241.2,
+      "eval_entropy": 0.2987076103687286,
+      "eval_frac_reward_zero_std": 0.6,
+      "eval_kl": 0.8131496548652649,
+      "eval_loss": -0.0002675331197679043,
+      "eval_num_tokens": 2012929.0,
+      "eval_reward": 0.5700000166893006,
+      "eval_reward_std": 0.2121320366859436,
+      "eval_rewards/env_goofspiel_reward/mean": 0.5700000166893006,
+      "eval_rewards/env_goofspiel_reward/std": 0.3252412259578705,
+      "eval_runtime": 2.5856,
+      "eval_samples_per_second": 3.868,
+      "eval_sampling/importance_sampling_ratio/max": 1.6977968692779541,
+      "eval_sampling/importance_sampling_ratio/mean": 0.947873055934906,
+      "eval_sampling/importance_sampling_ratio/min": 0.2541299909353256,
+      "eval_sampling/sampling_logp_difference/max": 1.3066397070884705,
+      "eval_sampling/sampling_logp_difference/mean": 0.14956869557499886,
+      "eval_steps_per_second": 1.16,
+      "step": 75
     }
   ],
   "logging_steps": 5,
+  "max_steps": 18750,
+  "num_input_tokens_seen": 2012929,
   "num_train_epochs": 3,
   "save_steps": 500,
   "stateful_callbacks": {
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
   "total_flos": 0.0,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3300a89e75ebdeb9256a0a6e1d02dfcc2078fc90cd06fa1740dbf57e59261452
 size 7185

 version https://git-lfs.github.com/spec/v1
+oid sha256:906bc07f18d85f3fdbe47d01e60bbe6f967852d19caecc88d502ce07c5e4aa78
 size 7185