Instructions to use bimabk/environment_test_affine with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries
PEFT
How to use bimabk/environment_test_affine with PEFT:
```
Base model is not found.
```

How to use bimabk/environment_test_affine with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="bimabk/environment_test_affine")
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("bimabk/environment_test_affine")
model = AutoModelForCausalLM.from_pretrained("bimabk/environment_test_affine", device_map="auto")
messages = [
    {"role": "user", "content": "Who are you?"},
]
inputs = tokenizer.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=40)
print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use bimabk/environment_test_affine with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "bimabk/environment_test_affine"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test_affine",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/bimabk/environment_test_affine

SGLang

How to use bimabk/environment_test_affine with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "bimabk/environment_test_affine" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test_affine",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "bimabk/environment_test_affine" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "bimabk/environment_test_affine",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use bimabk/environment_test_affine with Docker Model Runner:
```
docker model run hf.co/bimabk/environment_test_affine
```

bimabk commited on Feb 9

Commit

eb8900c

verified ·

1 Parent(s): aab54a2

Upload task output 1

Browse files

Files changed (6) hide show

config.json +8 -20
generation_config.json +10 -0
loss.txt +1 -1
model.safetensors +2 -2
trainer_state.json +368 -236
training_args.bin +1 -1

config.json CHANGED Viewed

@@ -3,13 +3,13 @@
     "Qwen2ForCausalLM"
   ],
   "attention_dropout": 0.0,
-  "bos_token_id": 151643,
   "dtype": "bfloat16",
   "eos_token_id": 151645,
   "hidden_act": "silu",
-  "hidden_size": 2048,
   "initializer_range": 0.02,
-  "intermediate_size": 11008,
   "layer_types": [
     "full_attention",
     "full_attention",
@@ -34,27 +34,15 @@
     "full_attention",
     "full_attention",
     "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
     "full_attention"
   ],
   "max_position_embeddings": 32768,
-  "max_window_layers": 70,
   "model_type": "qwen2",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 36,
   "num_key_value_heads": 2,
-  "pad_token_id": null,
   "rms_norm_eps": 1e-06,
   "rope_parameters": {
     "rope_theta": 1000000.0,
@@ -63,7 +51,7 @@
   "sliding_window": null,
   "tie_word_embeddings": true,
   "transformers_version": "5.1.0",
-  "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

     "Qwen2ForCausalLM"
   ],
   "attention_dropout": 0.0,
+  "bos_token_id": null,
   "dtype": "bfloat16",
   "eos_token_id": 151645,
   "hidden_act": "silu",
+  "hidden_size": 896,
   "initializer_range": 0.02,
+  "intermediate_size": 4864,
   "layer_types": [
     "full_attention",
     "full_attention",
     "full_attention",
     "full_attention",
     "full_attention",
     "full_attention"
   ],
   "max_position_embeddings": 32768,
+  "max_window_layers": 21,
   "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
   "num_key_value_heads": 2,
+  "pad_token_id": 151643,
   "rms_norm_eps": 1e-06,
   "rope_parameters": {
     "rope_theta": 1000000.0,
   "sliding_window": null,
   "tie_word_embeddings": true,
   "transformers_version": "5.1.0",
+  "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

generation_config.json CHANGED Viewed

@@ -1,3 +1,13 @@
 {
   "transformers_version": "5.1.0"
 }

 {
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
   "transformers_version": "5.1.0"
 }

loss.txt CHANGED Viewed

	@@ -1 +1 @@
1	- 49,no_eval


1	+ 68,no_eval

model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84a818b6851ccc1ef74d75dd1041a284ff97e169342cff118f06248349f68f61
-size 6171927112

 version https://git-lfs.github.com/spec/v1
+oid sha256:6396161d7682c5b922070eaba3cbfd3497f329b60c83c004a56b1c82f0a44163
+size 988097824

trainer_state.json CHANGED Viewed

@@ -2,314 +2,446 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.007350735073507351,
   "eval_steps": 500,
-  "global_step": 49,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "clip_ratio/high_max": 0.012122603878378869,
-      "clip_ratio/high_mean": 0.009002478327602148,
-      "clip_ratio/low_mean": 0.009024427458643913,
-      "clip_ratio/low_min": 0.00597147960215807,
-      "clip_ratio/region_mean": 0.018026905506849288,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 374.2,
-      "completions/max_terminated_length": 374.2,
-      "completions/mean_length": 294.0400146484375,
-      "completions/mean_terminated_length": 294.0400146484375,
-      "completions/min_length": 201.8,
-      "completions/min_terminated_length": 201.8,
-      "entropy": 0.3647212088108063,
       "epoch": 0.00075007500750075,
-      "frac_reward_zero_std": 0.4400000274181366,
-      "grad_norm": 0.13166534900665283,
-      "kl": 0.007049627602100372,
       "learning_rate": 1.137216e-06,
-      "loss": 0.00026110392063856126,
-      "num_tokens": 127464.0,
-      "reward": 0.2516666889190674,
-      "reward_std": 0.24371615052223206,
-      "rewards/env_goofspiel_reward/mean": 0.25166667699813844,
-      "rewards/env_goofspiel_reward/std": 0.3760793387889862,
-      "sampling/importance_sampling_ratio/max": 2.2995895385742187,
-      "sampling/importance_sampling_ratio/mean": 0.9868913888931274,
-      "sampling/importance_sampling_ratio/min": 0.2817646384239197,
-      "sampling/sampling_logp_difference/max": 1.188546347618103,
-      "sampling/sampling_logp_difference/mean": 0.07895174026489257,
       "step": 5,
-      "step_time": 5.119130537400087
     },
     {
-      "clip_ratio/high_max": 0.03911839425563812,
-      "clip_ratio/high_mean": 0.02565017491579056,
-      "clip_ratio/low_mean": 0.023128375131636857,
-      "clip_ratio/low_min": 0.014993287436664105,
-      "clip_ratio/region_mean": 0.04877855032682419,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.8,
-      "completions/max_terminated_length": 373.8,
-      "completions/mean_length": 294.0533447265625,
-      "completions/mean_terminated_length": 294.0533447265625,
-      "completions/min_length": 206.8,
-      "completions/min_terminated_length": 206.8,
-      "entropy": 0.36778058409690856,
       "epoch": 0.0015001500150015,
-      "frac_reward_zero_std": 0.5466666936874389,
-      "grad_norm": 0.14173148572444916,
-      "kl": 0.05584366247057915,
       "learning_rate": 2.5587359999999995e-06,
-      "loss": 0.0008797919377684593,
-      "num_tokens": 254821.0,
-      "reward": 0.29593334794044496,
-      "reward_std": 0.22636846899986268,
-      "rewards/env_goofspiel_reward/mean": 0.295933336019516,
-      "rewards/env_goofspiel_reward/std": 0.37524943351745604,
-      "sampling/importance_sampling_ratio/max": 1.936799955368042,
-      "sampling/importance_sampling_ratio/mean": 0.9827824354171752,
-      "sampling/importance_sampling_ratio/min": 0.13101087883114815,
-      "sampling/sampling_logp_difference/max": 1.8672266960144044,
-      "sampling/sampling_logp_difference/mean": 0.0872830867767334,
       "step": 10,
-      "step_time": 4.894469680600014
     },
     {
-      "clip_ratio/high_max": 0.018904919549822808,
-      "clip_ratio/high_mean": 0.009452459774911404,
-      "clip_ratio/low_mean": 0.010854446236044169,
-      "clip_ratio/low_min": 0.00317460335791111,
-      "clip_ratio/region_mean": 0.020306906104087828,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.8,
-      "completions/max_terminated_length": 373.8,
-      "completions/mean_length": 289.2733520507812,
-      "completions/mean_terminated_length": 289.2733520507812,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
-      "entropy": 0.3574877142906189,
       "epoch": 0.0022502250225022503,
-      "frac_reward_zero_std": 0.5066666901111603,
-      "grad_norm": 0.1312570422887802,
-      "kl": 0.08231633007526398,
       "learning_rate": 3.9802559999999995e-06,
-      "loss": 5.0917407497763635e-05,
-      "num_tokens": 381014.0,
-      "reward": 0.33580002784729,
-      "reward_std": 0.249184450507164,
-      "rewards/env_goofspiel_reward/mean": 0.3358000159263611,
-      "rewards/env_goofspiel_reward/std": 0.3957887411117554,
-      "sampling/importance_sampling_ratio/max": 2.0427069664001465,
-      "sampling/importance_sampling_ratio/mean": 0.9270470976829529,
-      "sampling/importance_sampling_ratio/min": 0.1582688070833683,
-      "sampling/sampling_logp_difference/max": 2.064827084541321,
-      "sampling/sampling_logp_difference/mean": 0.08485963344573974,
       "step": 15,
-      "step_time": 4.857296707200021
     },
     {
-      "clip_ratio/high_max": 0.009682540223002434,
-      "clip_ratio/high_mean": 0.007896307110786437,
-      "clip_ratio/low_mean": 0.014026808738708495,
-      "clip_ratio/low_min": 0.006318408064544201,
-      "clip_ratio/region_mean": 0.021923115849494933,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.4,
-      "completions/max_terminated_length": 373.4,
-      "completions/mean_length": 273.9466796875,
-      "completions/mean_terminated_length": 273.9466796875,
-      "completions/min_length": 194.6,
-      "completions/min_terminated_length": 194.6,
-      "entropy": 0.36010334491729734,
       "epoch": 0.003000300030003,
-      "frac_reward_zero_std": 0.45333335399627683,
-      "grad_norm": 0.08147107064723969,
-      "kl": 0.4074278950691223,
       "learning_rate": 5.401775999999999e-06,
-      "loss": -0.0003812873270362616,
-      "num_tokens": 502616.0,
-      "reward": 0.3597333669662476,
-      "reward_std": 0.2605924427509308,
-      "rewards/env_goofspiel_reward/mean": 0.3597333550453186,
-      "rewards/env_goofspiel_reward/std": 0.41715014576911924,
-      "sampling/importance_sampling_ratio/max": 2.339983344078064,
-      "sampling/importance_sampling_ratio/mean": 0.977954626083374,
-      "sampling/importance_sampling_ratio/min": 0.17428167462348937,
-      "sampling/sampling_logp_difference/max": 2.17000892162323,
-      "sampling/sampling_logp_difference/mean": 0.10334575623273849,
       "step": 20,
-      "step_time": 4.810719416000029
     },
     {
-      "clip_ratio/high_max": 0.00952540971338749,
-      "clip_ratio/high_mean": 0.004762704856693745,
-      "clip_ratio/low_mean": 0.012625571712851525,
-      "clip_ratio/low_min": 0.009376526437699794,
-      "clip_ratio/region_mean": 0.017388276383280755,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 373.8,
-      "completions/max_terminated_length": 373.8,
-      "completions/mean_length": 291.360009765625,
-      "completions/mean_terminated_length": 291.360009765625,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
-      "entropy": 0.317078959941864,
       "epoch": 0.0037503750375037503,
-      "frac_reward_zero_std": 0.45333335399627683,
-      "grad_norm": 0.27843180298805237,
-      "kl": 0.6531880825757981,
       "learning_rate": 6.8232959999999994e-06,
-      "loss": -0.0005337921902537346,
-      "num_tokens": 629159.0,
-      "reward": 0.4559333860874176,
-      "reward_std": 0.2489958882331848,
-      "rewards/env_goofspiel_reward/mean": 0.4559333622455597,
-      "rewards/env_goofspiel_reward/std": 0.37337183952331543,
-      "sampling/importance_sampling_ratio/max": 2.3218794584274294,
-      "sampling/importance_sampling_ratio/mean": 0.9727706193923951,
-      "sampling/importance_sampling_ratio/min": 0.03638465753756463,
-      "sampling/sampling_logp_difference/max": 2.5760634660720827,
-      "sampling/sampling_logp_difference/mean": 0.11733063012361526,
       "step": 25,
-      "step_time": 4.816599215199949
     },
     {
-      "clip_ratio/high_max": 0.006507936865091324,
-      "clip_ratio/high_mean": 0.003253968432545662,
-      "clip_ratio/low_mean": 0.027271623164415358,
-      "clip_ratio/low_min": 0.015977022424340247,
-      "clip_ratio/region_mean": 0.03052559234201908,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 374.4,
-      "completions/max_terminated_length": 374.4,
-      "completions/mean_length": 289.25335083007815,
-      "completions/mean_terminated_length": 289.25335083007815,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
-      "entropy": 0.33378886580467226,
       "epoch": 0.004500450045004501,
-      "frac_reward_zero_std": 0.3600000202655792,
-      "grad_norm": 0.1027369573712349,
-      "kl": 1.2258114516735077,
       "learning_rate": 8.244816e-06,
-      "loss": -0.0006984985433518886,
-      "num_tokens": 754860.0,
-      "reward": 0.4718667149543762,
-      "reward_std": 0.3165953040122986,
-      "rewards/env_goofspiel_reward/mean": 0.4718666851520538,
-      "rewards/env_goofspiel_reward/std": 0.41961100697517395,
-      "sampling/importance_sampling_ratio/max": 2.5107172966003417,
-      "sampling/importance_sampling_ratio/mean": 0.9468889474868775,
-      "sampling/importance_sampling_ratio/min": 0.049952364061027765,
-      "sampling/sampling_logp_difference/max": 2.5360853910446166,
-      "sampling/sampling_logp_difference/mean": 0.14082961529493332,
       "step": 30,
-      "step_time": 4.918840783200176
     },
     {
-      "clip_ratio/high_max": 0.012909946218132972,
-      "clip_ratio/high_mean": 0.00809431727975607,
-      "clip_ratio/low_mean": 0.009740076586604118,
-      "clip_ratio/low_min": 0.006559139862656593,
-      "clip_ratio/region_mean": 0.017834394052624703,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 374.0,
-      "completions/max_terminated_length": 374.0,
-      "completions/mean_length": 294.533349609375,
-      "completions/mean_terminated_length": 294.533349609375,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
-      "entropy": 0.4855224847793579,
       "epoch": 0.005250525052505251,
-      "frac_reward_zero_std": 0.5466666936874389,
-      "grad_norm": 0.1376088261604309,
-      "kl": 0.6020630955696106,
       "learning_rate": 9.666336e-06,
-      "loss": -2.6161287678405643e-05,
-      "num_tokens": 881908.0,
-      "reward": 0.3080000221729279,
-      "reward_std": 0.22061732709407805,
-      "rewards/env_goofspiel_reward/mean": 0.30800001621246337,
-      "rewards/env_goofspiel_reward/std": 0.4081439733505249,
-      "sampling/importance_sampling_ratio/max": 2.0464017391204834,
-      "sampling/importance_sampling_ratio/mean": 0.8832055687904358,
-      "sampling/importance_sampling_ratio/min": 0.08903740048408508,
-      "sampling/sampling_logp_difference/max": 1.4421878337860108,
-      "sampling/sampling_logp_difference/mean": 0.1307734727859497,
       "step": 35,
-      "step_time": 4.927365056799954
     },
     {
-      "clip_ratio/high_max": 0.019359875097870828,
-      "clip_ratio/high_mean": 0.012933905981481076,
-      "clip_ratio/low_mean": 0.009454933740198613,
-      "clip_ratio/low_min": 0.0032258063554763796,
-      "clip_ratio/region_mean": 0.022388840094208717,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 365.4,
-      "completions/max_terminated_length": 365.4,
-      "completions/mean_length": 281.6333435058594,
-      "completions/mean_terminated_length": 281.6333435058594,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
-      "entropy": 0.49551063776016235,
       "epoch": 0.006000600060006,
-      "frac_reward_zero_std": 0.3866666853427887,
-      "grad_norm": 0.08914674818515778,
-      "kl": 0.8902900576591491,
       "learning_rate": 9.950639260700543e-06,
-      "loss": 0.0002991344779729843,
-      "num_tokens": 1005647.0,
-      "reward": 0.359933340549469,
-      "reward_std": 0.3053758591413498,
-      "rewards/env_goofspiel_reward/mean": 0.3599333316087723,
-      "rewards/env_goofspiel_reward/std": 0.40515289306640623,
-      "sampling/importance_sampling_ratio/max": 2.4499823093414306,
-      "sampling/importance_sampling_ratio/mean": 0.882733416557312,
-      "sampling/importance_sampling_ratio/min": 0.11329675018787384,
-      "sampling/sampling_logp_difference/max": 1.781545352935791,
-      "sampling/sampling_logp_difference/mean": 0.14177367389202117,
       "step": 40,
-      "step_time": 4.763557598199805
     },
     {
-      "clip_ratio/high_max": 0.013015710189938546,
-      "clip_ratio/high_mean": 0.008120758086442947,
-      "clip_ratio/low_mean": 0.004791666753590107,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.01291242502629757,
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 374.0,
       "completions/max_terminated_length": 374.0,
-      "completions/mean_length": 303.2200134277344,
-      "completions/mean_terminated_length": 303.2200134277344,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
-      "entropy": 0.5108859002590179,
       "epoch": 0.0067506750675067504,
-      "frac_reward_zero_std": 0.4266666889190674,
-      "grad_norm": 0.058294642716646194,
-      "kl": 0.37123287916183473,
       "learning_rate": 9.950636257297004e-06,
-      "loss": 6.670470465905965e-05,
-      "num_tokens": 1136078.0,
-      "reward": 0.32793336510658266,
-      "reward_std": 0.27162329852581024,
-      "rewards/env_goofspiel_reward/mean": 0.3279333531856537,
-      "rewards/env_goofspiel_reward/std": 0.40487490892410277,
-      "sampling/importance_sampling_ratio/max": 2.1823894500732424,
-      "sampling/importance_sampling_ratio/mean": 0.9206063866615295,
-      "sampling/importance_sampling_ratio/min": 0.2219469040632248,
-      "sampling/sampling_logp_difference/max": 1.034794545173645,
-      "sampling/sampling_logp_difference/mean": 0.10992220342159272,
       "step": 45,
-      "step_time": 4.865905853399909
     }
   ],
   "logging_steps": 5,
   "max_steps": 19998,
-  "num_input_tokens_seen": 1238614,
   "num_train_epochs": 3,
   "save_steps": 500,
   "stateful_callbacks": {
@@ -325,7 +457,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 15,
   "trial_name": null,
   "trial_params": null
 }

   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
+  "epoch": 0.0102010201020102,
   "eval_steps": 500,
+  "global_step": 68,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.4,
+      "completions/max_terminated_length": 374.4,
+      "completions/mean_length": 291.50001220703126,
+      "completions/mean_terminated_length": 291.50001220703126,
+      "completions/min_length": 174.8,
+      "completions/min_terminated_length": 174.8,
+      "entropy": 0.737431001663208,
       "epoch": 0.00075007500750075,
+      "frac_reward_zero_std": 0.6533333778381347,
+      "grad_norm": 0.369140625,
+      "kl": 0.00885026976466179,
       "learning_rate": 1.137216e-06,
+      "loss": 0.00034544954542070627,
+      "num_tokens": 126703.0,
+      "reward": 0.0383333370089531,
+      "reward_std": 0.05835988484323025,
+      "rewards/env_goofspiel_reward/mean": 0.03833333514630795,
+      "rewards/env_goofspiel_reward/std": 0.15159520953893663,
+      "sampling/importance_sampling_ratio/max": 1.7607571125030517,
+      "sampling/importance_sampling_ratio/mean": 0.9916059970855713,
+      "sampling/importance_sampling_ratio/min": 0.5116463124752044,
+      "sampling/sampling_logp_difference/max": 0.6887424349784851,
+      "sampling/sampling_logp_difference/mean": 0.06558751240372658,
       "step": 5,
+      "step_time": 3.541577606199735
     },
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 373.6,
+      "completions/max_terminated_length": 373.6,
+      "completions/mean_length": 292.0866760253906,
+      "completions/mean_terminated_length": 292.0866760253906,
+      "completions/min_length": 179.8,
+      "completions/min_terminated_length": 179.8,
+      "entropy": 0.7154561956723531,
       "epoch": 0.0015001500150015,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.462890625,
+      "kl": 0.009678552253171801,
       "learning_rate": 2.5587359999999995e-06,
+      "loss": -3.925009514205158e-05,
+      "num_tokens": 254525.0,
+      "reward": 0.03886667159385979,
+      "reward_std": 0.05817132312804461,
+      "rewards/env_goofspiel_reward/mean": 0.03886666861362755,
+      "rewards/env_goofspiel_reward/std": 0.14201362472958862,
+      "sampling/importance_sampling_ratio/max": 1.9776098012924195,
+      "sampling/importance_sampling_ratio/mean": 1.0280600190162659,
+      "sampling/importance_sampling_ratio/min": 0.4622887670993805,
+      "sampling/sampling_logp_difference/max": 0.8463176250457763,
+      "sampling/sampling_logp_difference/mean": 0.06613281443715095,
       "step": 10,
+      "step_time": 3.1668822380001074
     },
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 373.6,
+      "completions/max_terminated_length": 373.6,
+      "completions/mean_length": 288.72001953125,
+      "completions/mean_terminated_length": 288.72001953125,
+      "completions/min_length": 187.2,
+      "completions/min_terminated_length": 187.2,
+      "entropy": 0.6402347763379415,
       "epoch": 0.0022502250225022503,
+      "frac_reward_zero_std": 0.8000000476837158,
+      "grad_norm": 0.412109375,
+      "kl": 0.00951577623685201,
       "learning_rate": 3.9802559999999995e-06,
+      "loss": 0.00047838701866567137,
+      "num_tokens": 380635.0,
+      "reward": 0.07146667279303073,
+      "reward_std": 0.09088679552078247,
+      "rewards/env_goofspiel_reward/mean": 0.07146666683256626,
+      "rewards/env_goofspiel_reward/std": 0.22424434274435043,
+      "sampling/importance_sampling_ratio/max": 1.5242333889007569,
+      "sampling/importance_sampling_ratio/mean": 0.9860278725624084,
+      "sampling/importance_sampling_ratio/min": 0.5668014168739319,
+      "sampling/sampling_logp_difference/max": 0.5752804994583129,
+      "sampling/sampling_logp_difference/mean": 0.05473804771900177,
       "step": 15,
+      "step_time": 3.1726541418000123
     },
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 373.2,
+      "completions/max_terminated_length": 373.2,
+      "completions/mean_length": 273.82001037597655,
+      "completions/mean_terminated_length": 273.82001037597655,
+      "completions/min_length": 200.2,
+      "completions/min_terminated_length": 200.2,
+      "entropy": 0.5869409064451854,
       "epoch": 0.003000300030003,
+      "frac_reward_zero_std": 0.7600000500679016,
+      "grad_norm": 0.158203125,
+      "kl": 0.029618356159577766,
       "learning_rate": 5.401775999999999e-06,
+      "loss": 0.00020813762675970794,
+      "num_tokens": 501743.0,
+      "reward": 0.06360000669956208,
+      "reward_std": 0.07976165302097797,
+      "rewards/env_goofspiel_reward/mean": 0.06360000558197498,
+      "rewards/env_goofspiel_reward/std": 0.20161283165216445,
+      "sampling/importance_sampling_ratio/max": 1.7165520668029786,
+      "sampling/importance_sampling_ratio/mean": 0.9985299944877625,
+      "sampling/importance_sampling_ratio/min": 0.6464852690696716,
+      "sampling/sampling_logp_difference/max": 0.6010382175445557,
+      "sampling/sampling_logp_difference/mean": 0.05117494091391563,
       "step": 20,
+      "step_time": 3.1184157538001274
     },
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.2,
+      "completions/max_terminated_length": 374.2,
+      "completions/mean_length": 291.16668090820315,
+      "completions/mean_terminated_length": 291.16668090820315,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
+      "entropy": 0.5825250327587128,
       "epoch": 0.0037503750375037503,
+      "frac_reward_zero_std": 0.8133333921432495,
+      "grad_norm": 0.30859375,
+      "kl": 0.03789825042088826,
       "learning_rate": 6.8232959999999994e-06,
+      "loss": -5.795806646347046e-05,
+      "num_tokens": 629207.0,
+      "reward": 0.05593333579599857,
+      "reward_std": 0.0791016798466444,
+      "rewards/env_goofspiel_reward/mean": 0.055933335050940516,
+      "rewards/env_goofspiel_reward/std": 0.16461323350667953,
+      "sampling/importance_sampling_ratio/max": 1.6597614526748656,
+      "sampling/importance_sampling_ratio/mean": 0.9899526715278626,
+      "sampling/importance_sampling_ratio/min": 0.5461978197097779,
+      "sampling/sampling_logp_difference/max": 0.5474630713462829,
+      "sampling/sampling_logp_difference/mean": 0.056329603493213656,
       "step": 25,
+      "step_time": 3.1455567411999255
     },
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 373.8,
+      "completions/max_terminated_length": 373.8,
+      "completions/mean_length": 288.2733459472656,
+      "completions/mean_terminated_length": 288.2733459472656,
+      "completions/min_length": 205.4,
+      "completions/min_terminated_length": 205.4,
+      "entropy": 0.6367802540461223,
       "epoch": 0.004500450045004501,
+      "frac_reward_zero_std": 0.8133333921432495,
+      "grad_norm": 0.3203125,
+      "kl": 0.15099084513882796,
       "learning_rate": 8.244816e-06,
+      "loss": 2.9896479099988936e-05,
+      "num_tokens": 755141.0,
+      "reward": 0.04773333668708801,
+      "reward_std": 0.056945668533444405,
+      "rewards/env_goofspiel_reward/mean": 0.04773333407938481,
+      "rewards/env_goofspiel_reward/std": 0.1535952940583229,
+      "sampling/importance_sampling_ratio/max": 1.4472879409790038,
+      "sampling/importance_sampling_ratio/mean": 1.000359261035919,
+      "sampling/importance_sampling_ratio/min": 0.6299581527709961,
+      "sampling/sampling_logp_difference/max": 0.49155421257019044,
+      "sampling/sampling_logp_difference/mean": 0.04638729840517044,
       "step": 30,
+      "step_time": 3.104708982399825
     },
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 373.4,
+      "completions/max_terminated_length": 373.4,
+      "completions/mean_length": 294.12001953125,
+      "completions/mean_terminated_length": 294.12001953125,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
+      "entropy": 0.6307838877042135,
       "epoch": 0.005250525052505251,
+      "frac_reward_zero_std": 0.8133333921432495,
+      "grad_norm": 0.2041015625,
+      "kl": 0.21180881708860397,
       "learning_rate": 9.666336e-06,
+      "loss": 0.0003116762964054942,
+      "num_tokens": 881937.0,
+      "reward": 0.04366667197318748,
+      "reward_std": 0.0626968042459339,
+      "rewards/env_goofspiel_reward/mean": 0.04366666899295524,
+      "rewards/env_goofspiel_reward/std": 0.14476882207673042,
+      "sampling/importance_sampling_ratio/max": 1.540164875984192,
+      "sampling/importance_sampling_ratio/mean": 1.007632350921631,
+      "sampling/importance_sampling_ratio/min": 0.6116935849189759,
+      "sampling/sampling_logp_difference/max": 0.5826021313667298,
+      "sampling/sampling_logp_difference/mean": 0.054141230136156085,
       "step": 35,
+      "step_time": 3.152498085000025
     },
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
+      "completions/max_length": 365.8,
+      "completions/max_terminated_length": 365.8,
+      "completions/mean_length": 281.62001953125,
+      "completions/mean_terminated_length": 281.62001953125,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
+      "entropy": 0.553737320502599,
       "epoch": 0.006000600060006,
+      "frac_reward_zero_std": 0.8266667246818542,
+      "grad_norm": 0.0888671875,
+      "kl": 0.5656270523866017,
       "learning_rate": 9.950639260700543e-06,
+      "loss": 0.00017667778301984073,
+      "num_tokens": 1005294.0,
+      "reward": 0.056000004336237905,
+      "reward_std": 0.07919596247375012,
+      "rewards/env_goofspiel_reward/mean": 0.05600000284612179,
+      "rewards/env_goofspiel_reward/std": 0.18038542717695236,
+      "sampling/importance_sampling_ratio/max": 1.5142346620559692,
+      "sampling/importance_sampling_ratio/mean": 0.9889694690704346,
+      "sampling/importance_sampling_ratio/min": 0.6695701956748963,
+      "sampling/sampling_logp_difference/max": 0.45929052233695983,
+      "sampling/sampling_logp_difference/mean": 0.05039779171347618,
       "step": 40,
+      "step_time": 3.061584199799836
     },
     {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 374.0,
       "completions/max_terminated_length": 374.0,
+      "completions/mean_length": 303.58001708984375,
+      "completions/mean_terminated_length": 303.58001708984375,
       "completions/min_length": 212.0,
       "completions/min_terminated_length": 212.0,
+      "entropy": 0.48466593225797017,
       "epoch": 0.0067506750675067504,
+      "frac_reward_zero_std": 0.8266667246818542,
+      "grad_norm": 0.1748046875,
+      "kl": 1.0889011422793071,
       "learning_rate": 9.950636257297004e-06,
+      "loss": 0.00040990984998643396,
+      "num_tokens": 1135969.0,
+      "reward": 0.04786667115986347,
+      "reward_std": 0.06807081587612629,
+      "rewards/env_goofspiel_reward/mean": 0.047866668179631235,
+      "rewards/env_goofspiel_reward/std": 0.1721431568264961,
+      "sampling/importance_sampling_ratio/max": 1.4374094486236573,
+      "sampling/importance_sampling_ratio/mean": 0.9694242954254151,
+      "sampling/importance_sampling_ratio/min": 0.6029698491096497,
+      "sampling/sampling_logp_difference/max": 0.5160395622253418,
+      "sampling/sampling_logp_difference/mean": 0.04300674088299274,
       "step": 45,
+      "step_time": 3.19344236120005
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 375.6,
+      "completions/max_terminated_length": 375.6,
+      "completions/mean_length": 296.2133483886719,
+      "completions/mean_terminated_length": 296.2133483886719,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
+      "entropy": 0.3623567740122477,
+      "epoch": 0.007500750075007501,
+      "frac_reward_zero_std": 0.8800000429153443,
+      "grad_norm": 0.134765625,
+      "kl": 0.9536839803059896,
+      "learning_rate": 9.950630943585028e-06,
+      "loss": 0.00046116397716104983,
+      "num_tokens": 1262891.0,
+      "reward": 0.040000003576278684,
+      "reward_std": 0.05656854659318924,
+      "rewards/env_goofspiel_reward/mean": 0.04000000208616257,
+      "rewards/env_goofspiel_reward/std": 0.13237088024616242,
+      "sampling/importance_sampling_ratio/max": 1.4614337921142577,
+      "sampling/importance_sampling_ratio/mean": 0.9917921781539917,
+      "sampling/importance_sampling_ratio/min": 0.7945461988449096,
+      "sampling/sampling_logp_difference/max": 0.34459147453308103,
+      "sampling/sampling_logp_difference/mean": 0.023883017525076867,
+      "step": 50,
+      "step_time": 3.080125956599841
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 373.8,
+      "completions/max_terminated_length": 373.8,
+      "completions/mean_length": 297.3133483886719,
+      "completions/mean_terminated_length": 297.3133483886719,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
+      "entropy": 0.412084698677063,
+      "epoch": 0.00825082508250825,
+      "frac_reward_zero_std": 0.840000057220459,
+      "grad_norm": 0.11279296875,
+      "kl": 0.8989204486211141,
+      "learning_rate": 9.950623319567901e-06,
+      "loss": 0.00017252418911084533,
+      "num_tokens": 1391274.0,
+      "reward": 0.059933338314294815,
+      "reward_std": 0.07363338991999627,
+      "rewards/env_goofspiel_reward/mean": 0.059933335334062574,
+      "rewards/env_goofspiel_reward/std": 0.19352754354476928,
+      "sampling/importance_sampling_ratio/max": 1.510833191871643,
+      "sampling/importance_sampling_ratio/mean": 1.0025275230407715,
+      "sampling/importance_sampling_ratio/min": 0.7753824949264526,
+      "sampling/sampling_logp_difference/max": 0.37447161674499513,
+      "sampling/sampling_logp_difference/mean": 0.025511124357581138,
+      "step": 55,
+      "step_time": 3.1612697836000736
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.0,
+      "completions/max_terminated_length": 374.0,
+      "completions/mean_length": 293.233349609375,
+      "completions/mean_terminated_length": 293.233349609375,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
+      "entropy": 0.3592885712782542,
+      "epoch": 0.009000900090009001,
+      "frac_reward_zero_std": 0.8533333778381348,
+      "grad_norm": 0.0164794921875,
+      "kl": 0.9290923396746318,
+      "learning_rate": 9.950613385250344e-06,
+      "loss": 0.00017089147586375476,
+      "num_tokens": 1517875.0,
+      "reward": 0.06400000602006913,
+      "reward_std": 0.07919596470892429,
+      "rewards/env_goofspiel_reward/mean": 0.06400000229477883,
+      "rewards/env_goofspiel_reward/std": 0.192934051156044,
+      "sampling/importance_sampling_ratio/max": 1.3080760478973388,
+      "sampling/importance_sampling_ratio/mean": 1.00157231092453,
+      "sampling/importance_sampling_ratio/min": 0.809451448917389,
+      "sampling/sampling_logp_difference/max": 0.3210816144943237,
+      "sampling/sampling_logp_difference/mean": 0.021189583651721477,
+      "step": 60,
+      "step_time": 3.1453409107998596
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 374.0,
+      "completions/max_terminated_length": 374.0,
+      "completions/mean_length": 280.706689453125,
+      "completions/mean_terminated_length": 280.706689453125,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
+      "entropy": 0.46913262009620665,
+      "epoch": 0.00975097509750975,
+      "frac_reward_zero_std": 0.840000057220459,
+      "grad_norm": 0.04541015625,
+      "kl": 1.9159570078055064,
+      "learning_rate": 9.95060114063851e-06,
+      "loss": 0.00043630152940750123,
+      "num_tokens": 1640903.0,
+      "reward": 0.07593334019184113,
+      "reward_std": 0.07363339141011238,
+      "rewards/env_goofspiel_reward/mean": 0.07593333572149277,
+      "rewards/env_goofspiel_reward/std": 0.22238647043704987,
+      "sampling/importance_sampling_ratio/max": 1.42401442527771,
+      "sampling/importance_sampling_ratio/mean": 1.001141333580017,
+      "sampling/importance_sampling_ratio/min": 0.7154132127761841,
+      "sampling/sampling_logp_difference/max": 0.3389713287353516,
+      "sampling/sampling_logp_difference/mean": 0.026769372820854186,
+      "step": 65,
+      "step_time": 3.0763218426000094
     }
   ],
   "logging_steps": 5,
   "max_steps": 19998,
+  "num_input_tokens_seen": 1713916,
   "num_train_epochs": 3,
   "save_steps": 500,
   "stateful_callbacks": {
     }
   },
   "total_flos": 0.0,
+  "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac9929cb2fba579bddba18a776b644d1599eb6965f5f63006443dc3723b476a2
 size 7185

 version https://git-lfs.github.com/spec/v1
+oid sha256:3300a89e75ebdeb9256a0a6e1d02dfcc2078fc90cd06fa1740dbf57e59261452
 size 7185