Hugging Face's logo

priyaaaaaasharmaaaaa
/
trial1

Reinforcement Learning
Safetensors
openenv
agile
sprint-management
fastapi
gradio
Model card Files
xet
 Community
1
priyaaaaaasharmaaaaa commited on
Commit
30af6e8
·
verified ·
1 Parent(s): 35e9f20

Upload trained LoRA adapter from Kaggle

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -1
  2. results/phase1/README.md +1 -0
  3. results/phase1/adapter_config.json +4 -4
  4. results/phase1/adapter_model.safetensors +1 -1
  5. results/phase1/checkpoint-100/README.md +1 -0
  6. results/phase1/checkpoint-100/adapter_config.json +4 -4
  7. results/phase1/checkpoint-100/adapter_model.safetensors +1 -1
  8. results/phase1/checkpoint-100/optimizer.pt +1 -1
  9. results/phase1/checkpoint-100/rng_state.pth +1 -1
  10. results/phase1/checkpoint-100/scheduler.pt +1 -1
  11. results/phase1/checkpoint-100/trainer_state.json +347 -347
  12. results/phase1/checkpoint-150/README.md +1 -0
  13. results/phase1/checkpoint-150/adapter_config.json +4 -4
  14. results/phase1/checkpoint-150/adapter_model.safetensors +1 -1
  15. results/phase1/checkpoint-150/optimizer.pt +1 -1
  16. results/phase1/checkpoint-150/rng_state.pth +1 -1
  17. results/phase1/checkpoint-150/scheduler.pt +1 -1
  18. results/phase1/checkpoint-150/trainer_state.json +523 -523
  19. results/phase1/checkpoint-200/README.md +1 -0
  20. results/phase1/checkpoint-200/adapter_config.json +4 -4
  21. results/phase1/checkpoint-200/adapter_model.safetensors +1 -1
  22. results/phase1/checkpoint-200/optimizer.pt +1 -1
  23. results/phase1/checkpoint-200/rng_state.pth +1 -1
  24. results/phase1/checkpoint-200/scheduler.pt +1 -1
  25. results/phase1/checkpoint-200/trainer_state.json +695 -695
  26. results/phase1/checkpoint-225/README.md +211 -0
  27. results/phase1/checkpoint-225/adapter_config.json +50 -0
  28. results/phase1/checkpoint-225/adapter_model.safetensors +3 -0
  29. results/phase1/checkpoint-225/chat_template.jinja +54 -0
  30. results/phase1/checkpoint-225/optimizer.pt +3 -0
  31. results/phase1/checkpoint-225/rng_state.pth +3 -0
  32. results/phase1/checkpoint-225/scaler.pt +3 -0
  33. results/phase1/checkpoint-225/scheduler.pt +3 -0
  34. results/phase1/checkpoint-225/tokenizer.json +3 -0
  35. results/phase1/checkpoint-225/tokenizer_config.json +201 -0
  36. results/phase1/checkpoint-225/trainer_state.json +1249 -0
  37. results/phase1/checkpoint-225/training_args.bin +3 -0
  38. results/phase1/checkpoint-50/README.md +1 -0
  39. results/phase1/checkpoint-50/adapter_config.json +4 -4
  40. results/phase1/checkpoint-50/adapter_model.safetensors +1 -1
  41. results/phase1/checkpoint-50/optimizer.pt +1 -1
  42. results/phase1/checkpoint-50/rng_state.pth +1 -1
  43. results/phase1/checkpoint-50/scheduler.pt +1 -1
  44. results/phase1/checkpoint-50/trainer_state.json +173 -173
  45. results/phase1/sft_warmup/README.md +59 -0
  46. results/phase1/sft_warmup/checkpoint-48/README.md +210 -0
  47. results/phase1/sft_warmup/checkpoint-48/adapter_config.json +50 -0
  48. results/phase1/sft_warmup/checkpoint-48/adapter_model.safetensors +3 -0
  49. results/phase1/sft_warmup/checkpoint-48/chat_template.jinja +54 -0
  50. results/phase1/sft_warmup/checkpoint-48/optimizer.pt +3 -0
.gitattributes CHANGED
@@ -40,6 +40,7 @@ huggingface_tokenizers_cache/models--unsloth--qwen2.5-1.5b-instruct-unsloth-bnb-
40
  results/phase1/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
41
  results/phase1/checkpoint-150/tokenizer.json filter=lfs diff=lfs merge=lfs -text
42
  results/phase1/checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
43
- results/phase1/checkpoint-222/tokenizer.json filter=lfs diff=lfs merge=lfs -text
44
  results/phase1/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
45
  results/phase1/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
40
  results/phase1/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
41
  results/phase1/checkpoint-150/tokenizer.json filter=lfs diff=lfs merge=lfs -text
42
  results/phase1/checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
43
+ results/phase1/checkpoint-225/tokenizer.json filter=lfs diff=lfs merge=lfs -text
44
  results/phase1/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text
45
+ results/phase1/sft_warmup/checkpoint-48/tokenizer.json filter=lfs diff=lfs merge=lfs -text
46
  results/phase1/tokenizer.json filter=lfs diff=lfs merge=lfs -text
results/phase1/README.md CHANGED
@@ -6,6 +6,7 @@ tags:
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
 
9
  - transformers
10
  - trl
11
  - unsloth
 
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
9
+ - sft
10
  - transformers
11
  - trl
12
  - unsloth
results/phase1/adapter_config.json CHANGED
@@ -33,13 +33,13 @@
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
 
 
36
  "k_proj",
37
- "up_proj",
38
  "down_proj",
39
  "o_proj",
40
- "v_proj",
41
- "gate_proj",
42
- "q_proj"
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
 
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
36
+ "gate_proj",
37
+ "q_proj",
38
  "k_proj",
39
+ "v_proj",
40
  "down_proj",
41
  "o_proj",
42
+ "up_proj"
 
 
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
results/phase1/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6d9ce380266a065d31b8d4962851f4d105daeda1ef56d4e9cd0835e5d5d0644a
3
  size 73911112
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe1beb57ab89a2aeb9d572e64c9dedcaba34759e2e289c74a9a5464cb62a56c2
3
  size 73911112
results/phase1/checkpoint-100/README.md CHANGED
@@ -6,6 +6,7 @@ tags:
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
 
9
  - transformers
10
  - trl
11
  - unsloth
 
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
9
+ - sft
10
  - transformers
11
  - trl
12
  - unsloth
results/phase1/checkpoint-100/adapter_config.json CHANGED
@@ -33,13 +33,13 @@
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
 
 
36
  "k_proj",
37
- "up_proj",
38
  "down_proj",
39
  "o_proj",
40
- "v_proj",
41
- "gate_proj",
42
- "q_proj"
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
 
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
36
+ "gate_proj",
37
+ "q_proj",
38
  "k_proj",
39
+ "v_proj",
40
  "down_proj",
41
  "o_proj",
42
+ "up_proj"
 
 
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
results/phase1/checkpoint-100/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d430aea910e6d8b7347568b5b75ef11de9ff5b23b6d2010fd456033e1f218b07
3
  size 73911112
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9ce4a1cadb39ce5fa4f01584e98f33cbc09dc8c4d7425778d7edc33616448be8
3
  size 73911112
results/phase1/checkpoint-100/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:13de9f002823fbe63e71b9d1eb8ec6d2c4b8eca35a4ee367099322a459b3b1cb
3
  size 37969669
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7247a60399019dbce8eac389f415183266e38d6d0180c103cbec455696336c52
3
  size 37969669
results/phase1/checkpoint-100/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c1449e60d52f06cceec8589785267b0457345be74a8ca37687099a6addff91e0
3
  size 14645
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f27587650ba6b8ea25bd2cc4a67db408173e91796efadb6b47615bd0c9322135
3
  size 14645
results/phase1/checkpoint-100/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7254cd54e69ff9b68992c6e5838d1590c8e033e5a4605990d0b4034185923db6
3
  size 1465
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05a8fd04c04fd558f5a6d862a4a2344b4df262468cae8a6c245d1277ae908acb
3
  size 1465
results/phase1/checkpoint-100/trainer_state.json CHANGED
@@ -2,7 +2,7 @@
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
- "epoch": 0.45045045045045046,
6
  "eval_steps": 500,
7
  "global_step": 100,
8
  "is_hyper_param_search": false,
@@ -15,25 +15,25 @@
15
  "clip_ratio/low_mean": 0.0,
16
  "clip_ratio/low_min": 0.0,
17
  "clip_ratio/region_mean": 0.0,
18
- "completion_length": 29.05,
19
- "completions/clipped_ratio": 0.05,
20
- "completions/max_length": 44.2,
21
- "completions/max_terminated_length": 31.0,
22
- "completions/mean_length": 29.05,
23
- "completions/mean_terminated_length": 25.583333587646486,
24
- "completions/min_length": 21.4,
25
- "completions/min_terminated_length": 21.4,
26
- "epoch": 0.02252252252252252,
27
- "frac_reward_zero_std": 0.0,
28
- "grad_norm": 1.2577366828918457,
29
- "kl": 7.414049605358741e-06,
30
  "learning_rate": 1.6666666666666667e-06,
31
- "loss": 2.8312206268310547e-07,
32
- "num_tokens": 21378.0,
33
- "reward": 0.4944999933242798,
34
- "reward_std": 0.15089658200740813,
35
- "rewards/reward_fn/mean": 0.49449999928474425,
36
- "rewards/reward_fn/std": 0.12999328523874282,
37
  "step": 5
38
  },
39
  {
@@ -42,25 +42,25 @@
42
  "clip_ratio/low_mean": 0.0,
43
  "clip_ratio/low_min": 0.0,
44
  "clip_ratio/region_mean": 0.0,
45
- "completion_length": 29.175,
46
  "completions/clipped_ratio": 0.0,
47
- "completions/max_length": 41.8,
48
- "completions/max_terminated_length": 41.8,
49
- "completions/mean_length": 29.175,
50
- "completions/mean_terminated_length": 29.175,
51
- "completions/min_length": 19.2,
52
- "completions/min_terminated_length": 19.2,
53
- "epoch": 0.04504504504504504,
54
  "frac_reward_zero_std": 0.0,
55
- "grad_norm": 1.148110032081604,
56
- "kl": 2.58529256058182e-05,
57
  "learning_rate": 3.7500000000000005e-06,
58
- "loss": 9.343028068542481e-07,
59
- "num_tokens": 42709.0,
60
- "reward": 0.48589999675750734,
61
- "reward_std": 0.14184561967849732,
62
- "rewards/reward_fn/mean": 0.4859000027179718,
63
- "rewards/reward_fn/std": 0.12539554834365846,
64
  "step": 10
65
  },
66
  {
@@ -69,25 +69,25 @@
69
  "clip_ratio/low_mean": 0.0,
70
  "clip_ratio/low_min": 0.0,
71
  "clip_ratio/region_mean": 0.0,
72
- "completion_length": 28.825,
73
  "completions/clipped_ratio": 0.0,
74
- "completions/max_length": 44.2,
75
- "completions/max_terminated_length": 44.2,
76
- "completions/mean_length": 28.825,
77
- "completions/mean_terminated_length": 28.825,
78
- "completions/min_length": 21.2,
79
- "completions/min_terminated_length": 21.2,
80
- "epoch": 0.06756756756756757,
81
  "frac_reward_zero_std": 0.0,
82
- "grad_norm": 1.1520825624465942,
83
- "kl": 0.0001522944080875277,
84
- "learning_rate": 4.952380952380953e-06,
85
- "loss": 6.105005741119385e-06,
86
- "num_tokens": 63008.0,
87
- "reward": 0.4894999861717224,
88
- "reward_std": 0.15768481492996217,
89
- "rewards/reward_fn/mean": 0.48950000405311583,
90
- "rewards/reward_fn/std": 0.14185291826725005,
91
  "step": 15
92
  },
93
  {
@@ -96,25 +96,25 @@
96
  "clip_ratio/low_mean": 0.0,
97
  "clip_ratio/low_min": 0.0,
98
  "clip_ratio/region_mean": 0.0,
99
- "completion_length": 28.25,
100
  "completions/clipped_ratio": 0.0,
101
- "completions/max_length": 43.2,
102
- "completions/max_terminated_length": 43.2,
103
- "completions/mean_length": 28.25,
104
- "completions/mean_terminated_length": 28.25,
105
- "completions/min_length": 21.6,
106
- "completions/min_terminated_length": 21.6,
107
- "epoch": 0.09009009009009009,
108
  "frac_reward_zero_std": 0.0,
109
- "grad_norm": 1.0063226222991943,
110
- "kl": 0.0008983879492006963,
111
- "learning_rate": 4.833333333333333e-06,
112
- "loss": 3.593862056732178e-05,
113
- "num_tokens": 82836.0,
114
- "reward": 0.4881500005722046,
115
- "reward_std": 0.14941166192293168,
116
- "rewards/reward_fn/mean": 0.48814999461174013,
117
- "rewards/reward_fn/std": 0.12791907638311387,
118
  "step": 20
119
  },
120
  {
@@ -123,25 +123,25 @@
123
  "clip_ratio/low_mean": 0.0,
124
  "clip_ratio/low_min": 0.0,
125
  "clip_ratio/region_mean": 0.0,
126
- "completion_length": 27.25,
127
  "completions/clipped_ratio": 0.0,
128
- "completions/max_length": 45.6,
129
- "completions/max_terminated_length": 45.6,
130
- "completions/mean_length": 27.25,
131
- "completions/mean_terminated_length": 27.25,
132
- "completions/min_length": 18.8,
133
- "completions/min_terminated_length": 18.8,
134
- "epoch": 0.11261261261261261,
135
  "frac_reward_zero_std": 0.0,
136
- "grad_norm": 1.3132057189941406,
137
- "kl": 0.003781939600594342,
138
- "learning_rate": 4.714285714285715e-06,
139
- "loss": 0.00015127062797546387,
140
- "num_tokens": 103596.0,
141
- "reward": 0.4680500030517578,
142
- "reward_std": 0.17189764976501465,
143
- "rewards/reward_fn/mean": 0.46804999113082885,
144
- "rewards/reward_fn/std": 0.17421672195196153,
145
  "step": 25
146
  },
147
  {
@@ -150,25 +150,25 @@
150
  "clip_ratio/low_mean": 0.0,
151
  "clip_ratio/low_min": 0.0,
152
  "clip_ratio/region_mean": 0.0,
153
- "completion_length": 25.225,
154
  "completions/clipped_ratio": 0.0,
155
- "completions/max_length": 34.6,
156
- "completions/max_terminated_length": 34.6,
157
- "completions/mean_length": 25.225,
158
- "completions/mean_terminated_length": 25.225,
159
- "completions/min_length": 19.2,
160
- "completions/min_terminated_length": 19.2,
161
- "epoch": 0.13513513513513514,
162
  "frac_reward_zero_std": 0.0,
163
- "grad_norm": 1.5750545263290405,
164
- "kl": 0.006673775642411783,
165
- "learning_rate": 4.595238095238095e-06,
166
- "loss": 0.0002669498324394226,
167
- "num_tokens": 123651.0,
168
- "reward": 0.48049998879432676,
169
- "reward_std": 0.1656044065952301,
170
- "rewards/reward_fn/mean": 0.48049998879432676,
171
- "rewards/reward_fn/std": 0.13908967524766921,
172
  "step": 30
173
  },
174
  {
@@ -177,25 +177,25 @@
177
  "clip_ratio/low_mean": 0.0,
178
  "clip_ratio/low_min": 0.0,
179
  "clip_ratio/region_mean": 0.0,
180
- "completion_length": 28.775,
181
- "completions/clipped_ratio": 0.025,
182
- "completions/max_length": 54.8,
183
- "completions/max_terminated_length": 41.8,
184
- "completions/mean_length": 28.775,
185
- "completions/mean_terminated_length": 26.967857360839844,
186
- "completions/min_length": 15.4,
187
- "completions/min_terminated_length": 15.4,
188
- "epoch": 0.15765765765765766,
189
  "frac_reward_zero_std": 0.0,
190
- "grad_norm": 1.1105402708053589,
191
- "kl": 0.005106111426721327,
192
- "learning_rate": 4.476190476190477e-06,
193
- "loss": 0.0002042025327682495,
194
- "num_tokens": 144316.0,
195
- "reward": 0.4778999924659729,
196
- "reward_std": 0.17012988924980163,
197
- "rewards/reward_fn/mean": 0.47790001034736634,
198
- "rewards/reward_fn/std": 0.14147266000509262,
199
  "step": 35
200
  },
201
  {
@@ -204,25 +204,25 @@
204
  "clip_ratio/low_mean": 0.0,
205
  "clip_ratio/low_min": 0.0,
206
  "clip_ratio/region_mean": 0.0,
207
- "completion_length": 24.4,
208
- "completions/clipped_ratio": 0.0,
209
- "completions/max_length": 33.4,
210
- "completions/max_terminated_length": 33.4,
211
- "completions/mean_length": 24.4,
212
- "completions/mean_terminated_length": 24.4,
213
- "completions/min_length": 18.8,
214
- "completions/min_terminated_length": 18.8,
215
- "epoch": 0.18018018018018017,
216
  "frac_reward_zero_std": 0.0,
217
- "grad_norm": 1.0676746368408203,
218
- "kl": 0.006288998411764624,
219
- "learning_rate": 4.357142857142857e-06,
220
- "loss": 0.00025154203176498414,
221
- "num_tokens": 164334.0,
222
- "reward": 0.5014999985694886,
223
- "reward_std": 0.15358359068632127,
224
- "rewards/reward_fn/mean": 0.5014999985694886,
225
- "rewards/reward_fn/std": 0.13824734836816788,
226
  "step": 40
227
  },
228
  {
@@ -231,25 +231,25 @@
231
  "clip_ratio/low_mean": 0.0,
232
  "clip_ratio/low_min": 0.0,
233
  "clip_ratio/region_mean": 0.0,
234
- "completion_length": 27.35,
235
  "completions/clipped_ratio": 0.0,
236
- "completions/max_length": 43.6,
237
- "completions/max_terminated_length": 43.6,
238
- "completions/mean_length": 27.35,
239
- "completions/mean_terminated_length": 27.35,
240
- "completions/min_length": 19.6,
241
- "completions/min_terminated_length": 19.6,
242
- "epoch": 0.20270270270270271,
243
  "frac_reward_zero_std": 0.0,
244
- "grad_norm": 1.0749404430389404,
245
- "kl": 0.010629063473606948,
246
- "learning_rate": 4.238095238095239e-06,
247
- "loss": 0.0004251018166542053,
248
- "num_tokens": 185140.0,
249
- "reward": 0.524949985742569,
250
- "reward_std": 0.14630039632320405,
251
- "rewards/reward_fn/mean": 0.5249499917030335,
252
- "rewards/reward_fn/std": 0.1460244983434677,
253
  "step": 45
254
  },
255
  {
@@ -258,25 +258,25 @@
258
  "clip_ratio/low_mean": 0.0,
259
  "clip_ratio/low_min": 0.0,
260
  "clip_ratio/region_mean": 0.0,
261
- "completion_length": 25.425,
262
  "completions/clipped_ratio": 0.0,
263
  "completions/max_length": 35.2,
264
  "completions/max_terminated_length": 35.2,
265
- "completions/mean_length": 25.425,
266
- "completions/mean_terminated_length": 25.425,
267
- "completions/min_length": 17.6,
268
- "completions/min_terminated_length": 17.6,
269
- "epoch": 0.22522522522522523,
270
  "frac_reward_zero_std": 0.0,
271
- "grad_norm": 1.4353065490722656,
272
- "kl": 0.008290678875346203,
273
- "learning_rate": 4.119047619047619e-06,
274
- "loss": 0.0003315746784210205,
275
- "num_tokens": 206569.0,
276
- "reward": 0.4908999800682068,
277
- "reward_std": 0.17055415213108063,
278
- "rewards/reward_fn/mean": 0.49089999198913575,
279
- "rewards/reward_fn/std": 0.154354290664196,
280
  "step": 50
281
  },
282
  {
@@ -285,25 +285,25 @@
285
  "clip_ratio/low_mean": 0.0,
286
  "clip_ratio/low_min": 0.0,
287
  "clip_ratio/region_mean": 0.0,
288
- "completion_length": 27.7,
289
- "completions/clipped_ratio": 0.025,
290
- "completions/max_length": 50.6,
291
- "completions/max_terminated_length": 37.2,
292
- "completions/mean_length": 27.7,
293
- "completions/mean_terminated_length": 25.91071434020996,
294
- "completions/min_length": 18.2,
295
- "completions/min_terminated_length": 18.2,
296
- "epoch": 0.24774774774774774,
297
  "frac_reward_zero_std": 0.0,
298
- "grad_norm": 0.8640509843826294,
299
- "kl": 0.012456063569698018,
300
- "learning_rate": 4.000000000000001e-06,
301
- "loss": 0.0004982292652130127,
302
- "num_tokens": 226097.0,
303
- "reward": 0.4839499771595001,
304
- "reward_std": 0.1590283066034317,
305
- "rewards/reward_fn/mean": 0.48394999504089353,
306
- "rewards/reward_fn/std": 0.1380786642432213,
307
  "step": 55
308
  },
309
  {
@@ -312,25 +312,25 @@
312
  "clip_ratio/low_mean": 0.0,
313
  "clip_ratio/low_min": 0.0,
314
  "clip_ratio/region_mean": 0.0,
315
- "completion_length": 26.875,
316
- "completions/clipped_ratio": 0.0,
317
- "completions/max_length": 40.0,
318
- "completions/max_terminated_length": 40.0,
319
- "completions/mean_length": 26.875,
320
- "completions/mean_terminated_length": 26.875,
321
- "completions/min_length": 20.6,
322
- "completions/min_terminated_length": 20.6,
323
- "epoch": 0.2702702702702703,
324
  "frac_reward_zero_std": 0.0,
325
- "grad_norm": 1.0529346466064453,
326
- "kl": 0.007519814093393506,
327
- "learning_rate": 3.880952380952381e-06,
328
- "loss": 0.00030075013637542725,
329
- "num_tokens": 246836.0,
330
- "reward": 0.4914499819278717,
331
- "reward_std": 0.1727461814880371,
332
- "rewards/reward_fn/mean": 0.49144998788833616,
333
- "rewards/reward_fn/std": 0.1568704679608345,
334
  "step": 60
335
  },
336
  {
@@ -339,25 +339,25 @@
339
  "clip_ratio/low_mean": 0.0,
340
  "clip_ratio/low_min": 0.0,
341
  "clip_ratio/region_mean": 0.0,
342
- "completion_length": 27.825,
343
- "completions/clipped_ratio": 0.025,
344
- "completions/max_length": 51.2,
345
- "completions/max_terminated_length": 38.0,
346
- "completions/mean_length": 27.825,
347
- "completions/mean_terminated_length": 26.03214302062988,
348
- "completions/min_length": 17.6,
349
- "completions/min_terminated_length": 17.6,
350
- "epoch": 0.2927927927927928,
351
  "frac_reward_zero_std": 0.0,
352
- "grad_norm": 1.149415373802185,
353
- "kl": 0.00938094645498495,
354
- "learning_rate": 3.761904761904762e-06,
355
- "loss": 0.0003751114010810852,
356
- "num_tokens": 266985.0,
357
- "reward": 0.5003499746322632,
358
- "reward_std": 0.16892780363559723,
359
- "rewards/reward_fn/mean": 0.5003500044345855,
360
- "rewards/reward_fn/std": 0.14663992822170258,
361
  "step": 65
362
  },
363
  {
@@ -366,25 +366,25 @@
366
  "clip_ratio/low_mean": 0.0,
367
  "clip_ratio/low_min": 0.0,
368
  "clip_ratio/region_mean": 0.0,
369
- "completion_length": 27.0,
370
  "completions/clipped_ratio": 0.0,
371
- "completions/max_length": 40.6,
372
- "completions/max_terminated_length": 40.6,
373
- "completions/mean_length": 27.0,
374
- "completions/mean_terminated_length": 27.0,
375
- "completions/min_length": 17.8,
376
- "completions/min_terminated_length": 17.8,
377
- "epoch": 0.3153153153153153,
378
  "frac_reward_zero_std": 0.0,
379
- "grad_norm": 0.9893133640289307,
380
- "kl": 0.010053556466664304,
381
- "learning_rate": 3.642857142857143e-06,
382
- "loss": 0.00040204524993896483,
383
- "num_tokens": 288099.0,
384
- "reward": 0.4705499827861786,
385
- "reward_std": 0.16256383657455445,
386
- "rewards/reward_fn/mean": 0.470550000667572,
387
- "rewards/reward_fn/std": 0.13094386458396912,
388
  "step": 70
389
  },
390
  {
@@ -393,25 +393,25 @@
393
  "clip_ratio/low_mean": 0.0,
394
  "clip_ratio/low_min": 0.0,
395
  "clip_ratio/region_mean": 0.0,
396
- "completion_length": 25.75,
397
- "completions/clipped_ratio": 0.0,
398
- "completions/max_length": 34.8,
399
- "completions/max_terminated_length": 34.8,
400
- "completions/mean_length": 25.75,
401
- "completions/mean_terminated_length": 25.75,
402
- "completions/min_length": 19.8,
403
- "completions/min_terminated_length": 19.8,
404
- "epoch": 0.33783783783783783,
405
  "frac_reward_zero_std": 0.0,
406
- "grad_norm": 1.063340187072754,
407
- "kl": 0.008550376921630232,
408
- "learning_rate": 3.523809523809524e-06,
409
- "loss": 0.00034204572439193725,
410
- "num_tokens": 308531.0,
411
- "reward": 0.5000500082969666,
412
- "reward_std": 0.16242243051528932,
413
- "rewards/reward_fn/mean": 0.5000500082969666,
414
- "rewards/reward_fn/std": 0.14417539685964584,
415
  "step": 75
416
  },
417
  {
@@ -420,25 +420,25 @@
420
  "clip_ratio/low_mean": 0.0,
421
  "clip_ratio/low_min": 0.0,
422
  "clip_ratio/region_mean": 0.0,
423
- "completion_length": 27.025,
424
  "completions/clipped_ratio": 0.0,
425
- "completions/max_length": 40.0,
426
- "completions/max_terminated_length": 40.0,
427
- "completions/mean_length": 27.025,
428
- "completions/mean_terminated_length": 27.025,
429
- "completions/min_length": 19.4,
430
- "completions/min_terminated_length": 19.4,
431
- "epoch": 0.36036036036036034,
432
  "frac_reward_zero_std": 0.0,
433
- "grad_norm": 1.2858343124389648,
434
- "kl": 0.019255010827328077,
435
- "learning_rate": 3.404761904761905e-06,
436
- "loss": 0.0007702425122261047,
437
- "num_tokens": 328874.0,
438
- "reward": 0.5261500000953674,
439
- "reward_std": 0.11886464804410934,
440
- "rewards/reward_fn/mean": 0.5261499881744385,
441
- "rewards/reward_fn/std": 0.12691220343112947,
442
  "step": 80
443
  },
444
  {
@@ -447,25 +447,25 @@
447
  "clip_ratio/low_mean": 0.0,
448
  "clip_ratio/low_min": 0.0,
449
  "clip_ratio/region_mean": 0.0,
450
- "completion_length": 25.475,
451
  "completions/clipped_ratio": 0.0,
452
- "completions/max_length": 33.0,
453
- "completions/max_terminated_length": 33.0,
454
- "completions/mean_length": 25.475,
455
- "completions/mean_terminated_length": 25.475,
456
- "completions/min_length": 19.0,
457
- "completions/min_terminated_length": 19.0,
458
- "epoch": 0.38288288288288286,
459
  "frac_reward_zero_std": 0.0,
460
- "grad_norm": 1.118030309677124,
461
- "kl": 0.01090274843845691,
462
- "learning_rate": 3.285714285714286e-06,
463
- "loss": 0.00043606162071228025,
464
- "num_tokens": 348373.0,
465
- "reward": 0.49950000643730164,
466
- "reward_std": 0.15655344128608703,
467
- "rewards/reward_fn/mean": 0.49950000643730164,
468
- "rewards/reward_fn/std": 0.14320258051156998,
469
  "step": 85
470
  },
471
  {
@@ -474,25 +474,25 @@
474
  "clip_ratio/low_mean": 0.0,
475
  "clip_ratio/low_min": 0.0,
476
  "clip_ratio/region_mean": 0.0,
477
- "completion_length": 27.9,
478
- "completions/clipped_ratio": 0.05,
479
- "completions/max_length": 55.4,
480
- "completions/max_terminated_length": 31.6,
481
- "completions/mean_length": 27.9,
482
- "completions/mean_terminated_length": 24.28571472167969,
483
- "completions/min_length": 18.0,
484
- "completions/min_terminated_length": 18.0,
485
- "epoch": 0.40540540540540543,
486
  "frac_reward_zero_std": 0.0,
487
- "grad_norm": 0.6538166403770447,
488
- "kl": 0.008556566922561615,
489
- "learning_rate": 3.1666666666666667e-06,
490
- "loss": 0.0003422081470489502,
491
- "num_tokens": 368815.0,
492
- "reward": 0.5215500056743622,
493
- "reward_std": 0.16581654250621797,
494
- "rewards/reward_fn/mean": 0.5215500056743622,
495
- "rewards/reward_fn/std": 0.15472394227981567,
496
  "step": 90
497
  },
498
  {
@@ -501,25 +501,25 @@
501
  "clip_ratio/low_mean": 0.0,
502
  "clip_ratio/low_min": 0.0,
503
  "clip_ratio/region_mean": 0.0,
504
- "completion_length": 25.75,
505
  "completions/clipped_ratio": 0.0,
506
- "completions/max_length": 32.4,
507
- "completions/max_terminated_length": 32.4,
508
- "completions/mean_length": 25.75,
509
- "completions/mean_terminated_length": 25.75,
510
  "completions/min_length": 20.8,
511
  "completions/min_terminated_length": 20.8,
512
- "epoch": 0.42792792792792794,
513
  "frac_reward_zero_std": 0.0,
514
- "grad_norm": 1.180251955986023,
515
- "kl": 0.014757526386529208,
516
- "learning_rate": 3.047619047619048e-06,
517
- "loss": 0.0005902737379074096,
518
- "num_tokens": 388761.0,
519
- "reward": 0.514000016450882,
520
- "reward_std": 0.1465125188231468,
521
- "rewards/reward_fn/mean": 0.5139999985694885,
522
- "rewards/reward_fn/std": 0.14701166301965712,
523
  "step": 95
524
  },
525
  {
@@ -528,31 +528,31 @@
528
  "clip_ratio/low_mean": 0.0,
529
  "clip_ratio/low_min": 0.0,
530
  "clip_ratio/region_mean": 0.0,
531
- "completion_length": 26.35,
532
- "completions/clipped_ratio": 0.025,
533
- "completions/max_length": 46.6,
534
- "completions/max_terminated_length": 32.4,
535
- "completions/mean_length": 26.35,
536
- "completions/mean_terminated_length": 24.517857360839844,
537
- "completions/min_length": 17.8,
538
- "completions/min_terminated_length": 17.8,
539
- "epoch": 0.45045045045045046,
540
  "frac_reward_zero_std": 0.0,
541
- "grad_norm": 0.8863919973373413,
542
- "kl": 0.017311154678463937,
543
- "learning_rate": 2.928571428571429e-06,
544
- "loss": 0.0006924748420715332,
545
- "num_tokens": 408309.0,
546
- "reward": 0.5527999818325042,
547
- "reward_std": 0.1593818634748459,
548
- "rewards/reward_fn/mean": 0.5527999699115753,
549
- "rewards/reward_fn/std": 0.16823574155569077,
550
  "step": 100
551
  }
552
  ],
553
  "logging_steps": 5,
554
- "max_steps": 222,
555
- "num_input_tokens_seen": 408309,
556
  "num_train_epochs": 1,
557
  "save_steps": 50,
558
  "stateful_callbacks": {
 
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
+ "epoch": 0.4444444444444444,
6
  "eval_steps": 500,
7
  "global_step": 100,
8
  "is_hyper_param_search": false,
 
15
  "clip_ratio/low_mean": 0.0,
16
  "clip_ratio/low_min": 0.0,
17
  "clip_ratio/region_mean": 0.0,
18
+ "completion_length": 25.075,
19
+ "completions/clipped_ratio": 0.0,
20
+ "completions/max_length": 30.8,
21
+ "completions/max_terminated_length": 30.8,
22
+ "completions/mean_length": 25.075,
23
+ "completions/mean_terminated_length": 25.075,
24
+ "completions/min_length": 19.4,
25
+ "completions/min_terminated_length": 19.4,
26
+ "epoch": 0.022222222222222223,
27
+ "frac_reward_zero_std": 0.6,
28
+ "grad_norm": 1.3092279434204102,
29
+ "kl": 0.03290070100920275,
30
  "learning_rate": 1.6666666666666667e-06,
31
+ "loss": 0.0013160213828086853,
32
+ "num_tokens": 20275.0,
33
+ "reward": 0.48624999523162843,
34
+ "reward_std": 0.05833630859851837,
35
+ "rewards/reward_fn/mean": 0.4862500011920929,
36
+ "rewards/reward_fn/std": 0.04605271518230438,
37
  "step": 5
38
  },
39
  {
 
42
  "clip_ratio/low_mean": 0.0,
43
  "clip_ratio/low_min": 0.0,
44
  "clip_ratio/region_mean": 0.0,
45
+ "completion_length": 25.925,
46
  "completions/clipped_ratio": 0.0,
47
+ "completions/max_length": 33.0,
48
+ "completions/max_terminated_length": 33.0,
49
+ "completions/mean_length": 25.925,
50
+ "completions/mean_terminated_length": 25.925,
51
+ "completions/min_length": 19.4,
52
+ "completions/min_terminated_length": 19.4,
53
+ "epoch": 0.044444444444444446,
54
  "frac_reward_zero_std": 0.0,
55
+ "grad_norm": 1.4204450845718384,
56
+ "kl": 0.040759827199508436,
57
  "learning_rate": 3.7500000000000005e-06,
58
+ "loss": 0.0016303554177284241,
59
+ "num_tokens": 41560.0,
60
+ "reward": 0.5283999860286712,
61
+ "reward_std": 0.17366542518138886,
62
+ "rewards/reward_fn/mean": 0.5283999919891358,
63
+ "rewards/reward_fn/std": 0.16809422075748442,
64
  "step": 10
65
  },
66
  {
 
69
  "clip_ratio/low_mean": 0.0,
70
  "clip_ratio/low_min": 0.0,
71
  "clip_ratio/region_mean": 0.0,
72
+ "completion_length": 27.45,
73
  "completions/clipped_ratio": 0.0,
74
+ "completions/max_length": 33.6,
75
+ "completions/max_terminated_length": 33.6,
76
+ "completions/mean_length": 27.45,
77
+ "completions/mean_terminated_length": 27.45,
78
+ "completions/min_length": 24.0,
79
+ "completions/min_terminated_length": 24.0,
80
+ "epoch": 0.06666666666666667,
81
  "frac_reward_zero_std": 0.0,
82
+ "grad_norm": 0.9515441656112671,
83
+ "kl": 0.049849181214813146,
84
+ "learning_rate": 4.953051643192488e-06,
85
+ "loss": 0.0019938603043556215,
86
+ "num_tokens": 62246.0,
87
+ "reward": 0.4975499987602234,
88
+ "reward_std": 0.1564827263355255,
89
+ "rewards/reward_fn/mean": 0.49755001068115234,
90
+ "rewards/reward_fn/std": 0.14618260115385057,
91
  "step": 15
92
  },
93
  {
 
96
  "clip_ratio/low_mean": 0.0,
97
  "clip_ratio/low_min": 0.0,
98
  "clip_ratio/region_mean": 0.0,
99
+ "completion_length": 26.525,
100
  "completions/clipped_ratio": 0.0,
101
+ "completions/max_length": 33.0,
102
+ "completions/max_terminated_length": 33.0,
103
+ "completions/mean_length": 26.525,
104
+ "completions/mean_terminated_length": 26.525,
105
+ "completions/min_length": 22.2,
106
+ "completions/min_terminated_length": 22.2,
107
+ "epoch": 0.08888888888888889,
108
  "frac_reward_zero_std": 0.0,
109
+ "grad_norm": 0.6862999200820923,
110
+ "kl": 0.04350169296376407,
111
+ "learning_rate": 4.835680751173709e-06,
112
+ "loss": 0.0017400771379470826,
113
+ "num_tokens": 81479.0,
114
+ "reward": 0.5159000098705292,
115
+ "reward_std": 0.17465537190437316,
116
+ "rewards/reward_fn/mean": 0.5158999919891357,
117
+ "rewards/reward_fn/std": 0.16455023884773254,
118
  "step": 20
119
  },
120
  {
 
123
  "clip_ratio/low_mean": 0.0,
124
  "clip_ratio/low_min": 0.0,
125
  "clip_ratio/region_mean": 0.0,
126
+ "completion_length": 28.0,
127
  "completions/clipped_ratio": 0.0,
128
+ "completions/max_length": 40.6,
129
+ "completions/max_terminated_length": 40.6,
130
+ "completions/mean_length": 28.0,
131
+ "completions/mean_terminated_length": 28.0,
132
+ "completions/min_length": 22.6,
133
+ "completions/min_terminated_length": 22.6,
134
+ "epoch": 0.1111111111111111,
135
  "frac_reward_zero_std": 0.0,
136
+ "grad_norm": 1.7516958713531494,
137
+ "kl": 0.05824573401478119,
138
+ "learning_rate": 4.71830985915493e-06,
139
+ "loss": 0.002329717576503754,
140
+ "num_tokens": 102833.0,
141
+ "reward": 0.5547999978065491,
142
+ "reward_std": 0.18328206837177277,
143
+ "rewards/reward_fn/mean": 0.5547999858856201,
144
+ "rewards/reward_fn/std": 0.1974634051322937,
145
  "step": 25
146
  },
147
  {
 
150
  "clip_ratio/low_mean": 0.0,
151
  "clip_ratio/low_min": 0.0,
152
  "clip_ratio/region_mean": 0.0,
153
+ "completion_length": 28.675,
154
  "completions/clipped_ratio": 0.0,
155
+ "completions/max_length": 37.4,
156
+ "completions/max_terminated_length": 37.4,
157
+ "completions/mean_length": 28.675,
158
+ "completions/mean_terminated_length": 28.675,
159
+ "completions/min_length": 22.2,
160
+ "completions/min_terminated_length": 22.2,
161
+ "epoch": 0.13333333333333333,
162
  "frac_reward_zero_std": 0.0,
163
+ "grad_norm": 1.5611271858215332,
164
+ "kl": 0.09752920938190072,
165
+ "learning_rate": 4.60093896713615e-06,
166
+ "loss": 0.0039011374115943908,
167
+ "num_tokens": 123876.0,
168
+ "reward": 0.5165499806404114,
169
+ "reward_std": 0.16765501499176025,
170
+ "rewards/reward_fn/mean": 0.5165499806404114,
171
+ "rewards/reward_fn/std": 0.16748485416173936,
172
  "step": 30
173
  },
174
  {
 
177
  "clip_ratio/low_mean": 0.0,
178
  "clip_ratio/low_min": 0.0,
179
  "clip_ratio/region_mean": 0.0,
180
+ "completion_length": 27.475,
181
+ "completions/clipped_ratio": 0.0,
182
+ "completions/max_length": 36.8,
183
+ "completions/max_terminated_length": 36.8,
184
+ "completions/mean_length": 27.475,
185
+ "completions/mean_terminated_length": 27.475,
186
+ "completions/min_length": 20.6,
187
+ "completions/min_terminated_length": 20.6,
188
+ "epoch": 0.15555555555555556,
189
  "frac_reward_zero_std": 0.0,
190
+ "grad_norm": 0.9550014138221741,
191
+ "kl": 0.08032013729680329,
192
+ "learning_rate": 4.483568075117371e-06,
193
+ "loss": 0.003212757408618927,
194
+ "num_tokens": 145019.0,
195
+ "reward": 0.5004499793052674,
196
+ "reward_std": 0.18391846716403962,
197
+ "rewards/reward_fn/mean": 0.5004500031471253,
198
+ "rewards/reward_fn/std": 0.16948954164981841,
199
  "step": 35
200
  },
201
  {
 
204
  "clip_ratio/low_mean": 0.0,
205
  "clip_ratio/low_min": 0.0,
206
  "clip_ratio/region_mean": 0.0,
207
+ "completion_length": 32.125,
208
+ "completions/clipped_ratio": 0.025,
209
+ "completions/max_length": 54.6,
210
+ "completions/max_terminated_length": 46.0,
211
+ "completions/mean_length": 32.125,
212
+ "completions/mean_terminated_length": 30.521428680419923,
213
+ "completions/min_length": 23.2,
214
+ "completions/min_terminated_length": 23.2,
215
+ "epoch": 0.17777777777777778,
216
  "frac_reward_zero_std": 0.0,
217
+ "grad_norm": 1.1203869581222534,
218
+ "kl": 0.08563535290304572,
219
+ "learning_rate": 4.3661971830985915e-06,
220
+ "loss": 0.003425435721874237,
221
+ "num_tokens": 164800.0,
222
+ "reward": 0.5472500026226044,
223
+ "reward_std": 0.19579786211252212,
224
+ "rewards/reward_fn/mean": 0.5472500085830688,
225
+ "rewards/reward_fn/std": 0.19857290089130403,
226
  "step": 40
227
  },
228
  {
 
231
  "clip_ratio/low_mean": 0.0,
232
  "clip_ratio/low_min": 0.0,
233
  "clip_ratio/region_mean": 0.0,
234
+ "completion_length": 27.875,
235
  "completions/clipped_ratio": 0.0,
236
+ "completions/max_length": 35.0,
237
+ "completions/max_terminated_length": 35.0,
238
+ "completions/mean_length": 27.875,
239
+ "completions/mean_terminated_length": 27.875,
240
+ "completions/min_length": 21.4,
241
+ "completions/min_terminated_length": 21.4,
242
+ "epoch": 0.2,
243
  "frac_reward_zero_std": 0.0,
244
+ "grad_norm": 0.95042884349823,
245
+ "kl": 0.07295196709455923,
246
+ "learning_rate": 4.248826291079813e-06,
247
+ "loss": 0.0029180020093917845,
248
+ "num_tokens": 185761.0,
249
+ "reward": 0.5225999832153321,
250
+ "reward_std": 0.16489729881286622,
251
+ "rewards/reward_fn/mean": 0.5226000070571899,
252
+ "rewards/reward_fn/std": 0.16541497856378556,
253
  "step": 45
254
  },
255
  {
 
258
  "clip_ratio/low_mean": 0.0,
259
  "clip_ratio/low_min": 0.0,
260
  "clip_ratio/region_mean": 0.0,
261
+ "completion_length": 27.875,
262
  "completions/clipped_ratio": 0.0,
263
  "completions/max_length": 35.2,
264
  "completions/max_terminated_length": 35.2,
265
+ "completions/mean_length": 27.875,
266
+ "completions/mean_terminated_length": 27.875,
267
+ "completions/min_length": 21.4,
268
+ "completions/min_terminated_length": 21.4,
269
+ "epoch": 0.2222222222222222,
270
  "frac_reward_zero_std": 0.0,
271
+ "grad_norm": 1.1037548780441284,
272
+ "kl": 0.08246680488809943,
273
+ "learning_rate": 4.131455399061034e-06,
274
+ "loss": 0.0032985761761665346,
275
+ "num_tokens": 206674.0,
276
+ "reward": 0.5245000004768372,
277
+ "reward_std": 0.1653215616941452,
278
+ "rewards/reward_fn/mean": 0.5245000004768372,
279
+ "rewards/reward_fn/std": 0.16877340227365495,
280
  "step": 50
281
  },
282
  {
 
285
  "clip_ratio/low_mean": 0.0,
286
  "clip_ratio/low_min": 0.0,
287
  "clip_ratio/region_mean": 0.0,
288
+ "completion_length": 29.225,
289
+ "completions/clipped_ratio": 0.0,
290
+ "completions/max_length": 47.4,
291
+ "completions/max_terminated_length": 47.4,
292
+ "completions/mean_length": 29.225,
293
+ "completions/mean_terminated_length": 29.225,
294
+ "completions/min_length": 20.2,
295
+ "completions/min_terminated_length": 20.2,
296
+ "epoch": 0.24444444444444444,
297
  "frac_reward_zero_std": 0.0,
298
+ "grad_norm": 0.8115681409835815,
299
+ "kl": 0.08297618771903217,
300
+ "learning_rate": 4.014084507042254e-06,
301
+ "loss": 0.0033189669251441956,
302
+ "num_tokens": 227351.0,
303
+ "reward": 0.5301500022411346,
304
+ "reward_std": 0.20612163245677947,
305
+ "rewards/reward_fn/mean": 0.5301500082015991,
306
+ "rewards/reward_fn/std": 0.19000594317913055,
307
  "step": 55
308
  },
309
  {
 
312
  "clip_ratio/low_mean": 0.0,
313
  "clip_ratio/low_min": 0.0,
314
  "clip_ratio/region_mean": 0.0,
315
+ "completion_length": 27.475,
316
+ "completions/clipped_ratio": 0.025,
317
+ "completions/max_length": 45.0,
318
+ "completions/max_terminated_length": 31.8,
319
+ "completions/mean_length": 27.475,
320
+ "completions/mean_terminated_length": 25.646428680419923,
321
+ "completions/min_length": 18.8,
322
+ "completions/min_terminated_length": 18.8,
323
+ "epoch": 0.26666666666666666,
324
  "frac_reward_zero_std": 0.0,
325
+ "grad_norm": 1.1722970008850098,
326
+ "kl": 0.06283304298995063,
327
+ "learning_rate": 3.896713615023475e-06,
328
+ "loss": 0.002513286471366882,
329
+ "num_tokens": 246464.0,
330
+ "reward": 0.513349997997284,
331
+ "reward_std": 0.18999958634376526,
332
+ "rewards/reward_fn/mean": 0.5133499920368194,
333
+ "rewards/reward_fn/std": 0.1748345360159874,
334
  "step": 60
335
  },
336
  {
 
339
  "clip_ratio/low_mean": 0.0,
340
  "clip_ratio/low_min": 0.0,
341
  "clip_ratio/region_mean": 0.0,
342
+ "completion_length": 26.075,
343
+ "completions/clipped_ratio": 0.0,
344
+ "completions/max_length": 34.2,
345
+ "completions/max_terminated_length": 34.2,
346
+ "completions/mean_length": 26.075,
347
+ "completions/mean_terminated_length": 26.075,
348
+ "completions/min_length": 19.4,
349
+ "completions/min_terminated_length": 19.4,
350
+ "epoch": 0.28888888888888886,
351
  "frac_reward_zero_std": 0.0,
352
+ "grad_norm": 1.2825119495391846,
353
+ "kl": 0.07132846353342756,
354
+ "learning_rate": 3.779342723004695e-06,
355
+ "loss": 0.0028530970215797425,
356
+ "num_tokens": 267279.0,
357
+ "reward": 0.5383000016212464,
358
+ "reward_std": 0.20746512711048126,
359
+ "rewards/reward_fn/mean": 0.5383000195026397,
360
+ "rewards/reward_fn/std": 0.1835445523262024,
361
  "step": 65
362
  },
363
  {
 
366
  "clip_ratio/low_mean": 0.0,
367
  "clip_ratio/low_min": 0.0,
368
  "clip_ratio/region_mean": 0.0,
369
+ "completion_length": 27.925,
370
  "completions/clipped_ratio": 0.0,
371
+ "completions/max_length": 44.0,
372
+ "completions/max_terminated_length": 44.0,
373
+ "completions/mean_length": 27.925,
374
+ "completions/mean_terminated_length": 27.925,
375
+ "completions/min_length": 22.0,
376
+ "completions/min_terminated_length": 22.0,
377
+ "epoch": 0.3111111111111111,
378
  "frac_reward_zero_std": 0.0,
379
+ "grad_norm": 0.5832120776176453,
380
+ "kl": 0.05197672065114602,
381
+ "learning_rate": 3.6619718309859158e-06,
382
+ "loss": 0.0020790368318557738,
383
+ "num_tokens": 287984.0,
384
+ "reward": 0.5424999892711639,
385
+ "reward_std": 0.18002938330173493,
386
+ "rewards/reward_fn/mean": 0.5424999952316284,
387
+ "rewards/reward_fn/std": 0.18769851326942444,
388
  "step": 70
389
  },
390
  {
 
393
  "clip_ratio/low_mean": 0.0,
394
  "clip_ratio/low_min": 0.0,
395
  "clip_ratio/region_mean": 0.0,
396
+ "completion_length": 27.6,
397
+ "completions/clipped_ratio": 0.025,
398
+ "completions/max_length": 44.6,
399
+ "completions/max_terminated_length": 30.6,
400
+ "completions/mean_length": 27.6,
401
+ "completions/mean_terminated_length": 25.807143020629884,
402
+ "completions/min_length": 22.2,
403
+ "completions/min_terminated_length": 22.2,
404
+ "epoch": 0.3333333333333333,
405
  "frac_reward_zero_std": 0.0,
406
+ "grad_norm": 0.6677760481834412,
407
+ "kl": 0.03671608620206825,
408
+ "learning_rate": 3.5446009389671364e-06,
409
+ "loss": 0.0014685407280921937,
410
+ "num_tokens": 307800.0,
411
+ "reward": 0.5715999841690064,
412
+ "reward_std": 0.18639334440231323,
413
+ "rewards/reward_fn/mean": 0.5715999960899353,
414
+ "rewards/reward_fn/std": 0.19429495334625244,
415
  "step": 75
416
  },
417
  {
 
420
  "clip_ratio/low_mean": 0.0,
421
  "clip_ratio/low_min": 0.0,
422
  "clip_ratio/region_mean": 0.0,
423
+ "completion_length": 25.6,
424
  "completions/clipped_ratio": 0.0,
425
+ "completions/max_length": 30.2,
426
+ "completions/max_terminated_length": 30.2,
427
+ "completions/mean_length": 25.6,
428
+ "completions/mean_terminated_length": 25.6,
429
+ "completions/min_length": 21.4,
430
+ "completions/min_terminated_length": 21.4,
431
+ "epoch": 0.35555555555555557,
432
  "frac_reward_zero_std": 0.0,
433
+ "grad_norm": 0.7261527180671692,
434
+ "kl": 0.05616153636947274,
435
+ "learning_rate": 3.427230046948357e-06,
436
+ "loss": 0.0022463813424110413,
437
+ "num_tokens": 327794.0,
438
+ "reward": 0.5464499950408935,
439
+ "reward_std": 0.18264567852020264,
440
+ "rewards/reward_fn/mean": 0.5464499950408935,
441
+ "rewards/reward_fn/std": 0.17480863779783248,
442
  "step": 80
443
  },
444
  {
 
447
  "clip_ratio/low_mean": 0.0,
448
  "clip_ratio/low_min": 0.0,
449
  "clip_ratio/region_mean": 0.0,
450
+ "completion_length": 24.075,
451
  "completions/clipped_ratio": 0.0,
452
+ "completions/max_length": 29.6,
453
+ "completions/max_terminated_length": 29.6,
454
+ "completions/mean_length": 24.075,
455
+ "completions/mean_terminated_length": 24.075,
456
+ "completions/min_length": 16.6,
457
+ "completions/min_terminated_length": 16.6,
458
+ "epoch": 0.37777777777777777,
459
  "frac_reward_zero_std": 0.0,
460
+ "grad_norm": 1.1086981296539307,
461
+ "kl": 0.07743949705036357,
462
+ "learning_rate": 3.3098591549295777e-06,
463
+ "loss": 0.0030974715948104857,
464
+ "num_tokens": 348233.0,
465
+ "reward": 0.5536999821662902,
466
+ "reward_std": 0.18073648810386658,
467
+ "rewards/reward_fn/mean": 0.5536999821662902,
468
+ "rewards/reward_fn/std": 0.1820658951997757,
469
  "step": 85
470
  },
471
  {
 
474
  "clip_ratio/low_mean": 0.0,
475
  "clip_ratio/low_min": 0.0,
476
  "clip_ratio/region_mean": 0.0,
477
+ "completion_length": 25.375,
478
+ "completions/clipped_ratio": 0.0,
479
+ "completions/max_length": 34.4,
480
+ "completions/max_terminated_length": 34.4,
481
+ "completions/mean_length": 25.375,
482
+ "completions/mean_terminated_length": 25.375,
483
+ "completions/min_length": 18.6,
484
+ "completions/min_terminated_length": 18.6,
485
+ "epoch": 0.4,
486
  "frac_reward_zero_std": 0.0,
487
+ "grad_norm": 1.1903071403503418,
488
+ "kl": 0.058934826811309904,
489
+ "learning_rate": 3.1924882629107983e-06,
490
+ "loss": 0.002357317507266998,
491
+ "num_tokens": 367732.0,
492
+ "reward": 0.5361499905586242,
493
+ "reward_std": 0.179675829410553,
494
+ "rewards/reward_fn/mean": 0.5361500144004822,
495
+ "rewards/reward_fn/std": 0.18574500381946563,
496
  "step": 90
497
  },
498
  {
 
501
  "clip_ratio/low_mean": 0.0,
502
  "clip_ratio/low_min": 0.0,
503
  "clip_ratio/region_mean": 0.0,
504
+ "completion_length": 26.825,
505
  "completions/clipped_ratio": 0.0,
506
+ "completions/max_length": 37.2,
507
+ "completions/max_terminated_length": 37.2,
508
+ "completions/mean_length": 26.825,
509
+ "completions/mean_terminated_length": 26.825,
510
  "completions/min_length": 20.8,
511
  "completions/min_terminated_length": 20.8,
512
+ "epoch": 0.4222222222222222,
513
  "frac_reward_zero_std": 0.0,
514
+ "grad_norm": 1.0952492952346802,
515
+ "kl": 0.040232469444163144,
516
+ "learning_rate": 3.075117370892019e-06,
517
+ "loss": 0.0016092658042907715,
518
+ "num_tokens": 387075.0,
519
+ "reward": 0.5278500020503998,
520
+ "reward_std": 0.1932522773742676,
521
+ "rewards/reward_fn/mean": 0.5278499901294709,
522
+ "rewards/reward_fn/std": 0.1796583503484726,
523
  "step": 95
524
  },
525
  {
 
528
  "clip_ratio/low_mean": 0.0,
529
  "clip_ratio/low_min": 0.0,
530
  "clip_ratio/region_mean": 0.0,
531
+ "completion_length": 26.125,
532
+ "completions/clipped_ratio": 0.0,
533
+ "completions/max_length": 35.6,
534
+ "completions/max_terminated_length": 35.6,
535
+ "completions/mean_length": 26.125,
536
+ "completions/mean_terminated_length": 26.125,
537
+ "completions/min_length": 22.6,
538
+ "completions/min_terminated_length": 22.6,
539
+ "epoch": 0.4444444444444444,
540
  "frac_reward_zero_std": 0.0,
541
+ "grad_norm": 0.8072858452796936,
542
+ "kl": 0.06188266044482589,
543
+ "learning_rate": 2.9577464788732396e-06,
544
+ "loss": 0.0024752289056777952,
545
+ "num_tokens": 406884.0,
546
+ "reward": 0.5849499881267548,
547
+ "reward_std": 0.20499025285243988,
548
+ "rewards/reward_fn/mean": 0.5849500000476837,
549
+ "rewards/reward_fn/std": 0.2056175708770752,
550
  "step": 100
551
  }
552
  ],
553
  "logging_steps": 5,
554
+ "max_steps": 225,
555
+ "num_input_tokens_seen": 406884,
556
  "num_train_epochs": 1,
557
  "save_steps": 50,
558
  "stateful_callbacks": {
results/phase1/checkpoint-150/README.md CHANGED
@@ -6,6 +6,7 @@ tags:
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
 
9
  - transformers
10
  - trl
11
  - unsloth
 
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
9
+ - sft
10
  - transformers
11
  - trl
12
  - unsloth
results/phase1/checkpoint-150/adapter_config.json CHANGED
@@ -33,13 +33,13 @@
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
 
 
36
  "k_proj",
37
- "up_proj",
38
  "down_proj",
39
  "o_proj",
40
- "v_proj",
41
- "gate_proj",
42
- "q_proj"
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
 
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
36
+ "gate_proj",
37
+ "q_proj",
38
  "k_proj",
39
+ "v_proj",
40
  "down_proj",
41
  "o_proj",
42
+ "up_proj"
 
 
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
results/phase1/checkpoint-150/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4f2ea6f37aa971ffcfb6e901dff891103ce7361942994c3c9c4764d57335315a
3
  size 73911112
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32acfb9063f0acb0116e611afccc783f35f4c73cff621c31498366c2b5a56064
3
  size 73911112
results/phase1/checkpoint-150/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ac5a839e2d17ed3303e7dcd56b82670c235a4599960cf3f5bed14b6d0184b795
3
  size 37969669
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f3f1173d331f3f50e08ce052b94de7042ea5521c647c894e9b2fb1fc318e914
3
  size 37969669
results/phase1/checkpoint-150/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:df9f68fada42e0a51f4516e482fb448af8d3c5d6419e5f87e2cb03ca9217eaef
3
  size 14645
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b6ca1b31c2efde8457f21b4590f2ea6c56859d8863161b4e4a23778d49b3bda3
3
  size 14645
results/phase1/checkpoint-150/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f50a1d8632a9c468db3f4baf075e2db44d9458b659d8b9169b83f39c755aa704
3
  size 1465
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c67b5ab96e955e978a225afc8c7ba252edc2deb95112be875bab2391d90e77ab
3
  size 1465
results/phase1/checkpoint-150/trainer_state.json CHANGED
@@ -2,7 +2,7 @@
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
- "epoch": 0.6756756756756757,
6
  "eval_steps": 500,
7
  "global_step": 150,
8
  "is_hyper_param_search": false,
@@ -15,25 +15,25 @@
15
  "clip_ratio/low_mean": 0.0,
16
  "clip_ratio/low_min": 0.0,
17
  "clip_ratio/region_mean": 0.0,
18
- "completion_length": 29.05,
19
- "completions/clipped_ratio": 0.05,
20
- "completions/max_length": 44.2,
21
- "completions/max_terminated_length": 31.0,
22
- "completions/mean_length": 29.05,
23
- "completions/mean_terminated_length": 25.583333587646486,
24
- "completions/min_length": 21.4,
25
- "completions/min_terminated_length": 21.4,
26
- "epoch": 0.02252252252252252,
27
- "frac_reward_zero_std": 0.0,
28
- "grad_norm": 1.2577366828918457,
29
- "kl": 7.414049605358741e-06,
30
  "learning_rate": 1.6666666666666667e-06,
31
- "loss": 2.8312206268310547e-07,
32
- "num_tokens": 21378.0,
33
- "reward": 0.4944999933242798,
34
- "reward_std": 0.15089658200740813,
35
- "rewards/reward_fn/mean": 0.49449999928474425,
36
- "rewards/reward_fn/std": 0.12999328523874282,
37
  "step": 5
38
  },
39
  {
@@ -42,25 +42,25 @@
42
  "clip_ratio/low_mean": 0.0,
43
  "clip_ratio/low_min": 0.0,
44
  "clip_ratio/region_mean": 0.0,
45
- "completion_length": 29.175,
46
  "completions/clipped_ratio": 0.0,
47
- "completions/max_length": 41.8,
48
- "completions/max_terminated_length": 41.8,
49
- "completions/mean_length": 29.175,
50
- "completions/mean_terminated_length": 29.175,
51
- "completions/min_length": 19.2,
52
- "completions/min_terminated_length": 19.2,
53
- "epoch": 0.04504504504504504,
54
  "frac_reward_zero_std": 0.0,
55
- "grad_norm": 1.148110032081604,
56
- "kl": 2.58529256058182e-05,
57
  "learning_rate": 3.7500000000000005e-06,
58
- "loss": 9.343028068542481e-07,
59
- "num_tokens": 42709.0,
60
- "reward": 0.48589999675750734,
61
- "reward_std": 0.14184561967849732,
62
- "rewards/reward_fn/mean": 0.4859000027179718,
63
- "rewards/reward_fn/std": 0.12539554834365846,
64
  "step": 10
65
  },
66
  {
@@ -69,25 +69,25 @@
69
  "clip_ratio/low_mean": 0.0,
70
  "clip_ratio/low_min": 0.0,
71
  "clip_ratio/region_mean": 0.0,
72
- "completion_length": 28.825,
73
  "completions/clipped_ratio": 0.0,
74
- "completions/max_length": 44.2,
75
- "completions/max_terminated_length": 44.2,
76
- "completions/mean_length": 28.825,
77
- "completions/mean_terminated_length": 28.825,
78
- "completions/min_length": 21.2,
79
- "completions/min_terminated_length": 21.2,
80
- "epoch": 0.06756756756756757,
81
  "frac_reward_zero_std": 0.0,
82
- "grad_norm": 1.1520825624465942,
83
- "kl": 0.0001522944080875277,
84
- "learning_rate": 4.952380952380953e-06,
85
- "loss": 6.105005741119385e-06,
86
- "num_tokens": 63008.0,
87
- "reward": 0.4894999861717224,
88
- "reward_std": 0.15768481492996217,
89
- "rewards/reward_fn/mean": 0.48950000405311583,
90
- "rewards/reward_fn/std": 0.14185291826725005,
91
  "step": 15
92
  },
93
  {
@@ -96,25 +96,25 @@
96
  "clip_ratio/low_mean": 0.0,
97
  "clip_ratio/low_min": 0.0,
98
  "clip_ratio/region_mean": 0.0,
99
- "completion_length": 28.25,
100
  "completions/clipped_ratio": 0.0,
101
- "completions/max_length": 43.2,
102
- "completions/max_terminated_length": 43.2,
103
- "completions/mean_length": 28.25,
104
- "completions/mean_terminated_length": 28.25,
105
- "completions/min_length": 21.6,
106
- "completions/min_terminated_length": 21.6,
107
- "epoch": 0.09009009009009009,
108
  "frac_reward_zero_std": 0.0,
109
- "grad_norm": 1.0063226222991943,
110
- "kl": 0.0008983879492006963,
111
- "learning_rate": 4.833333333333333e-06,
112
- "loss": 3.593862056732178e-05,
113
- "num_tokens": 82836.0,
114
- "reward": 0.4881500005722046,
115
- "reward_std": 0.14941166192293168,
116
- "rewards/reward_fn/mean": 0.48814999461174013,
117
- "rewards/reward_fn/std": 0.12791907638311387,
118
  "step": 20
119
  },
120
  {
@@ -123,25 +123,25 @@
123
  "clip_ratio/low_mean": 0.0,
124
  "clip_ratio/low_min": 0.0,
125
  "clip_ratio/region_mean": 0.0,
126
- "completion_length": 27.25,
127
  "completions/clipped_ratio": 0.0,
128
- "completions/max_length": 45.6,
129
- "completions/max_terminated_length": 45.6,
130
- "completions/mean_length": 27.25,
131
- "completions/mean_terminated_length": 27.25,
132
- "completions/min_length": 18.8,
133
- "completions/min_terminated_length": 18.8,
134
- "epoch": 0.11261261261261261,
135
  "frac_reward_zero_std": 0.0,
136
- "grad_norm": 1.3132057189941406,
137
- "kl": 0.003781939600594342,
138
- "learning_rate": 4.714285714285715e-06,
139
- "loss": 0.00015127062797546387,
140
- "num_tokens": 103596.0,
141
- "reward": 0.4680500030517578,
142
- "reward_std": 0.17189764976501465,
143
- "rewards/reward_fn/mean": 0.46804999113082885,
144
- "rewards/reward_fn/std": 0.17421672195196153,
145
  "step": 25
146
  },
147
  {
@@ -150,25 +150,25 @@
150
  "clip_ratio/low_mean": 0.0,
151
  "clip_ratio/low_min": 0.0,
152
  "clip_ratio/region_mean": 0.0,
153
- "completion_length": 25.225,
154
  "completions/clipped_ratio": 0.0,
155
- "completions/max_length": 34.6,
156
- "completions/max_terminated_length": 34.6,
157
- "completions/mean_length": 25.225,
158
- "completions/mean_terminated_length": 25.225,
159
- "completions/min_length": 19.2,
160
- "completions/min_terminated_length": 19.2,
161
- "epoch": 0.13513513513513514,
162
  "frac_reward_zero_std": 0.0,
163
- "grad_norm": 1.5750545263290405,
164
- "kl": 0.006673775642411783,
165
- "learning_rate": 4.595238095238095e-06,
166
- "loss": 0.0002669498324394226,
167
- "num_tokens": 123651.0,
168
- "reward": 0.48049998879432676,
169
- "reward_std": 0.1656044065952301,
170
- "rewards/reward_fn/mean": 0.48049998879432676,
171
- "rewards/reward_fn/std": 0.13908967524766921,
172
  "step": 30
173
  },
174
  {
@@ -177,25 +177,25 @@
177
  "clip_ratio/low_mean": 0.0,
178
  "clip_ratio/low_min": 0.0,
179
  "clip_ratio/region_mean": 0.0,
180
- "completion_length": 28.775,
181
- "completions/clipped_ratio": 0.025,
182
- "completions/max_length": 54.8,
183
- "completions/max_terminated_length": 41.8,
184
- "completions/mean_length": 28.775,
185
- "completions/mean_terminated_length": 26.967857360839844,
186
- "completions/min_length": 15.4,
187
- "completions/min_terminated_length": 15.4,
188
- "epoch": 0.15765765765765766,
189
  "frac_reward_zero_std": 0.0,
190
- "grad_norm": 1.1105402708053589,
191
- "kl": 0.005106111426721327,
192
- "learning_rate": 4.476190476190477e-06,
193
- "loss": 0.0002042025327682495,
194
- "num_tokens": 144316.0,
195
- "reward": 0.4778999924659729,
196
- "reward_std": 0.17012988924980163,
197
- "rewards/reward_fn/mean": 0.47790001034736634,
198
- "rewards/reward_fn/std": 0.14147266000509262,
199
  "step": 35
200
  },
201
  {
@@ -204,25 +204,25 @@
204
  "clip_ratio/low_mean": 0.0,
205
  "clip_ratio/low_min": 0.0,
206
  "clip_ratio/region_mean": 0.0,
207
- "completion_length": 24.4,
208
- "completions/clipped_ratio": 0.0,
209
- "completions/max_length": 33.4,
210
- "completions/max_terminated_length": 33.4,
211
- "completions/mean_length": 24.4,
212
- "completions/mean_terminated_length": 24.4,
213
- "completions/min_length": 18.8,
214
- "completions/min_terminated_length": 18.8,
215
- "epoch": 0.18018018018018017,
216
  "frac_reward_zero_std": 0.0,
217
- "grad_norm": 1.0676746368408203,
218
- "kl": 0.006288998411764624,
219
- "learning_rate": 4.357142857142857e-06,
220
- "loss": 0.00025154203176498414,
221
- "num_tokens": 164334.0,
222
- "reward": 0.5014999985694886,
223
- "reward_std": 0.15358359068632127,
224
- "rewards/reward_fn/mean": 0.5014999985694886,
225
- "rewards/reward_fn/std": 0.13824734836816788,
226
  "step": 40
227
  },
228
  {
@@ -231,25 +231,25 @@
231
  "clip_ratio/low_mean": 0.0,
232
  "clip_ratio/low_min": 0.0,
233
  "clip_ratio/region_mean": 0.0,
234
- "completion_length": 27.35,
235
  "completions/clipped_ratio": 0.0,
236
- "completions/max_length": 43.6,
237
- "completions/max_terminated_length": 43.6,
238
- "completions/mean_length": 27.35,
239
- "completions/mean_terminated_length": 27.35,
240
- "completions/min_length": 19.6,
241
- "completions/min_terminated_length": 19.6,
242
- "epoch": 0.20270270270270271,
243
  "frac_reward_zero_std": 0.0,
244
- "grad_norm": 1.0749404430389404,
245
- "kl": 0.010629063473606948,
246
- "learning_rate": 4.238095238095239e-06,
247
- "loss": 0.0004251018166542053,
248
- "num_tokens": 185140.0,
249
- "reward": 0.524949985742569,
250
- "reward_std": 0.14630039632320405,
251
- "rewards/reward_fn/mean": 0.5249499917030335,
252
- "rewards/reward_fn/std": 0.1460244983434677,
253
  "step": 45
254
  },
255
  {
@@ -258,25 +258,25 @@
258
  "clip_ratio/low_mean": 0.0,
259
  "clip_ratio/low_min": 0.0,
260
  "clip_ratio/region_mean": 0.0,
261
- "completion_length": 25.425,
262
  "completions/clipped_ratio": 0.0,
263
  "completions/max_length": 35.2,
264
  "completions/max_terminated_length": 35.2,
265
- "completions/mean_length": 25.425,
266
- "completions/mean_terminated_length": 25.425,
267
- "completions/min_length": 17.6,
268
- "completions/min_terminated_length": 17.6,
269
- "epoch": 0.22522522522522523,
270
  "frac_reward_zero_std": 0.0,
271
- "grad_norm": 1.4353065490722656,
272
- "kl": 0.008290678875346203,
273
- "learning_rate": 4.119047619047619e-06,
274
- "loss": 0.0003315746784210205,
275
- "num_tokens": 206569.0,
276
- "reward": 0.4908999800682068,
277
- "reward_std": 0.17055415213108063,
278
- "rewards/reward_fn/mean": 0.49089999198913575,
279
- "rewards/reward_fn/std": 0.154354290664196,
280
  "step": 50
281
  },
282
  {
@@ -285,25 +285,25 @@
285
  "clip_ratio/low_mean": 0.0,
286
  "clip_ratio/low_min": 0.0,
287
  "clip_ratio/region_mean": 0.0,
288
- "completion_length": 27.7,
289
- "completions/clipped_ratio": 0.025,
290
- "completions/max_length": 50.6,
291
- "completions/max_terminated_length": 37.2,
292
- "completions/mean_length": 27.7,
293
- "completions/mean_terminated_length": 25.91071434020996,
294
- "completions/min_length": 18.2,
295
- "completions/min_terminated_length": 18.2,
296
- "epoch": 0.24774774774774774,
297
  "frac_reward_zero_std": 0.0,
298
- "grad_norm": 0.8640509843826294,
299
- "kl": 0.012456063569698018,
300
- "learning_rate": 4.000000000000001e-06,
301
- "loss": 0.0004982292652130127,
302
- "num_tokens": 226097.0,
303
- "reward": 0.4839499771595001,
304
- "reward_std": 0.1590283066034317,
305
- "rewards/reward_fn/mean": 0.48394999504089353,
306
- "rewards/reward_fn/std": 0.1380786642432213,
307
  "step": 55
308
  },
309
  {
@@ -312,25 +312,25 @@
312
  "clip_ratio/low_mean": 0.0,
313
  "clip_ratio/low_min": 0.0,
314
  "clip_ratio/region_mean": 0.0,
315
- "completion_length": 26.875,
316
- "completions/clipped_ratio": 0.0,
317
- "completions/max_length": 40.0,
318
- "completions/max_terminated_length": 40.0,
319
- "completions/mean_length": 26.875,
320
- "completions/mean_terminated_length": 26.875,
321
- "completions/min_length": 20.6,
322
- "completions/min_terminated_length": 20.6,
323
- "epoch": 0.2702702702702703,
324
  "frac_reward_zero_std": 0.0,
325
- "grad_norm": 1.0529346466064453,
326
- "kl": 0.007519814093393506,
327
- "learning_rate": 3.880952380952381e-06,
328
- "loss": 0.00030075013637542725,
329
- "num_tokens": 246836.0,
330
- "reward": 0.4914499819278717,
331
- "reward_std": 0.1727461814880371,
332
- "rewards/reward_fn/mean": 0.49144998788833616,
333
- "rewards/reward_fn/std": 0.1568704679608345,
334
  "step": 60
335
  },
336
  {
@@ -339,25 +339,25 @@
339
  "clip_ratio/low_mean": 0.0,
340
  "clip_ratio/low_min": 0.0,
341
  "clip_ratio/region_mean": 0.0,
342
- "completion_length": 27.825,
343
- "completions/clipped_ratio": 0.025,
344
- "completions/max_length": 51.2,
345
- "completions/max_terminated_length": 38.0,
346
- "completions/mean_length": 27.825,
347
- "completions/mean_terminated_length": 26.03214302062988,
348
- "completions/min_length": 17.6,
349
- "completions/min_terminated_length": 17.6,
350
- "epoch": 0.2927927927927928,
351
  "frac_reward_zero_std": 0.0,
352
- "grad_norm": 1.149415373802185,
353
- "kl": 0.00938094645498495,
354
- "learning_rate": 3.761904761904762e-06,
355
- "loss": 0.0003751114010810852,
356
- "num_tokens": 266985.0,
357
- "reward": 0.5003499746322632,
358
- "reward_std": 0.16892780363559723,
359
- "rewards/reward_fn/mean": 0.5003500044345855,
360
- "rewards/reward_fn/std": 0.14663992822170258,
361
  "step": 65
362
  },
363
  {
@@ -366,25 +366,25 @@
366
  "clip_ratio/low_mean": 0.0,
367
  "clip_ratio/low_min": 0.0,
368
  "clip_ratio/region_mean": 0.0,
369
- "completion_length": 27.0,
370
  "completions/clipped_ratio": 0.0,
371
- "completions/max_length": 40.6,
372
- "completions/max_terminated_length": 40.6,
373
- "completions/mean_length": 27.0,
374
- "completions/mean_terminated_length": 27.0,
375
- "completions/min_length": 17.8,
376
- "completions/min_terminated_length": 17.8,
377
- "epoch": 0.3153153153153153,
378
  "frac_reward_zero_std": 0.0,
379
- "grad_norm": 0.9893133640289307,
380
- "kl": 0.010053556466664304,
381
- "learning_rate": 3.642857142857143e-06,
382
- "loss": 0.00040204524993896483,
383
- "num_tokens": 288099.0,
384
- "reward": 0.4705499827861786,
385
- "reward_std": 0.16256383657455445,
386
- "rewards/reward_fn/mean": 0.470550000667572,
387
- "rewards/reward_fn/std": 0.13094386458396912,
388
  "step": 70
389
  },
390
  {
@@ -393,25 +393,25 @@
393
  "clip_ratio/low_mean": 0.0,
394
  "clip_ratio/low_min": 0.0,
395
  "clip_ratio/region_mean": 0.0,
396
- "completion_length": 25.75,
397
- "completions/clipped_ratio": 0.0,
398
- "completions/max_length": 34.8,
399
- "completions/max_terminated_length": 34.8,
400
- "completions/mean_length": 25.75,
401
- "completions/mean_terminated_length": 25.75,
402
- "completions/min_length": 19.8,
403
- "completions/min_terminated_length": 19.8,
404
- "epoch": 0.33783783783783783,
405
  "frac_reward_zero_std": 0.0,
406
- "grad_norm": 1.063340187072754,
407
- "kl": 0.008550376921630232,
408
- "learning_rate": 3.523809523809524e-06,
409
- "loss": 0.00034204572439193725,
410
- "num_tokens": 308531.0,
411
- "reward": 0.5000500082969666,
412
- "reward_std": 0.16242243051528932,
413
- "rewards/reward_fn/mean": 0.5000500082969666,
414
- "rewards/reward_fn/std": 0.14417539685964584,
415
  "step": 75
416
  },
417
  {
@@ -420,25 +420,25 @@
420
  "clip_ratio/low_mean": 0.0,
421
  "clip_ratio/low_min": 0.0,
422
  "clip_ratio/region_mean": 0.0,
423
- "completion_length": 27.025,
424
  "completions/clipped_ratio": 0.0,
425
- "completions/max_length": 40.0,
426
- "completions/max_terminated_length": 40.0,
427
- "completions/mean_length": 27.025,
428
- "completions/mean_terminated_length": 27.025,
429
- "completions/min_length": 19.4,
430
- "completions/min_terminated_length": 19.4,
431
- "epoch": 0.36036036036036034,
432
  "frac_reward_zero_std": 0.0,
433
- "grad_norm": 1.2858343124389648,
434
- "kl": 0.019255010827328077,
435
- "learning_rate": 3.404761904761905e-06,
436
- "loss": 0.0007702425122261047,
437
- "num_tokens": 328874.0,
438
- "reward": 0.5261500000953674,
439
- "reward_std": 0.11886464804410934,
440
- "rewards/reward_fn/mean": 0.5261499881744385,
441
- "rewards/reward_fn/std": 0.12691220343112947,
442
  "step": 80
443
  },
444
  {
@@ -447,25 +447,25 @@
447
  "clip_ratio/low_mean": 0.0,
448
  "clip_ratio/low_min": 0.0,
449
  "clip_ratio/region_mean": 0.0,
450
- "completion_length": 25.475,
451
  "completions/clipped_ratio": 0.0,
452
- "completions/max_length": 33.0,
453
- "completions/max_terminated_length": 33.0,
454
- "completions/mean_length": 25.475,
455
- "completions/mean_terminated_length": 25.475,
456
- "completions/min_length": 19.0,
457
- "completions/min_terminated_length": 19.0,
458
- "epoch": 0.38288288288288286,
459
  "frac_reward_zero_std": 0.0,
460
- "grad_norm": 1.118030309677124,
461
- "kl": 0.01090274843845691,
462
- "learning_rate": 3.285714285714286e-06,
463
- "loss": 0.00043606162071228025,
464
- "num_tokens": 348373.0,
465
- "reward": 0.49950000643730164,
466
- "reward_std": 0.15655344128608703,
467
- "rewards/reward_fn/mean": 0.49950000643730164,
468
- "rewards/reward_fn/std": 0.14320258051156998,
469
  "step": 85
470
  },
471
  {
@@ -474,25 +474,25 @@
474
  "clip_ratio/low_mean": 0.0,
475
  "clip_ratio/low_min": 0.0,
476
  "clip_ratio/region_mean": 0.0,
477
- "completion_length": 27.9,
478
- "completions/clipped_ratio": 0.05,
479
- "completions/max_length": 55.4,
480
- "completions/max_terminated_length": 31.6,
481
- "completions/mean_length": 27.9,
482
- "completions/mean_terminated_length": 24.28571472167969,
483
- "completions/min_length": 18.0,
484
- "completions/min_terminated_length": 18.0,
485
- "epoch": 0.40540540540540543,
486
  "frac_reward_zero_std": 0.0,
487
- "grad_norm": 0.6538166403770447,
488
- "kl": 0.008556566922561615,
489
- "learning_rate": 3.1666666666666667e-06,
490
- "loss": 0.0003422081470489502,
491
- "num_tokens": 368815.0,
492
- "reward": 0.5215500056743622,
493
- "reward_std": 0.16581654250621797,
494
- "rewards/reward_fn/mean": 0.5215500056743622,
495
- "rewards/reward_fn/std": 0.15472394227981567,
496
  "step": 90
497
  },
498
  {
@@ -501,25 +501,25 @@
501
  "clip_ratio/low_mean": 0.0,
502
  "clip_ratio/low_min": 0.0,
503
  "clip_ratio/region_mean": 0.0,
504
- "completion_length": 25.75,
505
  "completions/clipped_ratio": 0.0,
506
- "completions/max_length": 32.4,
507
- "completions/max_terminated_length": 32.4,
508
- "completions/mean_length": 25.75,
509
- "completions/mean_terminated_length": 25.75,
510
  "completions/min_length": 20.8,
511
  "completions/min_terminated_length": 20.8,
512
- "epoch": 0.42792792792792794,
513
  "frac_reward_zero_std": 0.0,
514
- "grad_norm": 1.180251955986023,
515
- "kl": 0.014757526386529208,
516
- "learning_rate": 3.047619047619048e-06,
517
- "loss": 0.0005902737379074096,
518
- "num_tokens": 388761.0,
519
- "reward": 0.514000016450882,
520
- "reward_std": 0.1465125188231468,
521
- "rewards/reward_fn/mean": 0.5139999985694885,
522
- "rewards/reward_fn/std": 0.14701166301965712,
523
  "step": 95
524
  },
525
  {
@@ -528,25 +528,25 @@
528
  "clip_ratio/low_mean": 0.0,
529
  "clip_ratio/low_min": 0.0,
530
  "clip_ratio/region_mean": 0.0,
531
- "completion_length": 26.35,
532
- "completions/clipped_ratio": 0.025,
533
- "completions/max_length": 46.6,
534
- "completions/max_terminated_length": 32.4,
535
- "completions/mean_length": 26.35,
536
- "completions/mean_terminated_length": 24.517857360839844,
537
- "completions/min_length": 17.8,
538
- "completions/min_terminated_length": 17.8,
539
- "epoch": 0.45045045045045046,
540
  "frac_reward_zero_std": 0.0,
541
- "grad_norm": 0.8863919973373413,
542
- "kl": 0.017311154678463937,
543
- "learning_rate": 2.928571428571429e-06,
544
- "loss": 0.0006924748420715332,
545
- "num_tokens": 408309.0,
546
- "reward": 0.5527999818325042,
547
- "reward_std": 0.1593818634748459,
548
- "rewards/reward_fn/mean": 0.5527999699115753,
549
- "rewards/reward_fn/std": 0.16823574155569077,
550
  "step": 100
551
  },
552
  {
@@ -555,25 +555,25 @@
555
  "clip_ratio/low_mean": 0.0,
556
  "clip_ratio/low_min": 0.0,
557
  "clip_ratio/region_mean": 0.0,
558
- "completion_length": 28.675,
559
- "completions/clipped_ratio": 0.05,
560
- "completions/max_length": 45.6,
561
- "completions/max_terminated_length": 33.2,
562
- "completions/mean_length": 28.675,
563
- "completions/mean_terminated_length": 25.291666793823243,
564
- "completions/min_length": 20.6,
565
- "completions/min_terminated_length": 20.6,
566
- "epoch": 0.47297297297297297,
567
  "frac_reward_zero_std": 0.0,
568
- "grad_norm": 0.7957925200462341,
569
- "kl": 0.021478149453469087,
570
- "learning_rate": 2.8095238095238096e-06,
571
- "loss": 0.0008591204881668091,
572
- "num_tokens": 429872.0,
573
- "reward": 0.5294999718666077,
574
- "reward_std": 0.15301789939403534,
575
- "rewards/reward_fn/mean": 0.5294999837875366,
576
- "rewards/reward_fn/std": 0.15534335970878602,
577
  "step": 105
578
  },
579
  {
@@ -582,25 +582,25 @@
582
  "clip_ratio/low_mean": 0.0,
583
  "clip_ratio/low_min": 0.0,
584
  "clip_ratio/region_mean": 0.0,
585
- "completion_length": 27.325,
586
  "completions/clipped_ratio": 0.0,
587
- "completions/max_length": 47.2,
588
- "completions/max_terminated_length": 47.2,
589
- "completions/mean_length": 27.325,
590
- "completions/mean_terminated_length": 27.325,
591
- "completions/min_length": 19.2,
592
- "completions/min_terminated_length": 19.2,
593
- "epoch": 0.4954954954954955,
594
  "frac_reward_zero_std": 0.0,
595
- "grad_norm": 1.187671184539795,
596
- "kl": 0.01666262859798735,
597
- "learning_rate": 2.6904761904761906e-06,
598
- "loss": 0.0006665512919425964,
599
- "num_tokens": 450229.0,
600
- "reward": 0.5364000022411346,
601
- "reward_std": 0.16447303295135499,
602
- "rewards/reward_fn/mean": 0.5364000022411346,
603
- "rewards/reward_fn/std": 0.17169796973466872,
604
  "step": 110
605
  },
606
  {
@@ -609,25 +609,25 @@
609
  "clip_ratio/low_mean": 0.0,
610
  "clip_ratio/low_min": 0.0,
611
  "clip_ratio/region_mean": 0.0,
612
- "completion_length": 27.6,
613
- "completions/clipped_ratio": 0.025,
614
- "completions/max_length": 46.6,
615
- "completions/max_terminated_length": 37.2,
616
- "completions/mean_length": 27.6,
617
- "completions/mean_terminated_length": 25.853571701049805,
618
- "completions/min_length": 18.8,
619
- "completions/min_terminated_length": 18.8,
620
- "epoch": 0.5180180180180181,
621
  "frac_reward_zero_std": 0.0,
622
- "grad_norm": 1.3785444498062134,
623
- "kl": 0.015608730388339608,
624
- "learning_rate": 2.571428571428571e-06,
625
- "loss": 0.0006243243813514709,
626
- "num_tokens": 470911.0,
627
- "reward": 0.5131499886512756,
628
- "reward_std": 0.15916974246501922,
629
- "rewards/reward_fn/mean": 0.513150018453598,
630
- "rewards/reward_fn/std": 0.15986726433038712,
631
  "step": 115
632
  },
633
  {
@@ -636,25 +636,25 @@
636
  "clip_ratio/low_mean": 0.0,
637
  "clip_ratio/low_min": 0.0,
638
  "clip_ratio/region_mean": 0.0,
639
- "completion_length": 26.725,
640
- "completions/clipped_ratio": 0.025,
641
- "completions/max_length": 43.8,
642
- "completions/max_terminated_length": 30.4,
643
- "completions/mean_length": 26.725,
644
- "completions/mean_terminated_length": 24.953571701049803,
645
- "completions/min_length": 20.4,
646
- "completions/min_terminated_length": 20.4,
647
- "epoch": 0.5405405405405406,
648
- "frac_reward_zero_std": 0.0,
649
- "grad_norm": 0.8001337647438049,
650
- "kl": 0.016426509176380933,
651
- "learning_rate": 2.4523809523809526e-06,
652
- "loss": 0.0006570681929588318,
653
- "num_tokens": 490616.0,
654
- "reward": 0.5314500093460083,
655
- "reward_std": 0.16807928085327148,
656
- "rewards/reward_fn/mean": 0.5314499974250794,
657
- "rewards/reward_fn/std": 0.16840155124664308,
658
  "step": 120
659
  },
660
  {
@@ -663,25 +663,25 @@
663
  "clip_ratio/low_mean": 0.0,
664
  "clip_ratio/low_min": 0.0,
665
  "clip_ratio/region_mean": 0.0,
666
- "completion_length": 25.8,
667
- "completions/clipped_ratio": 0.0,
668
- "completions/max_length": 39.8,
669
- "completions/max_terminated_length": 39.8,
670
- "completions/mean_length": 25.8,
671
- "completions/mean_terminated_length": 25.8,
672
- "completions/min_length": 17.6,
673
- "completions/min_terminated_length": 17.6,
674
- "epoch": 0.5630630630630631,
675
  "frac_reward_zero_std": 0.0,
676
- "grad_norm": 1.0382933616638184,
677
- "kl": 0.016185989990481174,
678
- "learning_rate": 2.3333333333333336e-06,
679
- "loss": 0.000647495687007904,
680
- "num_tokens": 509092.0,
681
- "reward": 0.5144500017166138,
682
- "reward_std": 0.15323003232479096,
683
- "rewards/reward_fn/mean": 0.5144500017166138,
684
- "rewards/reward_fn/std": 0.13980331867933274,
685
  "step": 125
686
  },
687
  {
@@ -690,25 +690,25 @@
690
  "clip_ratio/low_mean": 0.0,
691
  "clip_ratio/low_min": 0.0,
692
  "clip_ratio/region_mean": 0.0,
693
- "completion_length": 26.275,
694
  "completions/clipped_ratio": 0.0,
695
- "completions/max_length": 36.4,
696
- "completions/max_terminated_length": 36.4,
697
- "completions/mean_length": 26.275,
698
- "completions/mean_terminated_length": 26.275,
699
- "completions/min_length": 19.6,
700
- "completions/min_terminated_length": 19.6,
701
- "epoch": 0.5855855855855856,
702
  "frac_reward_zero_std": 0.0,
703
- "grad_norm": 1.3675200939178467,
704
- "kl": 0.01404028357937932,
705
- "learning_rate": 2.2142857142857146e-06,
706
- "loss": 0.0005615666508674621,
707
- "num_tokens": 531083.0,
708
- "reward": 0.5205999791622162,
709
- "reward_std": 0.1824335426092148,
710
- "rewards/reward_fn/mean": 0.520600003004074,
711
- "rewards/reward_fn/std": 0.16652330607175828,
712
  "step": 130
713
  },
714
  {
@@ -717,25 +717,25 @@
717
  "clip_ratio/low_mean": 0.0,
718
  "clip_ratio/low_min": 0.0,
719
  "clip_ratio/region_mean": 0.0,
720
- "completion_length": 27.025,
721
  "completions/clipped_ratio": 0.0,
722
- "completions/max_length": 41.0,
723
- "completions/max_terminated_length": 41.0,
724
- "completions/mean_length": 27.025,
725
- "completions/mean_terminated_length": 27.025,
726
- "completions/min_length": 20.2,
727
- "completions/min_terminated_length": 20.2,
728
- "epoch": 0.6081081081081081,
729
  "frac_reward_zero_std": 0.0,
730
- "grad_norm": 1.5435948371887207,
731
- "kl": 0.01980702848522924,
732
- "learning_rate": 2.0952380952380955e-06,
733
- "loss": 0.0007923290133476258,
734
- "num_tokens": 551052.0,
735
- "reward": 0.5506500005722046,
736
- "reward_std": 0.15252292901277542,
737
- "rewards/reward_fn/mean": 0.5506499886512757,
738
- "rewards/reward_fn/std": 0.16045962125062943,
739
  "step": 135
740
  },
741
  {
@@ -744,25 +744,25 @@
744
  "clip_ratio/low_mean": 0.0,
745
  "clip_ratio/low_min": 0.0,
746
  "clip_ratio/region_mean": 0.0,
747
- "completion_length": 24.875,
748
  "completions/clipped_ratio": 0.0,
749
- "completions/max_length": 30.0,
750
- "completions/max_terminated_length": 30.0,
751
- "completions/mean_length": 24.875,
752
- "completions/mean_terminated_length": 24.875,
753
- "completions/min_length": 19.8,
754
- "completions/min_terminated_length": 19.8,
755
- "epoch": 0.6306306306306306,
756
  "frac_reward_zero_std": 0.0,
757
- "grad_norm": 1.0660195350646973,
758
- "kl": 0.014228896767599508,
759
- "learning_rate": 1.976190476190476e-06,
760
- "loss": 0.0005690708756446838,
761
- "num_tokens": 570993.0,
762
- "reward": 0.5141499876976013,
763
- "reward_std": 0.1573312520980835,
764
- "rewards/reward_fn/mean": 0.5141499936580658,
765
- "rewards/reward_fn/std": 0.14991891831159593,
766
  "step": 140
767
  },
768
  {
@@ -771,25 +771,25 @@
771
  "clip_ratio/low_mean": 0.0,
772
  "clip_ratio/low_min": 0.0,
773
  "clip_ratio/region_mean": 0.0,
774
- "completion_length": 24.775,
775
- "completions/clipped_ratio": 0.0,
776
- "completions/max_length": 35.4,
777
- "completions/max_terminated_length": 35.4,
778
- "completions/mean_length": 24.775,
779
- "completions/mean_terminated_length": 24.775,
780
- "completions/min_length": 17.0,
781
- "completions/min_terminated_length": 17.0,
782
- "epoch": 0.6531531531531531,
783
  "frac_reward_zero_std": 0.0,
784
- "grad_norm": 0.9134359359741211,
785
- "kl": 0.015506639698287472,
786
- "learning_rate": 1.8571428571428573e-06,
787
- "loss": 0.0006200879812240601,
788
- "num_tokens": 591118.0,
789
- "reward": 0.5515999913215637,
790
- "reward_std": 0.1493409514427185,
791
- "rewards/reward_fn/mean": 0.5516000032424927,
792
- "rewards/reward_fn/std": 0.15984065383672713,
793
  "step": 145
794
  },
795
  {
@@ -798,31 +798,31 @@
798
  "clip_ratio/low_mean": 0.0,
799
  "clip_ratio/low_min": 0.0,
800
  "clip_ratio/region_mean": 0.0,
801
- "completion_length": 25.45,
802
  "completions/clipped_ratio": 0.0,
803
- "completions/max_length": 34.6,
804
- "completions/max_terminated_length": 34.6,
805
- "completions/mean_length": 25.45,
806
- "completions/mean_terminated_length": 25.45,
807
- "completions/min_length": 20.6,
808
- "completions/min_terminated_length": 20.6,
809
- "epoch": 0.6756756756756757,
810
  "frac_reward_zero_std": 0.0,
811
- "grad_norm": 1.011871099472046,
812
- "kl": 0.018102088455634657,
813
- "learning_rate": 1.738095238095238e-06,
814
- "loss": 0.0007240131497383117,
815
- "num_tokens": 611492.0,
816
- "reward": 0.5036499917507171,
817
- "reward_std": 0.14262343645095826,
818
- "rewards/reward_fn/mean": 0.5036500096321106,
819
- "rewards/reward_fn/std": 0.14135744124650956,
820
  "step": 150
821
  }
822
  ],
823
  "logging_steps": 5,
824
- "max_steps": 222,
825
- "num_input_tokens_seen": 611492,
826
  "num_train_epochs": 1,
827
  "save_steps": 50,
828
  "stateful_callbacks": {
 
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
+ "epoch": 0.6666666666666666,
6
  "eval_steps": 500,
7
  "global_step": 150,
8
  "is_hyper_param_search": false,
 
15
  "clip_ratio/low_mean": 0.0,
16
  "clip_ratio/low_min": 0.0,
17
  "clip_ratio/region_mean": 0.0,
18
+ "completion_length": 25.075,
19
+ "completions/clipped_ratio": 0.0,
20
+ "completions/max_length": 30.8,
21
+ "completions/max_terminated_length": 30.8,
22
+ "completions/mean_length": 25.075,
23
+ "completions/mean_terminated_length": 25.075,
24
+ "completions/min_length": 19.4,
25
+ "completions/min_terminated_length": 19.4,
26
+ "epoch": 0.022222222222222223,
27
+ "frac_reward_zero_std": 0.6,
28
+ "grad_norm": 1.3092279434204102,
29
+ "kl": 0.03290070100920275,
30
  "learning_rate": 1.6666666666666667e-06,
31
+ "loss": 0.0013160213828086853,
32
+ "num_tokens": 20275.0,
33
+ "reward": 0.48624999523162843,
34
+ "reward_std": 0.05833630859851837,
35
+ "rewards/reward_fn/mean": 0.4862500011920929,
36
+ "rewards/reward_fn/std": 0.04605271518230438,
37
  "step": 5
38
  },
39
  {
 
42
  "clip_ratio/low_mean": 0.0,
43
  "clip_ratio/low_min": 0.0,
44
  "clip_ratio/region_mean": 0.0,
45
+ "completion_length": 25.925,
46
  "completions/clipped_ratio": 0.0,
47
+ "completions/max_length": 33.0,
48
+ "completions/max_terminated_length": 33.0,
49
+ "completions/mean_length": 25.925,
50
+ "completions/mean_terminated_length": 25.925,
51
+ "completions/min_length": 19.4,
52
+ "completions/min_terminated_length": 19.4,
53
+ "epoch": 0.044444444444444446,
54
  "frac_reward_zero_std": 0.0,
55
+ "grad_norm": 1.4204450845718384,
56
+ "kl": 0.040759827199508436,
57
  "learning_rate": 3.7500000000000005e-06,
58
+ "loss": 0.0016303554177284241,
59
+ "num_tokens": 41560.0,
60
+ "reward": 0.5283999860286712,
61
+ "reward_std": 0.17366542518138886,
62
+ "rewards/reward_fn/mean": 0.5283999919891358,
63
+ "rewards/reward_fn/std": 0.16809422075748442,
64
  "step": 10
65
  },
66
  {
 
69
  "clip_ratio/low_mean": 0.0,
70
  "clip_ratio/low_min": 0.0,
71
  "clip_ratio/region_mean": 0.0,
72
+ "completion_length": 27.45,
73
  "completions/clipped_ratio": 0.0,
74
+ "completions/max_length": 33.6,
75
+ "completions/max_terminated_length": 33.6,
76
+ "completions/mean_length": 27.45,
77
+ "completions/mean_terminated_length": 27.45,
78
+ "completions/min_length": 24.0,
79
+ "completions/min_terminated_length": 24.0,
80
+ "epoch": 0.06666666666666667,
81
  "frac_reward_zero_std": 0.0,
82
+ "grad_norm": 0.9515441656112671,
83
+ "kl": 0.049849181214813146,
84
+ "learning_rate": 4.953051643192488e-06,
85
+ "loss": 0.0019938603043556215,
86
+ "num_tokens": 62246.0,
87
+ "reward": 0.4975499987602234,
88
+ "reward_std": 0.1564827263355255,
89
+ "rewards/reward_fn/mean": 0.49755001068115234,
90
+ "rewards/reward_fn/std": 0.14618260115385057,
91
  "step": 15
92
  },
93
  {
 
96
  "clip_ratio/low_mean": 0.0,
97
  "clip_ratio/low_min": 0.0,
98
  "clip_ratio/region_mean": 0.0,
99
+ "completion_length": 26.525,
100
  "completions/clipped_ratio": 0.0,
101
+ "completions/max_length": 33.0,
102
+ "completions/max_terminated_length": 33.0,
103
+ "completions/mean_length": 26.525,
104
+ "completions/mean_terminated_length": 26.525,
105
+ "completions/min_length": 22.2,
106
+ "completions/min_terminated_length": 22.2,
107
+ "epoch": 0.08888888888888889,
108
  "frac_reward_zero_std": 0.0,
109
+ "grad_norm": 0.6862999200820923,
110
+ "kl": 0.04350169296376407,
111
+ "learning_rate": 4.835680751173709e-06,
112
+ "loss": 0.0017400771379470826,
113
+ "num_tokens": 81479.0,
114
+ "reward": 0.5159000098705292,
115
+ "reward_std": 0.17465537190437316,
116
+ "rewards/reward_fn/mean": 0.5158999919891357,
117
+ "rewards/reward_fn/std": 0.16455023884773254,
118
  "step": 20
119
  },
120
  {
 
123
  "clip_ratio/low_mean": 0.0,
124
  "clip_ratio/low_min": 0.0,
125
  "clip_ratio/region_mean": 0.0,
126
+ "completion_length": 28.0,
127
  "completions/clipped_ratio": 0.0,
128
+ "completions/max_length": 40.6,
129
+ "completions/max_terminated_length": 40.6,
130
+ "completions/mean_length": 28.0,
131
+ "completions/mean_terminated_length": 28.0,
132
+ "completions/min_length": 22.6,
133
+ "completions/min_terminated_length": 22.6,
134
+ "epoch": 0.1111111111111111,
135
  "frac_reward_zero_std": 0.0,
136
+ "grad_norm": 1.7516958713531494,
137
+ "kl": 0.05824573401478119,
138
+ "learning_rate": 4.71830985915493e-06,
139
+ "loss": 0.002329717576503754,
140
+ "num_tokens": 102833.0,
141
+ "reward": 0.5547999978065491,
142
+ "reward_std": 0.18328206837177277,
143
+ "rewards/reward_fn/mean": 0.5547999858856201,
144
+ "rewards/reward_fn/std": 0.1974634051322937,
145
  "step": 25
146
  },
147
  {
 
150
  "clip_ratio/low_mean": 0.0,
151
  "clip_ratio/low_min": 0.0,
152
  "clip_ratio/region_mean": 0.0,
153
+ "completion_length": 28.675,
154
  "completions/clipped_ratio": 0.0,
155
+ "completions/max_length": 37.4,
156
+ "completions/max_terminated_length": 37.4,
157
+ "completions/mean_length": 28.675,
158
+ "completions/mean_terminated_length": 28.675,
159
+ "completions/min_length": 22.2,
160
+ "completions/min_terminated_length": 22.2,
161
+ "epoch": 0.13333333333333333,
162
  "frac_reward_zero_std": 0.0,
163
+ "grad_norm": 1.5611271858215332,
164
+ "kl": 0.09752920938190072,
165
+ "learning_rate": 4.60093896713615e-06,
166
+ "loss": 0.0039011374115943908,
167
+ "num_tokens": 123876.0,
168
+ "reward": 0.5165499806404114,
169
+ "reward_std": 0.16765501499176025,
170
+ "rewards/reward_fn/mean": 0.5165499806404114,
171
+ "rewards/reward_fn/std": 0.16748485416173936,
172
  "step": 30
173
  },
174
  {
 
177
  "clip_ratio/low_mean": 0.0,
178
  "clip_ratio/low_min": 0.0,
179
  "clip_ratio/region_mean": 0.0,
180
+ "completion_length": 27.475,
181
+ "completions/clipped_ratio": 0.0,
182
+ "completions/max_length": 36.8,
183
+ "completions/max_terminated_length": 36.8,
184
+ "completions/mean_length": 27.475,
185
+ "completions/mean_terminated_length": 27.475,
186
+ "completions/min_length": 20.6,
187
+ "completions/min_terminated_length": 20.6,
188
+ "epoch": 0.15555555555555556,
189
  "frac_reward_zero_std": 0.0,
190
+ "grad_norm": 0.9550014138221741,
191
+ "kl": 0.08032013729680329,
192
+ "learning_rate": 4.483568075117371e-06,
193
+ "loss": 0.003212757408618927,
194
+ "num_tokens": 145019.0,
195
+ "reward": 0.5004499793052674,
196
+ "reward_std": 0.18391846716403962,
197
+ "rewards/reward_fn/mean": 0.5004500031471253,
198
+ "rewards/reward_fn/std": 0.16948954164981841,
199
  "step": 35
200
  },
201
  {
 
204
  "clip_ratio/low_mean": 0.0,
205
  "clip_ratio/low_min": 0.0,
206
  "clip_ratio/region_mean": 0.0,
207
+ "completion_length": 32.125,
208
+ "completions/clipped_ratio": 0.025,
209
+ "completions/max_length": 54.6,
210
+ "completions/max_terminated_length": 46.0,
211
+ "completions/mean_length": 32.125,
212
+ "completions/mean_terminated_length": 30.521428680419923,
213
+ "completions/min_length": 23.2,
214
+ "completions/min_terminated_length": 23.2,
215
+ "epoch": 0.17777777777777778,
216
  "frac_reward_zero_std": 0.0,
217
+ "grad_norm": 1.1203869581222534,
218
+ "kl": 0.08563535290304572,
219
+ "learning_rate": 4.3661971830985915e-06,
220
+ "loss": 0.003425435721874237,
221
+ "num_tokens": 164800.0,
222
+ "reward": 0.5472500026226044,
223
+ "reward_std": 0.19579786211252212,
224
+ "rewards/reward_fn/mean": 0.5472500085830688,
225
+ "rewards/reward_fn/std": 0.19857290089130403,
226
  "step": 40
227
  },
228
  {
 
231
  "clip_ratio/low_mean": 0.0,
232
  "clip_ratio/low_min": 0.0,
233
  "clip_ratio/region_mean": 0.0,
234
+ "completion_length": 27.875,
235
  "completions/clipped_ratio": 0.0,
236
+ "completions/max_length": 35.0,
237
+ "completions/max_terminated_length": 35.0,
238
+ "completions/mean_length": 27.875,
239
+ "completions/mean_terminated_length": 27.875,
240
+ "completions/min_length": 21.4,
241
+ "completions/min_terminated_length": 21.4,
242
+ "epoch": 0.2,
243
  "frac_reward_zero_std": 0.0,
244
+ "grad_norm": 0.95042884349823,
245
+ "kl": 0.07295196709455923,
246
+ "learning_rate": 4.248826291079813e-06,
247
+ "loss": 0.0029180020093917845,
248
+ "num_tokens": 185761.0,
249
+ "reward": 0.5225999832153321,
250
+ "reward_std": 0.16489729881286622,
251
+ "rewards/reward_fn/mean": 0.5226000070571899,
252
+ "rewards/reward_fn/std": 0.16541497856378556,
253
  "step": 45
254
  },
255
  {
 
258
  "clip_ratio/low_mean": 0.0,
259
  "clip_ratio/low_min": 0.0,
260
  "clip_ratio/region_mean": 0.0,
261
+ "completion_length": 27.875,
262
  "completions/clipped_ratio": 0.0,
263
  "completions/max_length": 35.2,
264
  "completions/max_terminated_length": 35.2,
265
+ "completions/mean_length": 27.875,
266
+ "completions/mean_terminated_length": 27.875,
267
+ "completions/min_length": 21.4,
268
+ "completions/min_terminated_length": 21.4,
269
+ "epoch": 0.2222222222222222,
270
  "frac_reward_zero_std": 0.0,
271
+ "grad_norm": 1.1037548780441284,
272
+ "kl": 0.08246680488809943,
273
+ "learning_rate": 4.131455399061034e-06,
274
+ "loss": 0.0032985761761665346,
275
+ "num_tokens": 206674.0,
276
+ "reward": 0.5245000004768372,
277
+ "reward_std": 0.1653215616941452,
278
+ "rewards/reward_fn/mean": 0.5245000004768372,
279
+ "rewards/reward_fn/std": 0.16877340227365495,
280
  "step": 50
281
  },
282
  {
 
285
  "clip_ratio/low_mean": 0.0,
286
  "clip_ratio/low_min": 0.0,
287
  "clip_ratio/region_mean": 0.0,
288
+ "completion_length": 29.225,
289
+ "completions/clipped_ratio": 0.0,
290
+ "completions/max_length": 47.4,
291
+ "completions/max_terminated_length": 47.4,
292
+ "completions/mean_length": 29.225,
293
+ "completions/mean_terminated_length": 29.225,
294
+ "completions/min_length": 20.2,
295
+ "completions/min_terminated_length": 20.2,
296
+ "epoch": 0.24444444444444444,
297
  "frac_reward_zero_std": 0.0,
298
+ "grad_norm": 0.8115681409835815,
299
+ "kl": 0.08297618771903217,
300
+ "learning_rate": 4.014084507042254e-06,
301
+ "loss": 0.0033189669251441956,
302
+ "num_tokens": 227351.0,
303
+ "reward": 0.5301500022411346,
304
+ "reward_std": 0.20612163245677947,
305
+ "rewards/reward_fn/mean": 0.5301500082015991,
306
+ "rewards/reward_fn/std": 0.19000594317913055,
307
  "step": 55
308
  },
309
  {
 
312
  "clip_ratio/low_mean": 0.0,
313
  "clip_ratio/low_min": 0.0,
314
  "clip_ratio/region_mean": 0.0,
315
+ "completion_length": 27.475,
316
+ "completions/clipped_ratio": 0.025,
317
+ "completions/max_length": 45.0,
318
+ "completions/max_terminated_length": 31.8,
319
+ "completions/mean_length": 27.475,
320
+ "completions/mean_terminated_length": 25.646428680419923,
321
+ "completions/min_length": 18.8,
322
+ "completions/min_terminated_length": 18.8,
323
+ "epoch": 0.26666666666666666,
324
  "frac_reward_zero_std": 0.0,
325
+ "grad_norm": 1.1722970008850098,
326
+ "kl": 0.06283304298995063,
327
+ "learning_rate": 3.896713615023475e-06,
328
+ "loss": 0.002513286471366882,
329
+ "num_tokens": 246464.0,
330
+ "reward": 0.513349997997284,
331
+ "reward_std": 0.18999958634376526,
332
+ "rewards/reward_fn/mean": 0.5133499920368194,
333
+ "rewards/reward_fn/std": 0.1748345360159874,
334
  "step": 60
335
  },
336
  {
 
339
  "clip_ratio/low_mean": 0.0,
340
  "clip_ratio/low_min": 0.0,
341
  "clip_ratio/region_mean": 0.0,
342
+ "completion_length": 26.075,
343
+ "completions/clipped_ratio": 0.0,
344
+ "completions/max_length": 34.2,
345
+ "completions/max_terminated_length": 34.2,
346
+ "completions/mean_length": 26.075,
347
+ "completions/mean_terminated_length": 26.075,
348
+ "completions/min_length": 19.4,
349
+ "completions/min_terminated_length": 19.4,
350
+ "epoch": 0.28888888888888886,
351
  "frac_reward_zero_std": 0.0,
352
+ "grad_norm": 1.2825119495391846,
353
+ "kl": 0.07132846353342756,
354
+ "learning_rate": 3.779342723004695e-06,
355
+ "loss": 0.0028530970215797425,
356
+ "num_tokens": 267279.0,
357
+ "reward": 0.5383000016212464,
358
+ "reward_std": 0.20746512711048126,
359
+ "rewards/reward_fn/mean": 0.5383000195026397,
360
+ "rewards/reward_fn/std": 0.1835445523262024,
361
  "step": 65
362
  },
363
  {
 
366
  "clip_ratio/low_mean": 0.0,
367
  "clip_ratio/low_min": 0.0,
368
  "clip_ratio/region_mean": 0.0,
369
+ "completion_length": 27.925,
370
  "completions/clipped_ratio": 0.0,
371
+ "completions/max_length": 44.0,
372
+ "completions/max_terminated_length": 44.0,
373
+ "completions/mean_length": 27.925,
374
+ "completions/mean_terminated_length": 27.925,
375
+ "completions/min_length": 22.0,
376
+ "completions/min_terminated_length": 22.0,
377
+ "epoch": 0.3111111111111111,
378
  "frac_reward_zero_std": 0.0,
379
+ "grad_norm": 0.5832120776176453,
380
+ "kl": 0.05197672065114602,
381
+ "learning_rate": 3.6619718309859158e-06,
382
+ "loss": 0.0020790368318557738,
383
+ "num_tokens": 287984.0,
384
+ "reward": 0.5424999892711639,
385
+ "reward_std": 0.18002938330173493,
386
+ "rewards/reward_fn/mean": 0.5424999952316284,
387
+ "rewards/reward_fn/std": 0.18769851326942444,
388
  "step": 70
389
  },
390
  {
 
393
  "clip_ratio/low_mean": 0.0,
394
  "clip_ratio/low_min": 0.0,
395
  "clip_ratio/region_mean": 0.0,
396
+ "completion_length": 27.6,
397
+ "completions/clipped_ratio": 0.025,
398
+ "completions/max_length": 44.6,
399
+ "completions/max_terminated_length": 30.6,
400
+ "completions/mean_length": 27.6,
401
+ "completions/mean_terminated_length": 25.807143020629884,
402
+ "completions/min_length": 22.2,
403
+ "completions/min_terminated_length": 22.2,
404
+ "epoch": 0.3333333333333333,
405
  "frac_reward_zero_std": 0.0,
406
+ "grad_norm": 0.6677760481834412,
407
+ "kl": 0.03671608620206825,
408
+ "learning_rate": 3.5446009389671364e-06,
409
+ "loss": 0.0014685407280921937,
410
+ "num_tokens": 307800.0,
411
+ "reward": 0.5715999841690064,
412
+ "reward_std": 0.18639334440231323,
413
+ "rewards/reward_fn/mean": 0.5715999960899353,
414
+ "rewards/reward_fn/std": 0.19429495334625244,
415
  "step": 75
416
  },
417
  {
 
420
  "clip_ratio/low_mean": 0.0,
421
  "clip_ratio/low_min": 0.0,
422
  "clip_ratio/region_mean": 0.0,
423
+ "completion_length": 25.6,
424
  "completions/clipped_ratio": 0.0,
425
+ "completions/max_length": 30.2,
426
+ "completions/max_terminated_length": 30.2,
427
+ "completions/mean_length": 25.6,
428
+ "completions/mean_terminated_length": 25.6,
429
+ "completions/min_length": 21.4,
430
+ "completions/min_terminated_length": 21.4,
431
+ "epoch": 0.35555555555555557,
432
  "frac_reward_zero_std": 0.0,
433
+ "grad_norm": 0.7261527180671692,
434
+ "kl": 0.05616153636947274,
435
+ "learning_rate": 3.427230046948357e-06,
436
+ "loss": 0.0022463813424110413,
437
+ "num_tokens": 327794.0,
438
+ "reward": 0.5464499950408935,
439
+ "reward_std": 0.18264567852020264,
440
+ "rewards/reward_fn/mean": 0.5464499950408935,
441
+ "rewards/reward_fn/std": 0.17480863779783248,
442
  "step": 80
443
  },
444
  {
 
447
  "clip_ratio/low_mean": 0.0,
448
  "clip_ratio/low_min": 0.0,
449
  "clip_ratio/region_mean": 0.0,
450
+ "completion_length": 24.075,
451
  "completions/clipped_ratio": 0.0,
452
+ "completions/max_length": 29.6,
453
+ "completions/max_terminated_length": 29.6,
454
+ "completions/mean_length": 24.075,
455
+ "completions/mean_terminated_length": 24.075,
456
+ "completions/min_length": 16.6,
457
+ "completions/min_terminated_length": 16.6,
458
+ "epoch": 0.37777777777777777,
459
  "frac_reward_zero_std": 0.0,
460
+ "grad_norm": 1.1086981296539307,
461
+ "kl": 0.07743949705036357,
462
+ "learning_rate": 3.3098591549295777e-06,
463
+ "loss": 0.0030974715948104857,
464
+ "num_tokens": 348233.0,
465
+ "reward": 0.5536999821662902,
466
+ "reward_std": 0.18073648810386658,
467
+ "rewards/reward_fn/mean": 0.5536999821662902,
468
+ "rewards/reward_fn/std": 0.1820658951997757,
469
  "step": 85
470
  },
471
  {
 
474
  "clip_ratio/low_mean": 0.0,
475
  "clip_ratio/low_min": 0.0,
476
  "clip_ratio/region_mean": 0.0,
477
+ "completion_length": 25.375,
478
+ "completions/clipped_ratio": 0.0,
479
+ "completions/max_length": 34.4,
480
+ "completions/max_terminated_length": 34.4,
481
+ "completions/mean_length": 25.375,
482
+ "completions/mean_terminated_length": 25.375,
483
+ "completions/min_length": 18.6,
484
+ "completions/min_terminated_length": 18.6,
485
+ "epoch": 0.4,
486
  "frac_reward_zero_std": 0.0,
487
+ "grad_norm": 1.1903071403503418,
488
+ "kl": 0.058934826811309904,
489
+ "learning_rate": 3.1924882629107983e-06,
490
+ "loss": 0.002357317507266998,
491
+ "num_tokens": 367732.0,
492
+ "reward": 0.5361499905586242,
493
+ "reward_std": 0.179675829410553,
494
+ "rewards/reward_fn/mean": 0.5361500144004822,
495
+ "rewards/reward_fn/std": 0.18574500381946563,
496
  "step": 90
497
  },
498
  {
 
501
  "clip_ratio/low_mean": 0.0,
502
  "clip_ratio/low_min": 0.0,
503
  "clip_ratio/region_mean": 0.0,
504
+ "completion_length": 26.825,
505
  "completions/clipped_ratio": 0.0,
506
+ "completions/max_length": 37.2,
507
+ "completions/max_terminated_length": 37.2,
508
+ "completions/mean_length": 26.825,
509
+ "completions/mean_terminated_length": 26.825,
510
  "completions/min_length": 20.8,
511
  "completions/min_terminated_length": 20.8,
512
+ "epoch": 0.4222222222222222,
513
  "frac_reward_zero_std": 0.0,
514
+ "grad_norm": 1.0952492952346802,
515
+ "kl": 0.040232469444163144,
516
+ "learning_rate": 3.075117370892019e-06,
517
+ "loss": 0.0016092658042907715,
518
+ "num_tokens": 387075.0,
519
+ "reward": 0.5278500020503998,
520
+ "reward_std": 0.1932522773742676,
521
+ "rewards/reward_fn/mean": 0.5278499901294709,
522
+ "rewards/reward_fn/std": 0.1796583503484726,
523
  "step": 95
524
  },
525
  {
 
528
  "clip_ratio/low_mean": 0.0,
529
  "clip_ratio/low_min": 0.0,
530
  "clip_ratio/region_mean": 0.0,
531
+ "completion_length": 26.125,
532
+ "completions/clipped_ratio": 0.0,
533
+ "completions/max_length": 35.6,
534
+ "completions/max_terminated_length": 35.6,
535
+ "completions/mean_length": 26.125,
536
+ "completions/mean_terminated_length": 26.125,
537
+ "completions/min_length": 22.6,
538
+ "completions/min_terminated_length": 22.6,
539
+ "epoch": 0.4444444444444444,
540
  "frac_reward_zero_std": 0.0,
541
+ "grad_norm": 0.8072858452796936,
542
+ "kl": 0.06188266044482589,
543
+ "learning_rate": 2.9577464788732396e-06,
544
+ "loss": 0.0024752289056777952,
545
+ "num_tokens": 406884.0,
546
+ "reward": 0.5849499881267548,
547
+ "reward_std": 0.20499025285243988,
548
+ "rewards/reward_fn/mean": 0.5849500000476837,
549
+ "rewards/reward_fn/std": 0.2056175708770752,
550
  "step": 100
551
  },
552
  {
 
555
  "clip_ratio/low_mean": 0.0,
556
  "clip_ratio/low_min": 0.0,
557
  "clip_ratio/region_mean": 0.0,
558
+ "completion_length": 26.45,
559
+ "completions/clipped_ratio": 0.025,
560
+ "completions/max_length": 43.2,
561
+ "completions/max_terminated_length": 29.0,
562
+ "completions/mean_length": 26.45,
563
+ "completions/mean_terminated_length": 24.639286041259766,
564
+ "completions/min_length": 20.4,
565
+ "completions/min_terminated_length": 20.4,
566
+ "epoch": 0.4666666666666667,
567
  "frac_reward_zero_std": 0.0,
568
+ "grad_norm": 0.9750375747680664,
569
+ "kl": 0.0710214663646184,
570
+ "learning_rate": 2.8403755868544603e-06,
571
+ "loss": 0.0028407976031303407,
572
+ "num_tokens": 427518.0,
573
+ "reward": 0.5496499896049499,
574
+ "reward_std": 0.16298811435699462,
575
+ "rewards/reward_fn/mean": 0.5496499896049499,
576
+ "rewards/reward_fn/std": 0.15672676265239716,
577
  "step": 105
578
  },
579
  {
 
582
  "clip_ratio/low_mean": 0.0,
583
  "clip_ratio/low_min": 0.0,
584
  "clip_ratio/region_mean": 0.0,
585
+ "completion_length": 25.925,
586
  "completions/clipped_ratio": 0.0,
587
+ "completions/max_length": 31.8,
588
+ "completions/max_terminated_length": 31.8,
589
+ "completions/mean_length": 25.925,
590
+ "completions/mean_terminated_length": 25.925,
591
+ "completions/min_length": 21.6,
592
+ "completions/min_terminated_length": 21.6,
593
+ "epoch": 0.4888888888888889,
594
  "frac_reward_zero_std": 0.0,
595
+ "grad_norm": 1.0184260606765747,
596
+ "kl": 0.0396142341895029,
597
+ "learning_rate": 2.723004694835681e-06,
598
+ "loss": 0.0015845373272895813,
599
+ "num_tokens": 447807.0,
600
+ "reward": 0.5718499898910523,
601
+ "reward_std": 0.20725298821926116,
602
+ "rewards/reward_fn/mean": 0.5718500018119812,
603
+ "rewards/reward_fn/std": 0.214348441362381,
604
  "step": 110
605
  },
606
  {
 
609
  "clip_ratio/low_mean": 0.0,
610
  "clip_ratio/low_min": 0.0,
611
  "clip_ratio/region_mean": 0.0,
612
+ "completion_length": 26.225,
613
+ "completions/clipped_ratio": 0.0,
614
+ "completions/max_length": 31.2,
615
+ "completions/max_terminated_length": 31.2,
616
+ "completions/mean_length": 26.225,
617
+ "completions/mean_terminated_length": 26.225,
618
+ "completions/min_length": 22.4,
619
+ "completions/min_terminated_length": 22.4,
620
+ "epoch": 0.5111111111111111,
621
  "frac_reward_zero_std": 0.0,
622
+ "grad_norm": 0.8158187866210938,
623
+ "kl": 0.048210305260727185,
624
+ "learning_rate": 2.6056338028169015e-06,
625
+ "loss": 0.0019283831119537354,
626
+ "num_tokens": 468650.0,
627
+ "reward": 0.5343500077724457,
628
+ "reward_std": 0.1919794887304306,
629
+ "rewards/reward_fn/mean": 0.5343500018119812,
630
+ "rewards/reward_fn/std": 0.17689327299594879,
631
  "step": 115
632
  },
633
  {
 
636
  "clip_ratio/low_mean": 0.0,
637
  "clip_ratio/low_min": 0.0,
638
  "clip_ratio/region_mean": 0.0,
639
+ "completion_length": 25.875,
640
+ "completions/clipped_ratio": 0.0,
641
+ "completions/max_length": 30.0,
642
+ "completions/max_terminated_length": 30.0,
643
+ "completions/mean_length": 25.875,
644
+ "completions/mean_terminated_length": 25.875,
645
+ "completions/min_length": 24.2,
646
+ "completions/min_terminated_length": 24.2,
647
+ "epoch": 0.5333333333333333,
648
+ "frac_reward_zero_std": 0.05,
649
+ "grad_norm": 0.8932302594184875,
650
+ "kl": 0.05402324852766469,
651
+ "learning_rate": 2.488262910798122e-06,
652
+ "loss": 0.0021608427166938783,
653
+ "num_tokens": 489051.0,
654
+ "reward": 0.6121499896049499,
655
+ "reward_std": 0.16680648624897004,
656
+ "rewards/reward_fn/mean": 0.612150001525879,
657
+ "rewards/reward_fn/std": 0.18321824073791504,
658
  "step": 120
659
  },
660
  {
 
663
  "clip_ratio/low_mean": 0.0,
664
  "clip_ratio/low_min": 0.0,
665
  "clip_ratio/region_mean": 0.0,
666
+ "completion_length": 27.3,
667
+ "completions/clipped_ratio": 0.025,
668
+ "completions/max_length": 42.6,
669
+ "completions/max_terminated_length": 29.8,
670
+ "completions/mean_length": 27.3,
671
+ "completions/mean_terminated_length": 25.514286041259766,
672
+ "completions/min_length": 21.4,
673
+ "completions/min_terminated_length": 21.4,
674
+ "epoch": 0.5555555555555556,
675
  "frac_reward_zero_std": 0.0,
676
+ "grad_norm": 0.922471821308136,
677
+ "kl": 0.04857689954806119,
678
+ "learning_rate": 2.370892018779343e-06,
679
+ "loss": 0.0019433587789535523,
680
+ "num_tokens": 508231.0,
681
+ "reward": 0.5884499907493591,
682
+ "reward_std": 0.17119054943323136,
683
+ "rewards/reward_fn/mean": 0.588450014591217,
684
+ "rewards/reward_fn/std": 0.18389662504196166,
685
  "step": 125
686
  },
687
  {
 
690
  "clip_ratio/low_mean": 0.0,
691
  "clip_ratio/low_min": 0.0,
692
  "clip_ratio/region_mean": 0.0,
693
+ "completion_length": 26.05,
694
  "completions/clipped_ratio": 0.0,
695
+ "completions/max_length": 31.0,
696
+ "completions/max_terminated_length": 31.0,
697
+ "completions/mean_length": 26.05,
698
+ "completions/mean_terminated_length": 26.05,
699
+ "completions/min_length": 22.0,
700
+ "completions/min_terminated_length": 22.0,
701
+ "epoch": 0.5777777777777777,
702
  "frac_reward_zero_std": 0.0,
703
+ "grad_norm": 0.7222557067871094,
704
+ "kl": 0.06997429557377473,
705
+ "learning_rate": 2.2535211267605635e-06,
706
+ "loss": 0.0027989834547042848,
707
+ "num_tokens": 528729.0,
708
+ "reward": 0.5927999973297119,
709
+ "reward_std": 0.22429427206516267,
710
+ "rewards/reward_fn/mean": 0.5928000092506409,
711
+ "rewards/reward_fn/std": 0.21314262747764587,
712
  "step": 130
713
  },
714
  {
 
717
  "clip_ratio/low_mean": 0.0,
718
  "clip_ratio/low_min": 0.0,
719
  "clip_ratio/region_mean": 0.0,
720
+ "completion_length": 25.925,
721
  "completions/clipped_ratio": 0.0,
722
+ "completions/max_length": 35.6,
723
+ "completions/max_terminated_length": 35.6,
724
+ "completions/mean_length": 25.925,
725
+ "completions/mean_terminated_length": 25.925,
726
+ "completions/min_length": 18.8,
727
+ "completions/min_terminated_length": 18.8,
728
+ "epoch": 0.6,
729
  "frac_reward_zero_std": 0.0,
730
+ "grad_norm": 1.2586678266525269,
731
+ "kl": 0.08584307442652062,
732
+ "learning_rate": 2.136150234741784e-06,
733
+ "loss": 0.0034336388111114503,
734
+ "num_tokens": 548748.0,
735
+ "reward": 0.5738499760627747,
736
+ "reward_std": 0.17571603059768676,
737
+ "rewards/reward_fn/mean": 0.5738499999046326,
738
+ "rewards/reward_fn/std": 0.19411510229110718,
739
  "step": 135
740
  },
741
  {
 
744
  "clip_ratio/low_mean": 0.0,
745
  "clip_ratio/low_min": 0.0,
746
  "clip_ratio/region_mean": 0.0,
747
+ "completion_length": 25.9,
748
  "completions/clipped_ratio": 0.0,
749
+ "completions/max_length": 29.4,
750
+ "completions/max_terminated_length": 29.4,
751
+ "completions/mean_length": 25.9,
752
+ "completions/mean_terminated_length": 25.9,
753
+ "completions/min_length": 23.4,
754
+ "completions/min_terminated_length": 23.4,
755
+ "epoch": 0.6222222222222222,
756
  "frac_reward_zero_std": 0.0,
757
+ "grad_norm": 1.0420219898223877,
758
+ "kl": 0.06210445412434638,
759
+ "learning_rate": 2.0187793427230047e-06,
760
+ "loss": 0.0024841248989105223,
761
+ "num_tokens": 569094.0,
762
+ "reward": 0.6095999956130982,
763
+ "reward_std": 0.2001112163066864,
764
+ "rewards/reward_fn/mean": 0.6095999956130982,
765
+ "rewards/reward_fn/std": 0.18984024077653885,
766
  "step": 140
767
  },
768
  {
 
771
  "clip_ratio/low_mean": 0.0,
772
  "clip_ratio/low_min": 0.0,
773
  "clip_ratio/region_mean": 0.0,
774
+ "completion_length": 27.85,
775
+ "completions/clipped_ratio": 0.025,
776
+ "completions/max_length": 46.0,
777
+ "completions/max_terminated_length": 34.8,
778
+ "completions/mean_length": 27.85,
779
+ "completions/mean_terminated_length": 26.164286041259764,
780
+ "completions/min_length": 21.2,
781
+ "completions/min_terminated_length": 21.2,
782
+ "epoch": 0.6444444444444445,
783
  "frac_reward_zero_std": 0.0,
784
+ "grad_norm": 0.9585386514663696,
785
+ "kl": 0.04591481959214434,
786
+ "learning_rate": 1.9014084507042254e-06,
787
+ "loss": 0.0018365621566772462,
788
+ "num_tokens": 589932.0,
789
+ "reward": 0.5361499905586242,
790
+ "reward_std": 0.20909147560596467,
791
+ "rewards/reward_fn/mean": 0.5361500024795532,
792
+ "rewards/reward_fn/std": 0.1902428910136223,
793
  "step": 145
794
  },
795
  {
 
798
  "clip_ratio/low_mean": 0.0,
799
  "clip_ratio/low_min": 0.0,
800
  "clip_ratio/region_mean": 0.0,
801
+ "completion_length": 25.25,
802
  "completions/clipped_ratio": 0.0,
803
+ "completions/max_length": 29.6,
804
+ "completions/max_terminated_length": 29.6,
805
+ "completions/mean_length": 25.25,
806
+ "completions/mean_terminated_length": 25.25,
807
+ "completions/min_length": 21.6,
808
+ "completions/min_terminated_length": 21.6,
809
+ "epoch": 0.6666666666666666,
810
  "frac_reward_zero_std": 0.0,
811
+ "grad_norm": 0.45393237471580505,
812
+ "kl": 0.04076897802297026,
813
+ "learning_rate": 1.784037558685446e-06,
814
+ "loss": 0.001630684733390808,
815
+ "num_tokens": 610754.0,
816
+ "reward": 0.5666499972343445,
817
+ "reward_std": 0.17826161682605743,
818
+ "rewards/reward_fn/mean": 0.5666500210762024,
819
+ "rewards/reward_fn/std": 0.1894826263189316,
820
  "step": 150
821
  }
822
  ],
823
  "logging_steps": 5,
824
+ "max_steps": 225,
825
+ "num_input_tokens_seen": 610754,
826
  "num_train_epochs": 1,
827
  "save_steps": 50,
828
  "stateful_callbacks": {
results/phase1/checkpoint-200/README.md CHANGED
@@ -6,6 +6,7 @@ tags:
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
 
9
  - transformers
10
  - trl
11
  - unsloth
 
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
9
+ - sft
10
  - transformers
11
  - trl
12
  - unsloth
results/phase1/checkpoint-200/adapter_config.json CHANGED
@@ -33,13 +33,13 @@
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
 
 
36
  "k_proj",
37
- "up_proj",
38
  "down_proj",
39
  "o_proj",
40
- "v_proj",
41
- "gate_proj",
42
- "q_proj"
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
 
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
36
+ "gate_proj",
37
+ "q_proj",
38
  "k_proj",
39
+ "v_proj",
40
  "down_proj",
41
  "o_proj",
42
+ "up_proj"
 
 
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
results/phase1/checkpoint-200/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4c26ece2f3d9d3d4dad0b3f24c8a55189a45b259a581746379eb5042892d1ab9
3
  size 73911112
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e679fe7063df93e5693ca5c4b4078626e9664ca0d3c4b56177004f47e81eb4c
3
  size 73911112
results/phase1/checkpoint-200/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:419e0e8af2f322b4b5a8c8665ec83a782bd5994d32068a47bde14d574cf26693
3
  size 37969669
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a08e8876ad967f49ce5843759247ca1b2a750c2ebc688dd5d7354d16fcfceab9
3
  size 37969669
results/phase1/checkpoint-200/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:31ac89641ae8e51a4d163ce36876b5f098ce283051a404ec657132b0ed089180
3
  size 14645
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25069db7710c4bef552b9b7fbd88f4d398b7354c1ed115dc6c7f865fcdb5a64c
3
  size 14645
results/phase1/checkpoint-200/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4d518c8bde4bc6896434cd6cc11418da6ed7291dd8afe1cbd0a22b41b24de64f
3
  size 1465
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2685d006b729d8986d8fc6e89b6e7f575ddbe4b9f764f7b9f7d433771599981c
3
  size 1465
results/phase1/checkpoint-200/trainer_state.json CHANGED
@@ -2,7 +2,7 @@
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
- "epoch": 0.9009009009009009,
6
  "eval_steps": 500,
7
  "global_step": 200,
8
  "is_hyper_param_search": false,
@@ -15,25 +15,25 @@
15
  "clip_ratio/low_mean": 0.0,
16
  "clip_ratio/low_min": 0.0,
17
  "clip_ratio/region_mean": 0.0,
18
- "completion_length": 29.05,
19
- "completions/clipped_ratio": 0.05,
20
- "completions/max_length": 44.2,
21
- "completions/max_terminated_length": 31.0,
22
- "completions/mean_length": 29.05,
23
- "completions/mean_terminated_length": 25.583333587646486,
24
- "completions/min_length": 21.4,
25
- "completions/min_terminated_length": 21.4,
26
- "epoch": 0.02252252252252252,
27
- "frac_reward_zero_std": 0.0,
28
- "grad_norm": 1.2577366828918457,
29
- "kl": 7.414049605358741e-06,
30
  "learning_rate": 1.6666666666666667e-06,
31
- "loss": 2.8312206268310547e-07,
32
- "num_tokens": 21378.0,
33
- "reward": 0.4944999933242798,
34
- "reward_std": 0.15089658200740813,
35
- "rewards/reward_fn/mean": 0.49449999928474425,
36
- "rewards/reward_fn/std": 0.12999328523874282,
37
  "step": 5
38
  },
39
  {
@@ -42,25 +42,25 @@
42
  "clip_ratio/low_mean": 0.0,
43
  "clip_ratio/low_min": 0.0,
44
  "clip_ratio/region_mean": 0.0,
45
- "completion_length": 29.175,
46
  "completions/clipped_ratio": 0.0,
47
- "completions/max_length": 41.8,
48
- "completions/max_terminated_length": 41.8,
49
- "completions/mean_length": 29.175,
50
- "completions/mean_terminated_length": 29.175,
51
- "completions/min_length": 19.2,
52
- "completions/min_terminated_length": 19.2,
53
- "epoch": 0.04504504504504504,
54
  "frac_reward_zero_std": 0.0,
55
- "grad_norm": 1.148110032081604,
56
- "kl": 2.58529256058182e-05,
57
  "learning_rate": 3.7500000000000005e-06,
58
- "loss": 9.343028068542481e-07,
59
- "num_tokens": 42709.0,
60
- "reward": 0.48589999675750734,
61
- "reward_std": 0.14184561967849732,
62
- "rewards/reward_fn/mean": 0.4859000027179718,
63
- "rewards/reward_fn/std": 0.12539554834365846,
64
  "step": 10
65
  },
66
  {
@@ -69,25 +69,25 @@
69
  "clip_ratio/low_mean": 0.0,
70
  "clip_ratio/low_min": 0.0,
71
  "clip_ratio/region_mean": 0.0,
72
- "completion_length": 28.825,
73
  "completions/clipped_ratio": 0.0,
74
- "completions/max_length": 44.2,
75
- "completions/max_terminated_length": 44.2,
76
- "completions/mean_length": 28.825,
77
- "completions/mean_terminated_length": 28.825,
78
- "completions/min_length": 21.2,
79
- "completions/min_terminated_length": 21.2,
80
- "epoch": 0.06756756756756757,
81
  "frac_reward_zero_std": 0.0,
82
- "grad_norm": 1.1520825624465942,
83
- "kl": 0.0001522944080875277,
84
- "learning_rate": 4.952380952380953e-06,
85
- "loss": 6.105005741119385e-06,
86
- "num_tokens": 63008.0,
87
- "reward": 0.4894999861717224,
88
- "reward_std": 0.15768481492996217,
89
- "rewards/reward_fn/mean": 0.48950000405311583,
90
- "rewards/reward_fn/std": 0.14185291826725005,
91
  "step": 15
92
  },
93
  {
@@ -96,25 +96,25 @@
96
  "clip_ratio/low_mean": 0.0,
97
  "clip_ratio/low_min": 0.0,
98
  "clip_ratio/region_mean": 0.0,
99
- "completion_length": 28.25,
100
  "completions/clipped_ratio": 0.0,
101
- "completions/max_length": 43.2,
102
- "completions/max_terminated_length": 43.2,
103
- "completions/mean_length": 28.25,
104
- "completions/mean_terminated_length": 28.25,
105
- "completions/min_length": 21.6,
106
- "completions/min_terminated_length": 21.6,
107
- "epoch": 0.09009009009009009,
108
  "frac_reward_zero_std": 0.0,
109
- "grad_norm": 1.0063226222991943,
110
- "kl": 0.0008983879492006963,
111
- "learning_rate": 4.833333333333333e-06,
112
- "loss": 3.593862056732178e-05,
113
- "num_tokens": 82836.0,
114
- "reward": 0.4881500005722046,
115
- "reward_std": 0.14941166192293168,
116
- "rewards/reward_fn/mean": 0.48814999461174013,
117
- "rewards/reward_fn/std": 0.12791907638311387,
118
  "step": 20
119
  },
120
  {
@@ -123,25 +123,25 @@
123
  "clip_ratio/low_mean": 0.0,
124
  "clip_ratio/low_min": 0.0,
125
  "clip_ratio/region_mean": 0.0,
126
- "completion_length": 27.25,
127
  "completions/clipped_ratio": 0.0,
128
- "completions/max_length": 45.6,
129
- "completions/max_terminated_length": 45.6,
130
- "completions/mean_length": 27.25,
131
- "completions/mean_terminated_length": 27.25,
132
- "completions/min_length": 18.8,
133
- "completions/min_terminated_length": 18.8,
134
- "epoch": 0.11261261261261261,
135
  "frac_reward_zero_std": 0.0,
136
- "grad_norm": 1.3132057189941406,
137
- "kl": 0.003781939600594342,
138
- "learning_rate": 4.714285714285715e-06,
139
- "loss": 0.00015127062797546387,
140
- "num_tokens": 103596.0,
141
- "reward": 0.4680500030517578,
142
- "reward_std": 0.17189764976501465,
143
- "rewards/reward_fn/mean": 0.46804999113082885,
144
- "rewards/reward_fn/std": 0.17421672195196153,
145
  "step": 25
146
  },
147
  {
@@ -150,25 +150,25 @@
150
  "clip_ratio/low_mean": 0.0,
151
  "clip_ratio/low_min": 0.0,
152
  "clip_ratio/region_mean": 0.0,
153
- "completion_length": 25.225,
154
  "completions/clipped_ratio": 0.0,
155
- "completions/max_length": 34.6,
156
- "completions/max_terminated_length": 34.6,
157
- "completions/mean_length": 25.225,
158
- "completions/mean_terminated_length": 25.225,
159
- "completions/min_length": 19.2,
160
- "completions/min_terminated_length": 19.2,
161
- "epoch": 0.13513513513513514,
162
  "frac_reward_zero_std": 0.0,
163
- "grad_norm": 1.5750545263290405,
164
- "kl": 0.006673775642411783,
165
- "learning_rate": 4.595238095238095e-06,
166
- "loss": 0.0002669498324394226,
167
- "num_tokens": 123651.0,
168
- "reward": 0.48049998879432676,
169
- "reward_std": 0.1656044065952301,
170
- "rewards/reward_fn/mean": 0.48049998879432676,
171
- "rewards/reward_fn/std": 0.13908967524766921,
172
  "step": 30
173
  },
174
  {
@@ -177,25 +177,25 @@
177
  "clip_ratio/low_mean": 0.0,
178
  "clip_ratio/low_min": 0.0,
179
  "clip_ratio/region_mean": 0.0,
180
- "completion_length": 28.775,
181
- "completions/clipped_ratio": 0.025,
182
- "completions/max_length": 54.8,
183
- "completions/max_terminated_length": 41.8,
184
- "completions/mean_length": 28.775,
185
- "completions/mean_terminated_length": 26.967857360839844,
186
- "completions/min_length": 15.4,
187
- "completions/min_terminated_length": 15.4,
188
- "epoch": 0.15765765765765766,
189
  "frac_reward_zero_std": 0.0,
190
- "grad_norm": 1.1105402708053589,
191
- "kl": 0.005106111426721327,
192
- "learning_rate": 4.476190476190477e-06,
193
- "loss": 0.0002042025327682495,
194
- "num_tokens": 144316.0,
195
- "reward": 0.4778999924659729,
196
- "reward_std": 0.17012988924980163,
197
- "rewards/reward_fn/mean": 0.47790001034736634,
198
- "rewards/reward_fn/std": 0.14147266000509262,
199
  "step": 35
200
  },
201
  {
@@ -204,25 +204,25 @@
204
  "clip_ratio/low_mean": 0.0,
205
  "clip_ratio/low_min": 0.0,
206
  "clip_ratio/region_mean": 0.0,
207
- "completion_length": 24.4,
208
- "completions/clipped_ratio": 0.0,
209
- "completions/max_length": 33.4,
210
- "completions/max_terminated_length": 33.4,
211
- "completions/mean_length": 24.4,
212
- "completions/mean_terminated_length": 24.4,
213
- "completions/min_length": 18.8,
214
- "completions/min_terminated_length": 18.8,
215
- "epoch": 0.18018018018018017,
216
  "frac_reward_zero_std": 0.0,
217
- "grad_norm": 1.0676746368408203,
218
- "kl": 0.006288998411764624,
219
- "learning_rate": 4.357142857142857e-06,
220
- "loss": 0.00025154203176498414,
221
- "num_tokens": 164334.0,
222
- "reward": 0.5014999985694886,
223
- "reward_std": 0.15358359068632127,
224
- "rewards/reward_fn/mean": 0.5014999985694886,
225
- "rewards/reward_fn/std": 0.13824734836816788,
226
  "step": 40
227
  },
228
  {
@@ -231,25 +231,25 @@
231
  "clip_ratio/low_mean": 0.0,
232
  "clip_ratio/low_min": 0.0,
233
  "clip_ratio/region_mean": 0.0,
234
- "completion_length": 27.35,
235
  "completions/clipped_ratio": 0.0,
236
- "completions/max_length": 43.6,
237
- "completions/max_terminated_length": 43.6,
238
- "completions/mean_length": 27.35,
239
- "completions/mean_terminated_length": 27.35,
240
- "completions/min_length": 19.6,
241
- "completions/min_terminated_length": 19.6,
242
- "epoch": 0.20270270270270271,
243
  "frac_reward_zero_std": 0.0,
244
- "grad_norm": 1.0749404430389404,
245
- "kl": 0.010629063473606948,
246
- "learning_rate": 4.238095238095239e-06,
247
- "loss": 0.0004251018166542053,
248
- "num_tokens": 185140.0,
249
- "reward": 0.524949985742569,
250
- "reward_std": 0.14630039632320405,
251
- "rewards/reward_fn/mean": 0.5249499917030335,
252
- "rewards/reward_fn/std": 0.1460244983434677,
253
  "step": 45
254
  },
255
  {
@@ -258,25 +258,25 @@
258
  "clip_ratio/low_mean": 0.0,
259
  "clip_ratio/low_min": 0.0,
260
  "clip_ratio/region_mean": 0.0,
261
- "completion_length": 25.425,
262
  "completions/clipped_ratio": 0.0,
263
  "completions/max_length": 35.2,
264
  "completions/max_terminated_length": 35.2,
265
- "completions/mean_length": 25.425,
266
- "completions/mean_terminated_length": 25.425,
267
- "completions/min_length": 17.6,
268
- "completions/min_terminated_length": 17.6,
269
- "epoch": 0.22522522522522523,
270
  "frac_reward_zero_std": 0.0,
271
- "grad_norm": 1.4353065490722656,
272
- "kl": 0.008290678875346203,
273
- "learning_rate": 4.119047619047619e-06,
274
- "loss": 0.0003315746784210205,
275
- "num_tokens": 206569.0,
276
- "reward": 0.4908999800682068,
277
- "reward_std": 0.17055415213108063,
278
- "rewards/reward_fn/mean": 0.49089999198913575,
279
- "rewards/reward_fn/std": 0.154354290664196,
280
  "step": 50
281
  },
282
  {
@@ -285,25 +285,25 @@
285
  "clip_ratio/low_mean": 0.0,
286
  "clip_ratio/low_min": 0.0,
287
  "clip_ratio/region_mean": 0.0,
288
- "completion_length": 27.7,
289
- "completions/clipped_ratio": 0.025,
290
- "completions/max_length": 50.6,
291
- "completions/max_terminated_length": 37.2,
292
- "completions/mean_length": 27.7,
293
- "completions/mean_terminated_length": 25.91071434020996,
294
- "completions/min_length": 18.2,
295
- "completions/min_terminated_length": 18.2,
296
- "epoch": 0.24774774774774774,
297
  "frac_reward_zero_std": 0.0,
298
- "grad_norm": 0.8640509843826294,
299
- "kl": 0.012456063569698018,
300
- "learning_rate": 4.000000000000001e-06,
301
- "loss": 0.0004982292652130127,
302
- "num_tokens": 226097.0,
303
- "reward": 0.4839499771595001,
304
- "reward_std": 0.1590283066034317,
305
- "rewards/reward_fn/mean": 0.48394999504089353,
306
- "rewards/reward_fn/std": 0.1380786642432213,
307
  "step": 55
308
  },
309
  {
@@ -312,25 +312,25 @@
312
  "clip_ratio/low_mean": 0.0,
313
  "clip_ratio/low_min": 0.0,
314
  "clip_ratio/region_mean": 0.0,
315
- "completion_length": 26.875,
316
- "completions/clipped_ratio": 0.0,
317
- "completions/max_length": 40.0,
318
- "completions/max_terminated_length": 40.0,
319
- "completions/mean_length": 26.875,
320
- "completions/mean_terminated_length": 26.875,
321
- "completions/min_length": 20.6,
322
- "completions/min_terminated_length": 20.6,
323
- "epoch": 0.2702702702702703,
324
  "frac_reward_zero_std": 0.0,
325
- "grad_norm": 1.0529346466064453,
326
- "kl": 0.007519814093393506,
327
- "learning_rate": 3.880952380952381e-06,
328
- "loss": 0.00030075013637542725,
329
- "num_tokens": 246836.0,
330
- "reward": 0.4914499819278717,
331
- "reward_std": 0.1727461814880371,
332
- "rewards/reward_fn/mean": 0.49144998788833616,
333
- "rewards/reward_fn/std": 0.1568704679608345,
334
  "step": 60
335
  },
336
  {
@@ -339,25 +339,25 @@
339
  "clip_ratio/low_mean": 0.0,
340
  "clip_ratio/low_min": 0.0,
341
  "clip_ratio/region_mean": 0.0,
342
- "completion_length": 27.825,
343
- "completions/clipped_ratio": 0.025,
344
- "completions/max_length": 51.2,
345
- "completions/max_terminated_length": 38.0,
346
- "completions/mean_length": 27.825,
347
- "completions/mean_terminated_length": 26.03214302062988,
348
- "completions/min_length": 17.6,
349
- "completions/min_terminated_length": 17.6,
350
- "epoch": 0.2927927927927928,
351
  "frac_reward_zero_std": 0.0,
352
- "grad_norm": 1.149415373802185,
353
- "kl": 0.00938094645498495,
354
- "learning_rate": 3.761904761904762e-06,
355
- "loss": 0.0003751114010810852,
356
- "num_tokens": 266985.0,
357
- "reward": 0.5003499746322632,
358
- "reward_std": 0.16892780363559723,
359
- "rewards/reward_fn/mean": 0.5003500044345855,
360
- "rewards/reward_fn/std": 0.14663992822170258,
361
  "step": 65
362
  },
363
  {
@@ -366,25 +366,25 @@
366
  "clip_ratio/low_mean": 0.0,
367
  "clip_ratio/low_min": 0.0,
368
  "clip_ratio/region_mean": 0.0,
369
- "completion_length": 27.0,
370
  "completions/clipped_ratio": 0.0,
371
- "completions/max_length": 40.6,
372
- "completions/max_terminated_length": 40.6,
373
- "completions/mean_length": 27.0,
374
- "completions/mean_terminated_length": 27.0,
375
- "completions/min_length": 17.8,
376
- "completions/min_terminated_length": 17.8,
377
- "epoch": 0.3153153153153153,
378
  "frac_reward_zero_std": 0.0,
379
- "grad_norm": 0.9893133640289307,
380
- "kl": 0.010053556466664304,
381
- "learning_rate": 3.642857142857143e-06,
382
- "loss": 0.00040204524993896483,
383
- "num_tokens": 288099.0,
384
- "reward": 0.4705499827861786,
385
- "reward_std": 0.16256383657455445,
386
- "rewards/reward_fn/mean": 0.470550000667572,
387
- "rewards/reward_fn/std": 0.13094386458396912,
388
  "step": 70
389
  },
390
  {
@@ -393,25 +393,25 @@
393
  "clip_ratio/low_mean": 0.0,
394
  "clip_ratio/low_min": 0.0,
395
  "clip_ratio/region_mean": 0.0,
396
- "completion_length": 25.75,
397
- "completions/clipped_ratio": 0.0,
398
- "completions/max_length": 34.8,
399
- "completions/max_terminated_length": 34.8,
400
- "completions/mean_length": 25.75,
401
- "completions/mean_terminated_length": 25.75,
402
- "completions/min_length": 19.8,
403
- "completions/min_terminated_length": 19.8,
404
- "epoch": 0.33783783783783783,
405
  "frac_reward_zero_std": 0.0,
406
- "grad_norm": 1.063340187072754,
407
- "kl": 0.008550376921630232,
408
- "learning_rate": 3.523809523809524e-06,
409
- "loss": 0.00034204572439193725,
410
- "num_tokens": 308531.0,
411
- "reward": 0.5000500082969666,
412
- "reward_std": 0.16242243051528932,
413
- "rewards/reward_fn/mean": 0.5000500082969666,
414
- "rewards/reward_fn/std": 0.14417539685964584,
415
  "step": 75
416
  },
417
  {
@@ -420,25 +420,25 @@
420
  "clip_ratio/low_mean": 0.0,
421
  "clip_ratio/low_min": 0.0,
422
  "clip_ratio/region_mean": 0.0,
423
- "completion_length": 27.025,
424
  "completions/clipped_ratio": 0.0,
425
- "completions/max_length": 40.0,
426
- "completions/max_terminated_length": 40.0,
427
- "completions/mean_length": 27.025,
428
- "completions/mean_terminated_length": 27.025,
429
- "completions/min_length": 19.4,
430
- "completions/min_terminated_length": 19.4,
431
- "epoch": 0.36036036036036034,
432
  "frac_reward_zero_std": 0.0,
433
- "grad_norm": 1.2858343124389648,
434
- "kl": 0.019255010827328077,
435
- "learning_rate": 3.404761904761905e-06,
436
- "loss": 0.0007702425122261047,
437
- "num_tokens": 328874.0,
438
- "reward": 0.5261500000953674,
439
- "reward_std": 0.11886464804410934,
440
- "rewards/reward_fn/mean": 0.5261499881744385,
441
- "rewards/reward_fn/std": 0.12691220343112947,
442
  "step": 80
443
  },
444
  {
@@ -447,25 +447,25 @@
447
  "clip_ratio/low_mean": 0.0,
448
  "clip_ratio/low_min": 0.0,
449
  "clip_ratio/region_mean": 0.0,
450
- "completion_length": 25.475,
451
  "completions/clipped_ratio": 0.0,
452
- "completions/max_length": 33.0,
453
- "completions/max_terminated_length": 33.0,
454
- "completions/mean_length": 25.475,
455
- "completions/mean_terminated_length": 25.475,
456
- "completions/min_length": 19.0,
457
- "completions/min_terminated_length": 19.0,
458
- "epoch": 0.38288288288288286,
459
  "frac_reward_zero_std": 0.0,
460
- "grad_norm": 1.118030309677124,
461
- "kl": 0.01090274843845691,
462
- "learning_rate": 3.285714285714286e-06,
463
- "loss": 0.00043606162071228025,
464
- "num_tokens": 348373.0,
465
- "reward": 0.49950000643730164,
466
- "reward_std": 0.15655344128608703,
467
- "rewards/reward_fn/mean": 0.49950000643730164,
468
- "rewards/reward_fn/std": 0.14320258051156998,
469
  "step": 85
470
  },
471
  {
@@ -474,25 +474,25 @@
474
  "clip_ratio/low_mean": 0.0,
475
  "clip_ratio/low_min": 0.0,
476
  "clip_ratio/region_mean": 0.0,
477
- "completion_length": 27.9,
478
- "completions/clipped_ratio": 0.05,
479
- "completions/max_length": 55.4,
480
- "completions/max_terminated_length": 31.6,
481
- "completions/mean_length": 27.9,
482
- "completions/mean_terminated_length": 24.28571472167969,
483
- "completions/min_length": 18.0,
484
- "completions/min_terminated_length": 18.0,
485
- "epoch": 0.40540540540540543,
486
  "frac_reward_zero_std": 0.0,
487
- "grad_norm": 0.6538166403770447,
488
- "kl": 0.008556566922561615,
489
- "learning_rate": 3.1666666666666667e-06,
490
- "loss": 0.0003422081470489502,
491
- "num_tokens": 368815.0,
492
- "reward": 0.5215500056743622,
493
- "reward_std": 0.16581654250621797,
494
- "rewards/reward_fn/mean": 0.5215500056743622,
495
- "rewards/reward_fn/std": 0.15472394227981567,
496
  "step": 90
497
  },
498
  {
@@ -501,25 +501,25 @@
501
  "clip_ratio/low_mean": 0.0,
502
  "clip_ratio/low_min": 0.0,
503
  "clip_ratio/region_mean": 0.0,
504
- "completion_length": 25.75,
505
  "completions/clipped_ratio": 0.0,
506
- "completions/max_length": 32.4,
507
- "completions/max_terminated_length": 32.4,
508
- "completions/mean_length": 25.75,
509
- "completions/mean_terminated_length": 25.75,
510
  "completions/min_length": 20.8,
511
  "completions/min_terminated_length": 20.8,
512
- "epoch": 0.42792792792792794,
513
  "frac_reward_zero_std": 0.0,
514
- "grad_norm": 1.180251955986023,
515
- "kl": 0.014757526386529208,
516
- "learning_rate": 3.047619047619048e-06,
517
- "loss": 0.0005902737379074096,
518
- "num_tokens": 388761.0,
519
- "reward": 0.514000016450882,
520
- "reward_std": 0.1465125188231468,
521
- "rewards/reward_fn/mean": 0.5139999985694885,
522
- "rewards/reward_fn/std": 0.14701166301965712,
523
  "step": 95
524
  },
525
  {
@@ -528,25 +528,25 @@
528
  "clip_ratio/low_mean": 0.0,
529
  "clip_ratio/low_min": 0.0,
530
  "clip_ratio/region_mean": 0.0,
531
- "completion_length": 26.35,
532
- "completions/clipped_ratio": 0.025,
533
- "completions/max_length": 46.6,
534
- "completions/max_terminated_length": 32.4,
535
- "completions/mean_length": 26.35,
536
- "completions/mean_terminated_length": 24.517857360839844,
537
- "completions/min_length": 17.8,
538
- "completions/min_terminated_length": 17.8,
539
- "epoch": 0.45045045045045046,
540
  "frac_reward_zero_std": 0.0,
541
- "grad_norm": 0.8863919973373413,
542
- "kl": 0.017311154678463937,
543
- "learning_rate": 2.928571428571429e-06,
544
- "loss": 0.0006924748420715332,
545
- "num_tokens": 408309.0,
546
- "reward": 0.5527999818325042,
547
- "reward_std": 0.1593818634748459,
548
- "rewards/reward_fn/mean": 0.5527999699115753,
549
- "rewards/reward_fn/std": 0.16823574155569077,
550
  "step": 100
551
  },
552
  {
@@ -555,25 +555,25 @@
555
  "clip_ratio/low_mean": 0.0,
556
  "clip_ratio/low_min": 0.0,
557
  "clip_ratio/region_mean": 0.0,
558
- "completion_length": 28.675,
559
- "completions/clipped_ratio": 0.05,
560
- "completions/max_length": 45.6,
561
- "completions/max_terminated_length": 33.2,
562
- "completions/mean_length": 28.675,
563
- "completions/mean_terminated_length": 25.291666793823243,
564
- "completions/min_length": 20.6,
565
- "completions/min_terminated_length": 20.6,
566
- "epoch": 0.47297297297297297,
567
  "frac_reward_zero_std": 0.0,
568
- "grad_norm": 0.7957925200462341,
569
- "kl": 0.021478149453469087,
570
- "learning_rate": 2.8095238095238096e-06,
571
- "loss": 0.0008591204881668091,
572
- "num_tokens": 429872.0,
573
- "reward": 0.5294999718666077,
574
- "reward_std": 0.15301789939403534,
575
- "rewards/reward_fn/mean": 0.5294999837875366,
576
- "rewards/reward_fn/std": 0.15534335970878602,
577
  "step": 105
578
  },
579
  {
@@ -582,25 +582,25 @@
582
  "clip_ratio/low_mean": 0.0,
583
  "clip_ratio/low_min": 0.0,
584
  "clip_ratio/region_mean": 0.0,
585
- "completion_length": 27.325,
586
  "completions/clipped_ratio": 0.0,
587
- "completions/max_length": 47.2,
588
- "completions/max_terminated_length": 47.2,
589
- "completions/mean_length": 27.325,
590
- "completions/mean_terminated_length": 27.325,
591
- "completions/min_length": 19.2,
592
- "completions/min_terminated_length": 19.2,
593
- "epoch": 0.4954954954954955,
594
  "frac_reward_zero_std": 0.0,
595
- "grad_norm": 1.187671184539795,
596
- "kl": 0.01666262859798735,
597
- "learning_rate": 2.6904761904761906e-06,
598
- "loss": 0.0006665512919425964,
599
- "num_tokens": 450229.0,
600
- "reward": 0.5364000022411346,
601
- "reward_std": 0.16447303295135499,
602
- "rewards/reward_fn/mean": 0.5364000022411346,
603
- "rewards/reward_fn/std": 0.17169796973466872,
604
  "step": 110
605
  },
606
  {
@@ -609,25 +609,25 @@
609
  "clip_ratio/low_mean": 0.0,
610
  "clip_ratio/low_min": 0.0,
611
  "clip_ratio/region_mean": 0.0,
612
- "completion_length": 27.6,
613
- "completions/clipped_ratio": 0.025,
614
- "completions/max_length": 46.6,
615
- "completions/max_terminated_length": 37.2,
616
- "completions/mean_length": 27.6,
617
- "completions/mean_terminated_length": 25.853571701049805,
618
- "completions/min_length": 18.8,
619
- "completions/min_terminated_length": 18.8,
620
- "epoch": 0.5180180180180181,
621
  "frac_reward_zero_std": 0.0,
622
- "grad_norm": 1.3785444498062134,
623
- "kl": 0.015608730388339608,
624
- "learning_rate": 2.571428571428571e-06,
625
- "loss": 0.0006243243813514709,
626
- "num_tokens": 470911.0,
627
- "reward": 0.5131499886512756,
628
- "reward_std": 0.15916974246501922,
629
- "rewards/reward_fn/mean": 0.513150018453598,
630
- "rewards/reward_fn/std": 0.15986726433038712,
631
  "step": 115
632
  },
633
  {
@@ -636,25 +636,25 @@
636
  "clip_ratio/low_mean": 0.0,
637
  "clip_ratio/low_min": 0.0,
638
  "clip_ratio/region_mean": 0.0,
639
- "completion_length": 26.725,
640
- "completions/clipped_ratio": 0.025,
641
- "completions/max_length": 43.8,
642
- "completions/max_terminated_length": 30.4,
643
- "completions/mean_length": 26.725,
644
- "completions/mean_terminated_length": 24.953571701049803,
645
- "completions/min_length": 20.4,
646
- "completions/min_terminated_length": 20.4,
647
- "epoch": 0.5405405405405406,
648
- "frac_reward_zero_std": 0.0,
649
- "grad_norm": 0.8001337647438049,
650
- "kl": 0.016426509176380933,
651
- "learning_rate": 2.4523809523809526e-06,
652
- "loss": 0.0006570681929588318,
653
- "num_tokens": 490616.0,
654
- "reward": 0.5314500093460083,
655
- "reward_std": 0.16807928085327148,
656
- "rewards/reward_fn/mean": 0.5314499974250794,
657
- "rewards/reward_fn/std": 0.16840155124664308,
658
  "step": 120
659
  },
660
  {
@@ -663,25 +663,25 @@
663
  "clip_ratio/low_mean": 0.0,
664
  "clip_ratio/low_min": 0.0,
665
  "clip_ratio/region_mean": 0.0,
666
- "completion_length": 25.8,
667
- "completions/clipped_ratio": 0.0,
668
- "completions/max_length": 39.8,
669
- "completions/max_terminated_length": 39.8,
670
- "completions/mean_length": 25.8,
671
- "completions/mean_terminated_length": 25.8,
672
- "completions/min_length": 17.6,
673
- "completions/min_terminated_length": 17.6,
674
- "epoch": 0.5630630630630631,
675
  "frac_reward_zero_std": 0.0,
676
- "grad_norm": 1.0382933616638184,
677
- "kl": 0.016185989990481174,
678
- "learning_rate": 2.3333333333333336e-06,
679
- "loss": 0.000647495687007904,
680
- "num_tokens": 509092.0,
681
- "reward": 0.5144500017166138,
682
- "reward_std": 0.15323003232479096,
683
- "rewards/reward_fn/mean": 0.5144500017166138,
684
- "rewards/reward_fn/std": 0.13980331867933274,
685
  "step": 125
686
  },
687
  {
@@ -690,25 +690,25 @@
690
  "clip_ratio/low_mean": 0.0,
691
  "clip_ratio/low_min": 0.0,
692
  "clip_ratio/region_mean": 0.0,
693
- "completion_length": 26.275,
694
  "completions/clipped_ratio": 0.0,
695
- "completions/max_length": 36.4,
696
- "completions/max_terminated_length": 36.4,
697
- "completions/mean_length": 26.275,
698
- "completions/mean_terminated_length": 26.275,
699
- "completions/min_length": 19.6,
700
- "completions/min_terminated_length": 19.6,
701
- "epoch": 0.5855855855855856,
702
  "frac_reward_zero_std": 0.0,
703
- "grad_norm": 1.3675200939178467,
704
- "kl": 0.01404028357937932,
705
- "learning_rate": 2.2142857142857146e-06,
706
- "loss": 0.0005615666508674621,
707
- "num_tokens": 531083.0,
708
- "reward": 0.5205999791622162,
709
- "reward_std": 0.1824335426092148,
710
- "rewards/reward_fn/mean": 0.520600003004074,
711
- "rewards/reward_fn/std": 0.16652330607175828,
712
  "step": 130
713
  },
714
  {
@@ -717,25 +717,25 @@
717
  "clip_ratio/low_mean": 0.0,
718
  "clip_ratio/low_min": 0.0,
719
  "clip_ratio/region_mean": 0.0,
720
- "completion_length": 27.025,
721
  "completions/clipped_ratio": 0.0,
722
- "completions/max_length": 41.0,
723
- "completions/max_terminated_length": 41.0,
724
- "completions/mean_length": 27.025,
725
- "completions/mean_terminated_length": 27.025,
726
- "completions/min_length": 20.2,
727
- "completions/min_terminated_length": 20.2,
728
- "epoch": 0.6081081081081081,
729
  "frac_reward_zero_std": 0.0,
730
- "grad_norm": 1.5435948371887207,
731
- "kl": 0.01980702848522924,
732
- "learning_rate": 2.0952380952380955e-06,
733
- "loss": 0.0007923290133476258,
734
- "num_tokens": 551052.0,
735
- "reward": 0.5506500005722046,
736
- "reward_std": 0.15252292901277542,
737
- "rewards/reward_fn/mean": 0.5506499886512757,
738
- "rewards/reward_fn/std": 0.16045962125062943,
739
  "step": 135
740
  },
741
  {
@@ -744,25 +744,25 @@
744
  "clip_ratio/low_mean": 0.0,
745
  "clip_ratio/low_min": 0.0,
746
  "clip_ratio/region_mean": 0.0,
747
- "completion_length": 24.875,
748
  "completions/clipped_ratio": 0.0,
749
- "completions/max_length": 30.0,
750
- "completions/max_terminated_length": 30.0,
751
- "completions/mean_length": 24.875,
752
- "completions/mean_terminated_length": 24.875,
753
- "completions/min_length": 19.8,
754
- "completions/min_terminated_length": 19.8,
755
- "epoch": 0.6306306306306306,
756
  "frac_reward_zero_std": 0.0,
757
- "grad_norm": 1.0660195350646973,
758
- "kl": 0.014228896767599508,
759
- "learning_rate": 1.976190476190476e-06,
760
- "loss": 0.0005690708756446838,
761
- "num_tokens": 570993.0,
762
- "reward": 0.5141499876976013,
763
- "reward_std": 0.1573312520980835,
764
- "rewards/reward_fn/mean": 0.5141499936580658,
765
- "rewards/reward_fn/std": 0.14991891831159593,
766
  "step": 140
767
  },
768
  {
@@ -771,25 +771,25 @@
771
  "clip_ratio/low_mean": 0.0,
772
  "clip_ratio/low_min": 0.0,
773
  "clip_ratio/region_mean": 0.0,
774
- "completion_length": 24.775,
775
- "completions/clipped_ratio": 0.0,
776
- "completions/max_length": 35.4,
777
- "completions/max_terminated_length": 35.4,
778
- "completions/mean_length": 24.775,
779
- "completions/mean_terminated_length": 24.775,
780
- "completions/min_length": 17.0,
781
- "completions/min_terminated_length": 17.0,
782
- "epoch": 0.6531531531531531,
783
  "frac_reward_zero_std": 0.0,
784
- "grad_norm": 0.9134359359741211,
785
- "kl": 0.015506639698287472,
786
- "learning_rate": 1.8571428571428573e-06,
787
- "loss": 0.0006200879812240601,
788
- "num_tokens": 591118.0,
789
- "reward": 0.5515999913215637,
790
- "reward_std": 0.1493409514427185,
791
- "rewards/reward_fn/mean": 0.5516000032424927,
792
- "rewards/reward_fn/std": 0.15984065383672713,
793
  "step": 145
794
  },
795
  {
@@ -798,25 +798,25 @@
798
  "clip_ratio/low_mean": 0.0,
799
  "clip_ratio/low_min": 0.0,
800
  "clip_ratio/region_mean": 0.0,
801
- "completion_length": 25.45,
802
  "completions/clipped_ratio": 0.0,
803
- "completions/max_length": 34.6,
804
- "completions/max_terminated_length": 34.6,
805
- "completions/mean_length": 25.45,
806
- "completions/mean_terminated_length": 25.45,
807
- "completions/min_length": 20.6,
808
- "completions/min_terminated_length": 20.6,
809
- "epoch": 0.6756756756756757,
810
  "frac_reward_zero_std": 0.0,
811
- "grad_norm": 1.011871099472046,
812
- "kl": 0.018102088455634657,
813
- "learning_rate": 1.738095238095238e-06,
814
- "loss": 0.0007240131497383117,
815
- "num_tokens": 611492.0,
816
- "reward": 0.5036499917507171,
817
- "reward_std": 0.14262343645095826,
818
- "rewards/reward_fn/mean": 0.5036500096321106,
819
- "rewards/reward_fn/std": 0.14135744124650956,
820
  "step": 150
821
  },
822
  {
@@ -825,25 +825,25 @@
825
  "clip_ratio/low_mean": 0.0,
826
  "clip_ratio/low_min": 0.0,
827
  "clip_ratio/region_mean": 0.0,
828
- "completion_length": 25.55,
829
  "completions/clipped_ratio": 0.0,
830
- "completions/max_length": 35.4,
831
- "completions/max_terminated_length": 35.4,
832
- "completions/mean_length": 25.55,
833
- "completions/mean_terminated_length": 25.55,
834
- "completions/min_length": 19.2,
835
- "completions/min_terminated_length": 19.2,
836
- "epoch": 0.6981981981981982,
837
  "frac_reward_zero_std": 0.0,
838
- "grad_norm": 1.136124849319458,
839
- "kl": 0.02021147561608814,
840
- "learning_rate": 1.6190476190476193e-06,
841
- "loss": 0.0008084163069725037,
842
- "num_tokens": 632232.0,
843
- "reward": 0.5214999973773956,
844
- "reward_std": 0.1552806466817856,
845
- "rewards/reward_fn/mean": 0.5214999914169312,
846
- "rewards/reward_fn/std": 0.14986062049865723,
847
  "step": 155
848
  },
849
  {
@@ -854,23 +854,23 @@
854
  "clip_ratio/region_mean": 0.0,
855
  "completion_length": 25.6,
856
  "completions/clipped_ratio": 0.0,
857
- "completions/max_length": 34.2,
858
- "completions/max_terminated_length": 34.2,
859
  "completions/mean_length": 25.6,
860
  "completions/mean_terminated_length": 25.6,
861
- "completions/min_length": 19.6,
862
- "completions/min_terminated_length": 19.6,
863
- "epoch": 0.7207207207207207,
864
  "frac_reward_zero_std": 0.0,
865
- "grad_norm": 0.7977616190910339,
866
- "kl": 0.015592006998485886,
867
- "learning_rate": 1.5e-06,
868
- "loss": 0.0006236553192138672,
869
- "num_tokens": 652692.0,
870
- "reward": 0.5022999882698059,
871
- "reward_std": 0.16786714345216752,
872
- "rewards/reward_fn/mean": 0.5023000061511993,
873
- "rewards/reward_fn/std": 0.1559548258781433,
874
  "step": 160
875
  },
876
  {
@@ -879,25 +879,25 @@
879
  "clip_ratio/low_mean": 0.0,
880
  "clip_ratio/low_min": 0.0,
881
  "clip_ratio/region_mean": 0.0,
882
- "completion_length": 29.225,
883
- "completions/clipped_ratio": 0.025,
884
- "completions/max_length": 62.2,
885
- "completions/max_terminated_length": 52.0,
886
- "completions/mean_length": 29.225,
887
- "completions/mean_terminated_length": 27.489286041259767,
888
- "completions/min_length": 17.6,
889
- "completions/min_terminated_length": 17.6,
890
- "epoch": 0.7432432432432432,
891
  "frac_reward_zero_std": 0.0,
892
- "grad_norm": 0.7920641899108887,
893
- "kl": 0.02950997047009878,
894
- "learning_rate": 1.3809523809523812e-06,
895
- "loss": 0.0011803746223449708,
896
- "num_tokens": 672595.0,
897
- "reward": 0.5018999874591827,
898
- "reward_std": 0.16178602278232573,
899
- "rewards/reward_fn/mean": 0.5018999993801116,
900
- "rewards/reward_fn/std": 0.1466786965727806,
901
  "step": 165
902
  },
903
  {
@@ -906,25 +906,25 @@
906
  "clip_ratio/low_mean": 0.0,
907
  "clip_ratio/low_min": 0.0,
908
  "clip_ratio/region_mean": 0.0,
909
- "completion_length": 25.225,
910
  "completions/clipped_ratio": 0.0,
911
- "completions/max_length": 33.0,
912
- "completions/max_terminated_length": 33.0,
913
- "completions/mean_length": 25.225,
914
- "completions/mean_terminated_length": 25.225,
915
- "completions/min_length": 18.4,
916
- "completions/min_terminated_length": 18.4,
917
- "epoch": 0.7657657657657657,
918
  "frac_reward_zero_std": 0.0,
919
- "grad_norm": 1.1562975645065308,
920
- "kl": 0.018220309726893903,
921
- "learning_rate": 1.261904761904762e-06,
922
- "loss": 0.0007288455963134766,
923
- "num_tokens": 692896.0,
924
- "reward": 0.5815999805927277,
925
- "reward_std": 0.14750247299671174,
926
- "rewards/reward_fn/mean": 0.5815999805927277,
927
- "rewards/reward_fn/std": 0.1825831338763237,
928
  "step": 170
929
  },
930
  {
@@ -933,25 +933,25 @@
933
  "clip_ratio/low_mean": 0.0,
934
  "clip_ratio/low_min": 0.0,
935
  "clip_ratio/region_mean": 0.0,
936
- "completion_length": 26.95,
937
- "completions/clipped_ratio": 0.025,
938
- "completions/max_length": 49.8,
939
- "completions/max_terminated_length": 35.6,
940
- "completions/mean_length": 26.95,
941
- "completions/mean_terminated_length": 25.167857360839843,
942
- "completions/min_length": 19.4,
943
- "completions/min_terminated_length": 19.4,
944
- "epoch": 0.7882882882882883,
945
  "frac_reward_zero_std": 0.0,
946
- "grad_norm": 1.2884600162506104,
947
- "kl": 0.021587877761339767,
948
- "learning_rate": 1.142857142857143e-06,
949
- "loss": 0.0008635029196739196,
950
- "num_tokens": 712396.0,
951
- "reward": 0.5006500005722045,
952
- "reward_std": 0.17076628804206848,
953
- "rewards/reward_fn/mean": 0.5006500005722045,
954
- "rewards/reward_fn/std": 0.14878996163606645,
955
  "step": 175
956
  },
957
  {
@@ -960,25 +960,25 @@
960
  "clip_ratio/low_mean": 0.0,
961
  "clip_ratio/low_min": 0.0,
962
  "clip_ratio/region_mean": 0.0,
963
- "completion_length": 26.35,
964
- "completions/clipped_ratio": 0.025,
965
- "completions/max_length": 46.0,
966
- "completions/max_terminated_length": 35.2,
967
- "completions/mean_length": 26.35,
968
- "completions/mean_terminated_length": 24.57142868041992,
969
- "completions/min_length": 15.4,
970
- "completions/min_terminated_length": 15.4,
971
- "epoch": 0.8108108108108109,
972
  "frac_reward_zero_std": 0.0,
973
- "grad_norm": 1.3578517436981201,
974
- "kl": 0.025513717756257392,
975
- "learning_rate": 1.023809523809524e-06,
976
- "loss": 0.001020367443561554,
977
- "num_tokens": 732648.0,
978
- "reward": 0.5190999984741211,
979
- "reward_std": 0.14453262090682983,
980
- "rewards/reward_fn/mean": 0.51910001039505,
981
- "rewards/reward_fn/std": 0.14755631536245345,
982
  "step": 180
983
  },
984
  {
@@ -987,25 +987,25 @@
987
  "clip_ratio/low_mean": 0.0,
988
  "clip_ratio/low_min": 0.0,
989
  "clip_ratio/region_mean": 0.0,
990
- "completion_length": 26.875,
991
- "completions/clipped_ratio": 0.025,
992
- "completions/max_length": 44.8,
993
- "completions/max_terminated_length": 30.8,
994
- "completions/mean_length": 26.875,
995
- "completions/mean_terminated_length": 25.082143020629882,
996
- "completions/min_length": 20.2,
997
- "completions/min_terminated_length": 20.2,
998
- "epoch": 0.8333333333333334,
999
  "frac_reward_zero_std": 0.0,
1000
- "grad_norm": 1.042319893836975,
1001
- "kl": 0.014541034388821572,
1002
- "learning_rate": 9.047619047619048e-07,
1003
- "loss": 0.0005815878510475158,
1004
- "num_tokens": 752821.0,
1005
- "reward": 0.49784999489784243,
1006
- "reward_std": 0.16807928085327148,
1007
- "rewards/reward_fn/mean": 0.49785000681877134,
1008
- "rewards/reward_fn/std": 0.14966378808021547,
1009
  "step": 185
1010
  },
1011
  {
@@ -1014,25 +1014,25 @@
1014
  "clip_ratio/low_mean": 0.0,
1015
  "clip_ratio/low_min": 0.0,
1016
  "clip_ratio/region_mean": 0.0,
1017
- "completion_length": 25.625,
1018
  "completions/clipped_ratio": 0.0,
1019
- "completions/max_length": 35.6,
1020
- "completions/max_terminated_length": 35.6,
1021
- "completions/mean_length": 25.625,
1022
- "completions/mean_terminated_length": 25.625,
1023
- "completions/min_length": 19.4,
1024
- "completions/min_terminated_length": 19.4,
1025
- "epoch": 0.8558558558558559,
1026
  "frac_reward_zero_std": 0.0,
1027
- "grad_norm": 0.8876456022262573,
1028
- "kl": 0.017240419032168573,
1029
- "learning_rate": 7.857142857142857e-07,
1030
- "loss": 0.0006895914673805236,
1031
- "num_tokens": 772888.0,
1032
- "reward": 0.4967499911785126,
1033
- "reward_std": 0.16355379521846772,
1034
- "rewards/reward_fn/mean": 0.4967499911785126,
1035
- "rewards/reward_fn/std": 0.14962645918130874,
1036
  "step": 190
1037
  },
1038
  {
@@ -1041,25 +1041,25 @@
1041
  "clip_ratio/low_mean": 0.0,
1042
  "clip_ratio/low_min": 0.0,
1043
  "clip_ratio/region_mean": 0.0,
1044
- "completion_length": 26.8,
1045
- "completions/clipped_ratio": 0.025,
1046
- "completions/max_length": 45.2,
1047
- "completions/max_terminated_length": 31.2,
1048
- "completions/mean_length": 26.8,
1049
- "completions/mean_terminated_length": 25.014286041259766,
1050
- "completions/min_length": 18.4,
1051
- "completions/min_terminated_length": 18.4,
1052
- "epoch": 0.8783783783783784,
1053
  "frac_reward_zero_std": 0.0,
1054
- "grad_norm": 1.3262248039245605,
1055
- "kl": 0.01487510468577966,
1056
- "learning_rate": 6.666666666666667e-07,
1057
- "loss": 0.0005949988961219787,
1058
- "num_tokens": 792706.0,
1059
- "reward": 0.5399499893188476,
1060
- "reward_std": 0.17628171145915986,
1061
- "rewards/reward_fn/mean": 0.5399500131607056,
1062
- "rewards/reward_fn/std": 0.1639223352074623,
1063
  "step": 195
1064
  },
1065
  {
@@ -1068,31 +1068,31 @@
1068
  "clip_ratio/low_mean": 0.0,
1069
  "clip_ratio/low_min": 0.0,
1070
  "clip_ratio/region_mean": 0.0,
1071
- "completion_length": 25.4,
1072
  "completions/clipped_ratio": 0.0,
1073
- "completions/max_length": 34.8,
1074
- "completions/max_terminated_length": 34.8,
1075
- "completions/mean_length": 25.4,
1076
- "completions/mean_terminated_length": 25.4,
1077
- "completions/min_length": 18.0,
1078
- "completions/min_terminated_length": 18.0,
1079
- "epoch": 0.9009009009009009,
1080
  "frac_reward_zero_std": 0.0,
1081
- "grad_norm": 1.9707448482513428,
1082
- "kl": 0.02140549005125649,
1083
- "learning_rate": 5.476190476190477e-07,
1084
- "loss": 0.0008562013506889343,
1085
- "num_tokens": 813816.0,
1086
- "reward": 0.5184999942779541,
1087
- "reward_std": 0.15726054608821868,
1088
- "rewards/reward_fn/mean": 0.5184999942779541,
1089
- "rewards/reward_fn/std": 0.15672532767057418,
1090
  "step": 200
1091
  }
1092
  ],
1093
  "logging_steps": 5,
1094
- "max_steps": 222,
1095
- "num_input_tokens_seen": 813816,
1096
  "num_train_epochs": 1,
1097
  "save_steps": 50,
1098
  "stateful_callbacks": {
 
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
+ "epoch": 0.8888888888888888,
6
  "eval_steps": 500,
7
  "global_step": 200,
8
  "is_hyper_param_search": false,
 
15
  "clip_ratio/low_mean": 0.0,
16
  "clip_ratio/low_min": 0.0,
17
  "clip_ratio/region_mean": 0.0,
18
+ "completion_length": 25.075,
19
+ "completions/clipped_ratio": 0.0,
20
+ "completions/max_length": 30.8,
21
+ "completions/max_terminated_length": 30.8,
22
+ "completions/mean_length": 25.075,
23
+ "completions/mean_terminated_length": 25.075,
24
+ "completions/min_length": 19.4,
25
+ "completions/min_terminated_length": 19.4,
26
+ "epoch": 0.022222222222222223,
27
+ "frac_reward_zero_std": 0.6,
28
+ "grad_norm": 1.3092279434204102,
29
+ "kl": 0.03290070100920275,
30
  "learning_rate": 1.6666666666666667e-06,
31
+ "loss": 0.0013160213828086853,
32
+ "num_tokens": 20275.0,
33
+ "reward": 0.48624999523162843,
34
+ "reward_std": 0.05833630859851837,
35
+ "rewards/reward_fn/mean": 0.4862500011920929,
36
+ "rewards/reward_fn/std": 0.04605271518230438,
37
  "step": 5
38
  },
39
  {
 
42
  "clip_ratio/low_mean": 0.0,
43
  "clip_ratio/low_min": 0.0,
44
  "clip_ratio/region_mean": 0.0,
45
+ "completion_length": 25.925,
46
  "completions/clipped_ratio": 0.0,
47
+ "completions/max_length": 33.0,
48
+ "completions/max_terminated_length": 33.0,
49
+ "completions/mean_length": 25.925,
50
+ "completions/mean_terminated_length": 25.925,
51
+ "completions/min_length": 19.4,
52
+ "completions/min_terminated_length": 19.4,
53
+ "epoch": 0.044444444444444446,
54
  "frac_reward_zero_std": 0.0,
55
+ "grad_norm": 1.4204450845718384,
56
+ "kl": 0.040759827199508436,
57
  "learning_rate": 3.7500000000000005e-06,
58
+ "loss": 0.0016303554177284241,
59
+ "num_tokens": 41560.0,
60
+ "reward": 0.5283999860286712,
61
+ "reward_std": 0.17366542518138886,
62
+ "rewards/reward_fn/mean": 0.5283999919891358,
63
+ "rewards/reward_fn/std": 0.16809422075748442,
64
  "step": 10
65
  },
66
  {
 
69
  "clip_ratio/low_mean": 0.0,
70
  "clip_ratio/low_min": 0.0,
71
  "clip_ratio/region_mean": 0.0,
72
+ "completion_length": 27.45,
73
  "completions/clipped_ratio": 0.0,
74
+ "completions/max_length": 33.6,
75
+ "completions/max_terminated_length": 33.6,
76
+ "completions/mean_length": 27.45,
77
+ "completions/mean_terminated_length": 27.45,
78
+ "completions/min_length": 24.0,
79
+ "completions/min_terminated_length": 24.0,
80
+ "epoch": 0.06666666666666667,
81
  "frac_reward_zero_std": 0.0,
82
+ "grad_norm": 0.9515441656112671,
83
+ "kl": 0.049849181214813146,
84
+ "learning_rate": 4.953051643192488e-06,
85
+ "loss": 0.0019938603043556215,
86
+ "num_tokens": 62246.0,
87
+ "reward": 0.4975499987602234,
88
+ "reward_std": 0.1564827263355255,
89
+ "rewards/reward_fn/mean": 0.49755001068115234,
90
+ "rewards/reward_fn/std": 0.14618260115385057,
91
  "step": 15
92
  },
93
  {
 
96
  "clip_ratio/low_mean": 0.0,
97
  "clip_ratio/low_min": 0.0,
98
  "clip_ratio/region_mean": 0.0,
99
+ "completion_length": 26.525,
100
  "completions/clipped_ratio": 0.0,
101
+ "completions/max_length": 33.0,
102
+ "completions/max_terminated_length": 33.0,
103
+ "completions/mean_length": 26.525,
104
+ "completions/mean_terminated_length": 26.525,
105
+ "completions/min_length": 22.2,
106
+ "completions/min_terminated_length": 22.2,
107
+ "epoch": 0.08888888888888889,
108
  "frac_reward_zero_std": 0.0,
109
+ "grad_norm": 0.6862999200820923,
110
+ "kl": 0.04350169296376407,
111
+ "learning_rate": 4.835680751173709e-06,
112
+ "loss": 0.0017400771379470826,
113
+ "num_tokens": 81479.0,
114
+ "reward": 0.5159000098705292,
115
+ "reward_std": 0.17465537190437316,
116
+ "rewards/reward_fn/mean": 0.5158999919891357,
117
+ "rewards/reward_fn/std": 0.16455023884773254,
118
  "step": 20
119
  },
120
  {
 
123
  "clip_ratio/low_mean": 0.0,
124
  "clip_ratio/low_min": 0.0,
125
  "clip_ratio/region_mean": 0.0,
126
+ "completion_length": 28.0,
127
  "completions/clipped_ratio": 0.0,
128
+ "completions/max_length": 40.6,
129
+ "completions/max_terminated_length": 40.6,
130
+ "completions/mean_length": 28.0,
131
+ "completions/mean_terminated_length": 28.0,
132
+ "completions/min_length": 22.6,
133
+ "completions/min_terminated_length": 22.6,
134
+ "epoch": 0.1111111111111111,
135
  "frac_reward_zero_std": 0.0,
136
+ "grad_norm": 1.7516958713531494,
137
+ "kl": 0.05824573401478119,
138
+ "learning_rate": 4.71830985915493e-06,
139
+ "loss": 0.002329717576503754,
140
+ "num_tokens": 102833.0,
141
+ "reward": 0.5547999978065491,
142
+ "reward_std": 0.18328206837177277,
143
+ "rewards/reward_fn/mean": 0.5547999858856201,
144
+ "rewards/reward_fn/std": 0.1974634051322937,
145
  "step": 25
146
  },
147
  {
 
150
  "clip_ratio/low_mean": 0.0,
151
  "clip_ratio/low_min": 0.0,
152
  "clip_ratio/region_mean": 0.0,
153
+ "completion_length": 28.675,
154
  "completions/clipped_ratio": 0.0,
155
+ "completions/max_length": 37.4,
156
+ "completions/max_terminated_length": 37.4,
157
+ "completions/mean_length": 28.675,
158
+ "completions/mean_terminated_length": 28.675,
159
+ "completions/min_length": 22.2,
160
+ "completions/min_terminated_length": 22.2,
161
+ "epoch": 0.13333333333333333,
162
  "frac_reward_zero_std": 0.0,
163
+ "grad_norm": 1.5611271858215332,
164
+ "kl": 0.09752920938190072,
165
+ "learning_rate": 4.60093896713615e-06,
166
+ "loss": 0.0039011374115943908,
167
+ "num_tokens": 123876.0,
168
+ "reward": 0.5165499806404114,
169
+ "reward_std": 0.16765501499176025,
170
+ "rewards/reward_fn/mean": 0.5165499806404114,
171
+ "rewards/reward_fn/std": 0.16748485416173936,
172
  "step": 30
173
  },
174
  {
 
177
  "clip_ratio/low_mean": 0.0,
178
  "clip_ratio/low_min": 0.0,
179
  "clip_ratio/region_mean": 0.0,
180
+ "completion_length": 27.475,
181
+ "completions/clipped_ratio": 0.0,
182
+ "completions/max_length": 36.8,
183
+ "completions/max_terminated_length": 36.8,
184
+ "completions/mean_length": 27.475,
185
+ "completions/mean_terminated_length": 27.475,
186
+ "completions/min_length": 20.6,
187
+ "completions/min_terminated_length": 20.6,
188
+ "epoch": 0.15555555555555556,
189
  "frac_reward_zero_std": 0.0,
190
+ "grad_norm": 0.9550014138221741,
191
+ "kl": 0.08032013729680329,
192
+ "learning_rate": 4.483568075117371e-06,
193
+ "loss": 0.003212757408618927,
194
+ "num_tokens": 145019.0,
195
+ "reward": 0.5004499793052674,
196
+ "reward_std": 0.18391846716403962,
197
+ "rewards/reward_fn/mean": 0.5004500031471253,
198
+ "rewards/reward_fn/std": 0.16948954164981841,
199
  "step": 35
200
  },
201
  {
 
204
  "clip_ratio/low_mean": 0.0,
205
  "clip_ratio/low_min": 0.0,
206
  "clip_ratio/region_mean": 0.0,
207
+ "completion_length": 32.125,
208
+ "completions/clipped_ratio": 0.025,
209
+ "completions/max_length": 54.6,
210
+ "completions/max_terminated_length": 46.0,
211
+ "completions/mean_length": 32.125,
212
+ "completions/mean_terminated_length": 30.521428680419923,
213
+ "completions/min_length": 23.2,
214
+ "completions/min_terminated_length": 23.2,
215
+ "epoch": 0.17777777777777778,
216
  "frac_reward_zero_std": 0.0,
217
+ "grad_norm": 1.1203869581222534,
218
+ "kl": 0.08563535290304572,
219
+ "learning_rate": 4.3661971830985915e-06,
220
+ "loss": 0.003425435721874237,
221
+ "num_tokens": 164800.0,
222
+ "reward": 0.5472500026226044,
223
+ "reward_std": 0.19579786211252212,
224
+ "rewards/reward_fn/mean": 0.5472500085830688,
225
+ "rewards/reward_fn/std": 0.19857290089130403,
226
  "step": 40
227
  },
228
  {
 
231
  "clip_ratio/low_mean": 0.0,
232
  "clip_ratio/low_min": 0.0,
233
  "clip_ratio/region_mean": 0.0,
234
+ "completion_length": 27.875,
235
  "completions/clipped_ratio": 0.0,
236
+ "completions/max_length": 35.0,
237
+ "completions/max_terminated_length": 35.0,
238
+ "completions/mean_length": 27.875,
239
+ "completions/mean_terminated_length": 27.875,
240
+ "completions/min_length": 21.4,
241
+ "completions/min_terminated_length": 21.4,
242
+ "epoch": 0.2,
243
  "frac_reward_zero_std": 0.0,
244
+ "grad_norm": 0.95042884349823,
245
+ "kl": 0.07295196709455923,
246
+ "learning_rate": 4.248826291079813e-06,
247
+ "loss": 0.0029180020093917845,
248
+ "num_tokens": 185761.0,
249
+ "reward": 0.5225999832153321,
250
+ "reward_std": 0.16489729881286622,
251
+ "rewards/reward_fn/mean": 0.5226000070571899,
252
+ "rewards/reward_fn/std": 0.16541497856378556,
253
  "step": 45
254
  },
255
  {
 
258
  "clip_ratio/low_mean": 0.0,
259
  "clip_ratio/low_min": 0.0,
260
  "clip_ratio/region_mean": 0.0,
261
+ "completion_length": 27.875,
262
  "completions/clipped_ratio": 0.0,
263
  "completions/max_length": 35.2,
264
  "completions/max_terminated_length": 35.2,
265
+ "completions/mean_length": 27.875,
266
+ "completions/mean_terminated_length": 27.875,
267
+ "completions/min_length": 21.4,
268
+ "completions/min_terminated_length": 21.4,
269
+ "epoch": 0.2222222222222222,
270
  "frac_reward_zero_std": 0.0,
271
+ "grad_norm": 1.1037548780441284,
272
+ "kl": 0.08246680488809943,
273
+ "learning_rate": 4.131455399061034e-06,
274
+ "loss": 0.0032985761761665346,
275
+ "num_tokens": 206674.0,
276
+ "reward": 0.5245000004768372,
277
+ "reward_std": 0.1653215616941452,
278
+ "rewards/reward_fn/mean": 0.5245000004768372,
279
+ "rewards/reward_fn/std": 0.16877340227365495,
280
  "step": 50
281
  },
282
  {
 
285
  "clip_ratio/low_mean": 0.0,
286
  "clip_ratio/low_min": 0.0,
287
  "clip_ratio/region_mean": 0.0,
288
+ "completion_length": 29.225,
289
+ "completions/clipped_ratio": 0.0,
290
+ "completions/max_length": 47.4,
291
+ "completions/max_terminated_length": 47.4,
292
+ "completions/mean_length": 29.225,
293
+ "completions/mean_terminated_length": 29.225,
294
+ "completions/min_length": 20.2,
295
+ "completions/min_terminated_length": 20.2,
296
+ "epoch": 0.24444444444444444,
297
  "frac_reward_zero_std": 0.0,
298
+ "grad_norm": 0.8115681409835815,
299
+ "kl": 0.08297618771903217,
300
+ "learning_rate": 4.014084507042254e-06,
301
+ "loss": 0.0033189669251441956,
302
+ "num_tokens": 227351.0,
303
+ "reward": 0.5301500022411346,
304
+ "reward_std": 0.20612163245677947,
305
+ "rewards/reward_fn/mean": 0.5301500082015991,
306
+ "rewards/reward_fn/std": 0.19000594317913055,
307
  "step": 55
308
  },
309
  {
 
312
  "clip_ratio/low_mean": 0.0,
313
  "clip_ratio/low_min": 0.0,
314
  "clip_ratio/region_mean": 0.0,
315
+ "completion_length": 27.475,
316
+ "completions/clipped_ratio": 0.025,
317
+ "completions/max_length": 45.0,
318
+ "completions/max_terminated_length": 31.8,
319
+ "completions/mean_length": 27.475,
320
+ "completions/mean_terminated_length": 25.646428680419923,
321
+ "completions/min_length": 18.8,
322
+ "completions/min_terminated_length": 18.8,
323
+ "epoch": 0.26666666666666666,
324
  "frac_reward_zero_std": 0.0,
325
+ "grad_norm": 1.1722970008850098,
326
+ "kl": 0.06283304298995063,
327
+ "learning_rate": 3.896713615023475e-06,
328
+ "loss": 0.002513286471366882,
329
+ "num_tokens": 246464.0,
330
+ "reward": 0.513349997997284,
331
+ "reward_std": 0.18999958634376526,
332
+ "rewards/reward_fn/mean": 0.5133499920368194,
333
+ "rewards/reward_fn/std": 0.1748345360159874,
334
  "step": 60
335
  },
336
  {
 
339
  "clip_ratio/low_mean": 0.0,
340
  "clip_ratio/low_min": 0.0,
341
  "clip_ratio/region_mean": 0.0,
342
+ "completion_length": 26.075,
343
+ "completions/clipped_ratio": 0.0,
344
+ "completions/max_length": 34.2,
345
+ "completions/max_terminated_length": 34.2,
346
+ "completions/mean_length": 26.075,
347
+ "completions/mean_terminated_length": 26.075,
348
+ "completions/min_length": 19.4,
349
+ "completions/min_terminated_length": 19.4,
350
+ "epoch": 0.28888888888888886,
351
  "frac_reward_zero_std": 0.0,
352
+ "grad_norm": 1.2825119495391846,
353
+ "kl": 0.07132846353342756,
354
+ "learning_rate": 3.779342723004695e-06,
355
+ "loss": 0.0028530970215797425,
356
+ "num_tokens": 267279.0,
357
+ "reward": 0.5383000016212464,
358
+ "reward_std": 0.20746512711048126,
359
+ "rewards/reward_fn/mean": 0.5383000195026397,
360
+ "rewards/reward_fn/std": 0.1835445523262024,
361
  "step": 65
362
  },
363
  {
 
366
  "clip_ratio/low_mean": 0.0,
367
  "clip_ratio/low_min": 0.0,
368
  "clip_ratio/region_mean": 0.0,
369
+ "completion_length": 27.925,
370
  "completions/clipped_ratio": 0.0,
371
+ "completions/max_length": 44.0,
372
+ "completions/max_terminated_length": 44.0,
373
+ "completions/mean_length": 27.925,
374
+ "completions/mean_terminated_length": 27.925,
375
+ "completions/min_length": 22.0,
376
+ "completions/min_terminated_length": 22.0,
377
+ "epoch": 0.3111111111111111,
378
  "frac_reward_zero_std": 0.0,
379
+ "grad_norm": 0.5832120776176453,
380
+ "kl": 0.05197672065114602,
381
+ "learning_rate": 3.6619718309859158e-06,
382
+ "loss": 0.0020790368318557738,
383
+ "num_tokens": 287984.0,
384
+ "reward": 0.5424999892711639,
385
+ "reward_std": 0.18002938330173493,
386
+ "rewards/reward_fn/mean": 0.5424999952316284,
387
+ "rewards/reward_fn/std": 0.18769851326942444,
388
  "step": 70
389
  },
390
  {
 
393
  "clip_ratio/low_mean": 0.0,
394
  "clip_ratio/low_min": 0.0,
395
  "clip_ratio/region_mean": 0.0,
396
+ "completion_length": 27.6,
397
+ "completions/clipped_ratio": 0.025,
398
+ "completions/max_length": 44.6,
399
+ "completions/max_terminated_length": 30.6,
400
+ "completions/mean_length": 27.6,
401
+ "completions/mean_terminated_length": 25.807143020629884,
402
+ "completions/min_length": 22.2,
403
+ "completions/min_terminated_length": 22.2,
404
+ "epoch": 0.3333333333333333,
405
  "frac_reward_zero_std": 0.0,
406
+ "grad_norm": 0.6677760481834412,
407
+ "kl": 0.03671608620206825,
408
+ "learning_rate": 3.5446009389671364e-06,
409
+ "loss": 0.0014685407280921937,
410
+ "num_tokens": 307800.0,
411
+ "reward": 0.5715999841690064,
412
+ "reward_std": 0.18639334440231323,
413
+ "rewards/reward_fn/mean": 0.5715999960899353,
414
+ "rewards/reward_fn/std": 0.19429495334625244,
415
  "step": 75
416
  },
417
  {
 
420
  "clip_ratio/low_mean": 0.0,
421
  "clip_ratio/low_min": 0.0,
422
  "clip_ratio/region_mean": 0.0,
423
+ "completion_length": 25.6,
424
  "completions/clipped_ratio": 0.0,
425
+ "completions/max_length": 30.2,
426
+ "completions/max_terminated_length": 30.2,
427
+ "completions/mean_length": 25.6,
428
+ "completions/mean_terminated_length": 25.6,
429
+ "completions/min_length": 21.4,
430
+ "completions/min_terminated_length": 21.4,
431
+ "epoch": 0.35555555555555557,
432
  "frac_reward_zero_std": 0.0,
433
+ "grad_norm": 0.7261527180671692,
434
+ "kl": 0.05616153636947274,
435
+ "learning_rate": 3.427230046948357e-06,
436
+ "loss": 0.0022463813424110413,
437
+ "num_tokens": 327794.0,
438
+ "reward": 0.5464499950408935,
439
+ "reward_std": 0.18264567852020264,
440
+ "rewards/reward_fn/mean": 0.5464499950408935,
441
+ "rewards/reward_fn/std": 0.17480863779783248,
442
  "step": 80
443
  },
444
  {
 
447
  "clip_ratio/low_mean": 0.0,
448
  "clip_ratio/low_min": 0.0,
449
  "clip_ratio/region_mean": 0.0,
450
+ "completion_length": 24.075,
451
  "completions/clipped_ratio": 0.0,
452
+ "completions/max_length": 29.6,
453
+ "completions/max_terminated_length": 29.6,
454
+ "completions/mean_length": 24.075,
455
+ "completions/mean_terminated_length": 24.075,
456
+ "completions/min_length": 16.6,
457
+ "completions/min_terminated_length": 16.6,
458
+ "epoch": 0.37777777777777777,
459
  "frac_reward_zero_std": 0.0,
460
+ "grad_norm": 1.1086981296539307,
461
+ "kl": 0.07743949705036357,
462
+ "learning_rate": 3.3098591549295777e-06,
463
+ "loss": 0.0030974715948104857,
464
+ "num_tokens": 348233.0,
465
+ "reward": 0.5536999821662902,
466
+ "reward_std": 0.18073648810386658,
467
+ "rewards/reward_fn/mean": 0.5536999821662902,
468
+ "rewards/reward_fn/std": 0.1820658951997757,
469
  "step": 85
470
  },
471
  {
 
474
  "clip_ratio/low_mean": 0.0,
475
  "clip_ratio/low_min": 0.0,
476
  "clip_ratio/region_mean": 0.0,
477
+ "completion_length": 25.375,
478
+ "completions/clipped_ratio": 0.0,
479
+ "completions/max_length": 34.4,
480
+ "completions/max_terminated_length": 34.4,
481
+ "completions/mean_length": 25.375,
482
+ "completions/mean_terminated_length": 25.375,
483
+ "completions/min_length": 18.6,
484
+ "completions/min_terminated_length": 18.6,
485
+ "epoch": 0.4,
486
  "frac_reward_zero_std": 0.0,
487
+ "grad_norm": 1.1903071403503418,
488
+ "kl": 0.058934826811309904,
489
+ "learning_rate": 3.1924882629107983e-06,
490
+ "loss": 0.002357317507266998,
491
+ "num_tokens": 367732.0,
492
+ "reward": 0.5361499905586242,
493
+ "reward_std": 0.179675829410553,
494
+ "rewards/reward_fn/mean": 0.5361500144004822,
495
+ "rewards/reward_fn/std": 0.18574500381946563,
496
  "step": 90
497
  },
498
  {
 
501
  "clip_ratio/low_mean": 0.0,
502
  "clip_ratio/low_min": 0.0,
503
  "clip_ratio/region_mean": 0.0,
504
+ "completion_length": 26.825,
505
  "completions/clipped_ratio": 0.0,
506
+ "completions/max_length": 37.2,
507
+ "completions/max_terminated_length": 37.2,
508
+ "completions/mean_length": 26.825,
509
+ "completions/mean_terminated_length": 26.825,
510
  "completions/min_length": 20.8,
511
  "completions/min_terminated_length": 20.8,
512
+ "epoch": 0.4222222222222222,
513
  "frac_reward_zero_std": 0.0,
514
+ "grad_norm": 1.0952492952346802,
515
+ "kl": 0.040232469444163144,
516
+ "learning_rate": 3.075117370892019e-06,
517
+ "loss": 0.0016092658042907715,
518
+ "num_tokens": 387075.0,
519
+ "reward": 0.5278500020503998,
520
+ "reward_std": 0.1932522773742676,
521
+ "rewards/reward_fn/mean": 0.5278499901294709,
522
+ "rewards/reward_fn/std": 0.1796583503484726,
523
  "step": 95
524
  },
525
  {
 
528
  "clip_ratio/low_mean": 0.0,
529
  "clip_ratio/low_min": 0.0,
530
  "clip_ratio/region_mean": 0.0,
531
+ "completion_length": 26.125,
532
+ "completions/clipped_ratio": 0.0,
533
+ "completions/max_length": 35.6,
534
+ "completions/max_terminated_length": 35.6,
535
+ "completions/mean_length": 26.125,
536
+ "completions/mean_terminated_length": 26.125,
537
+ "completions/min_length": 22.6,
538
+ "completions/min_terminated_length": 22.6,
539
+ "epoch": 0.4444444444444444,
540
  "frac_reward_zero_std": 0.0,
541
+ "grad_norm": 0.8072858452796936,
542
+ "kl": 0.06188266044482589,
543
+ "learning_rate": 2.9577464788732396e-06,
544
+ "loss": 0.0024752289056777952,
545
+ "num_tokens": 406884.0,
546
+ "reward": 0.5849499881267548,
547
+ "reward_std": 0.20499025285243988,
548
+ "rewards/reward_fn/mean": 0.5849500000476837,
549
+ "rewards/reward_fn/std": 0.2056175708770752,
550
  "step": 100
551
  },
552
  {
 
555
  "clip_ratio/low_mean": 0.0,
556
  "clip_ratio/low_min": 0.0,
557
  "clip_ratio/region_mean": 0.0,
558
+ "completion_length": 26.45,
559
+ "completions/clipped_ratio": 0.025,
560
+ "completions/max_length": 43.2,
561
+ "completions/max_terminated_length": 29.0,
562
+ "completions/mean_length": 26.45,
563
+ "completions/mean_terminated_length": 24.639286041259766,
564
+ "completions/min_length": 20.4,
565
+ "completions/min_terminated_length": 20.4,
566
+ "epoch": 0.4666666666666667,
567
  "frac_reward_zero_std": 0.0,
568
+ "grad_norm": 0.9750375747680664,
569
+ "kl": 0.0710214663646184,
570
+ "learning_rate": 2.8403755868544603e-06,
571
+ "loss": 0.0028407976031303407,
572
+ "num_tokens": 427518.0,
573
+ "reward": 0.5496499896049499,
574
+ "reward_std": 0.16298811435699462,
575
+ "rewards/reward_fn/mean": 0.5496499896049499,
576
+ "rewards/reward_fn/std": 0.15672676265239716,
577
  "step": 105
578
  },
579
  {
 
582
  "clip_ratio/low_mean": 0.0,
583
  "clip_ratio/low_min": 0.0,
584
  "clip_ratio/region_mean": 0.0,
585
+ "completion_length": 25.925,
586
  "completions/clipped_ratio": 0.0,
587
+ "completions/max_length": 31.8,
588
+ "completions/max_terminated_length": 31.8,
589
+ "completions/mean_length": 25.925,
590
+ "completions/mean_terminated_length": 25.925,
591
+ "completions/min_length": 21.6,
592
+ "completions/min_terminated_length": 21.6,
593
+ "epoch": 0.4888888888888889,
594
  "frac_reward_zero_std": 0.0,
595
+ "grad_norm": 1.0184260606765747,
596
+ "kl": 0.0396142341895029,
597
+ "learning_rate": 2.723004694835681e-06,
598
+ "loss": 0.0015845373272895813,
599
+ "num_tokens": 447807.0,
600
+ "reward": 0.5718499898910523,
601
+ "reward_std": 0.20725298821926116,
602
+ "rewards/reward_fn/mean": 0.5718500018119812,
603
+ "rewards/reward_fn/std": 0.214348441362381,
604
  "step": 110
605
  },
606
  {
 
609
  "clip_ratio/low_mean": 0.0,
610
  "clip_ratio/low_min": 0.0,
611
  "clip_ratio/region_mean": 0.0,
612
+ "completion_length": 26.225,
613
+ "completions/clipped_ratio": 0.0,
614
+ "completions/max_length": 31.2,
615
+ "completions/max_terminated_length": 31.2,
616
+ "completions/mean_length": 26.225,
617
+ "completions/mean_terminated_length": 26.225,
618
+ "completions/min_length": 22.4,
619
+ "completions/min_terminated_length": 22.4,
620
+ "epoch": 0.5111111111111111,
621
  "frac_reward_zero_std": 0.0,
622
+ "grad_norm": 0.8158187866210938,
623
+ "kl": 0.048210305260727185,
624
+ "learning_rate": 2.6056338028169015e-06,
625
+ "loss": 0.0019283831119537354,
626
+ "num_tokens": 468650.0,
627
+ "reward": 0.5343500077724457,
628
+ "reward_std": 0.1919794887304306,
629
+ "rewards/reward_fn/mean": 0.5343500018119812,
630
+ "rewards/reward_fn/std": 0.17689327299594879,
631
  "step": 115
632
  },
633
  {
 
636
  "clip_ratio/low_mean": 0.0,
637
  "clip_ratio/low_min": 0.0,
638
  "clip_ratio/region_mean": 0.0,
639
+ "completion_length": 25.875,
640
+ "completions/clipped_ratio": 0.0,
641
+ "completions/max_length": 30.0,
642
+ "completions/max_terminated_length": 30.0,
643
+ "completions/mean_length": 25.875,
644
+ "completions/mean_terminated_length": 25.875,
645
+ "completions/min_length": 24.2,
646
+ "completions/min_terminated_length": 24.2,
647
+ "epoch": 0.5333333333333333,
648
+ "frac_reward_zero_std": 0.05,
649
+ "grad_norm": 0.8932302594184875,
650
+ "kl": 0.05402324852766469,
651
+ "learning_rate": 2.488262910798122e-06,
652
+ "loss": 0.0021608427166938783,
653
+ "num_tokens": 489051.0,
654
+ "reward": 0.6121499896049499,
655
+ "reward_std": 0.16680648624897004,
656
+ "rewards/reward_fn/mean": 0.612150001525879,
657
+ "rewards/reward_fn/std": 0.18321824073791504,
658
  "step": 120
659
  },
660
  {
 
663
  "clip_ratio/low_mean": 0.0,
664
  "clip_ratio/low_min": 0.0,
665
  "clip_ratio/region_mean": 0.0,
666
+ "completion_length": 27.3,
667
+ "completions/clipped_ratio": 0.025,
668
+ "completions/max_length": 42.6,
669
+ "completions/max_terminated_length": 29.8,
670
+ "completions/mean_length": 27.3,
671
+ "completions/mean_terminated_length": 25.514286041259766,
672
+ "completions/min_length": 21.4,
673
+ "completions/min_terminated_length": 21.4,
674
+ "epoch": 0.5555555555555556,
675
  "frac_reward_zero_std": 0.0,
676
+ "grad_norm": 0.922471821308136,
677
+ "kl": 0.04857689954806119,
678
+ "learning_rate": 2.370892018779343e-06,
679
+ "loss": 0.0019433587789535523,
680
+ "num_tokens": 508231.0,
681
+ "reward": 0.5884499907493591,
682
+ "reward_std": 0.17119054943323136,
683
+ "rewards/reward_fn/mean": 0.588450014591217,
684
+ "rewards/reward_fn/std": 0.18389662504196166,
685
  "step": 125
686
  },
687
  {
 
690
  "clip_ratio/low_mean": 0.0,
691
  "clip_ratio/low_min": 0.0,
692
  "clip_ratio/region_mean": 0.0,
693
+ "completion_length": 26.05,
694
  "completions/clipped_ratio": 0.0,
695
+ "completions/max_length": 31.0,
696
+ "completions/max_terminated_length": 31.0,
697
+ "completions/mean_length": 26.05,
698
+ "completions/mean_terminated_length": 26.05,
699
+ "completions/min_length": 22.0,
700
+ "completions/min_terminated_length": 22.0,
701
+ "epoch": 0.5777777777777777,
702
  "frac_reward_zero_std": 0.0,
703
+ "grad_norm": 0.7222557067871094,
704
+ "kl": 0.06997429557377473,
705
+ "learning_rate": 2.2535211267605635e-06,
706
+ "loss": 0.0027989834547042848,
707
+ "num_tokens": 528729.0,
708
+ "reward": 0.5927999973297119,
709
+ "reward_std": 0.22429427206516267,
710
+ "rewards/reward_fn/mean": 0.5928000092506409,
711
+ "rewards/reward_fn/std": 0.21314262747764587,
712
  "step": 130
713
  },
714
  {
 
717
  "clip_ratio/low_mean": 0.0,
718
  "clip_ratio/low_min": 0.0,
719
  "clip_ratio/region_mean": 0.0,
720
+ "completion_length": 25.925,
721
  "completions/clipped_ratio": 0.0,
722
+ "completions/max_length": 35.6,
723
+ "completions/max_terminated_length": 35.6,
724
+ "completions/mean_length": 25.925,
725
+ "completions/mean_terminated_length": 25.925,
726
+ "completions/min_length": 18.8,
727
+ "completions/min_terminated_length": 18.8,
728
+ "epoch": 0.6,
729
  "frac_reward_zero_std": 0.0,
730
+ "grad_norm": 1.2586678266525269,
731
+ "kl": 0.08584307442652062,
732
+ "learning_rate": 2.136150234741784e-06,
733
+ "loss": 0.0034336388111114503,
734
+ "num_tokens": 548748.0,
735
+ "reward": 0.5738499760627747,
736
+ "reward_std": 0.17571603059768676,
737
+ "rewards/reward_fn/mean": 0.5738499999046326,
738
+ "rewards/reward_fn/std": 0.19411510229110718,
739
  "step": 135
740
  },
741
  {
 
744
  "clip_ratio/low_mean": 0.0,
745
  "clip_ratio/low_min": 0.0,
746
  "clip_ratio/region_mean": 0.0,
747
+ "completion_length": 25.9,
748
  "completions/clipped_ratio": 0.0,
749
+ "completions/max_length": 29.4,
750
+ "completions/max_terminated_length": 29.4,
751
+ "completions/mean_length": 25.9,
752
+ "completions/mean_terminated_length": 25.9,
753
+ "completions/min_length": 23.4,
754
+ "completions/min_terminated_length": 23.4,
755
+ "epoch": 0.6222222222222222,
756
  "frac_reward_zero_std": 0.0,
757
+ "grad_norm": 1.0420219898223877,
758
+ "kl": 0.06210445412434638,
759
+ "learning_rate": 2.0187793427230047e-06,
760
+ "loss": 0.0024841248989105223,
761
+ "num_tokens": 569094.0,
762
+ "reward": 0.6095999956130982,
763
+ "reward_std": 0.2001112163066864,
764
+ "rewards/reward_fn/mean": 0.6095999956130982,
765
+ "rewards/reward_fn/std": 0.18984024077653885,
766
  "step": 140
767
  },
768
  {
 
771
  "clip_ratio/low_mean": 0.0,
772
  "clip_ratio/low_min": 0.0,
773
  "clip_ratio/region_mean": 0.0,
774
+ "completion_length": 27.85,
775
+ "completions/clipped_ratio": 0.025,
776
+ "completions/max_length": 46.0,
777
+ "completions/max_terminated_length": 34.8,
778
+ "completions/mean_length": 27.85,
779
+ "completions/mean_terminated_length": 26.164286041259764,
780
+ "completions/min_length": 21.2,
781
+ "completions/min_terminated_length": 21.2,
782
+ "epoch": 0.6444444444444445,
783
  "frac_reward_zero_std": 0.0,
784
+ "grad_norm": 0.9585386514663696,
785
+ "kl": 0.04591481959214434,
786
+ "learning_rate": 1.9014084507042254e-06,
787
+ "loss": 0.0018365621566772462,
788
+ "num_tokens": 589932.0,
789
+ "reward": 0.5361499905586242,
790
+ "reward_std": 0.20909147560596467,
791
+ "rewards/reward_fn/mean": 0.5361500024795532,
792
+ "rewards/reward_fn/std": 0.1902428910136223,
793
  "step": 145
794
  },
795
  {
 
798
  "clip_ratio/low_mean": 0.0,
799
  "clip_ratio/low_min": 0.0,
800
  "clip_ratio/region_mean": 0.0,
801
+ "completion_length": 25.25,
802
  "completions/clipped_ratio": 0.0,
803
+ "completions/max_length": 29.6,
804
+ "completions/max_terminated_length": 29.6,
805
+ "completions/mean_length": 25.25,
806
+ "completions/mean_terminated_length": 25.25,
807
+ "completions/min_length": 21.6,
808
+ "completions/min_terminated_length": 21.6,
809
+ "epoch": 0.6666666666666666,
810
  "frac_reward_zero_std": 0.0,
811
+ "grad_norm": 0.45393237471580505,
812
+ "kl": 0.04076897802297026,
813
+ "learning_rate": 1.784037558685446e-06,
814
+ "loss": 0.001630684733390808,
815
+ "num_tokens": 610754.0,
816
+ "reward": 0.5666499972343445,
817
+ "reward_std": 0.17826161682605743,
818
+ "rewards/reward_fn/mean": 0.5666500210762024,
819
+ "rewards/reward_fn/std": 0.1894826263189316,
820
  "step": 150
821
  },
822
  {
 
825
  "clip_ratio/low_mean": 0.0,
826
  "clip_ratio/low_min": 0.0,
827
  "clip_ratio/region_mean": 0.0,
828
+ "completion_length": 24.85,
829
  "completions/clipped_ratio": 0.0,
830
+ "completions/max_length": 29.2,
831
+ "completions/max_terminated_length": 29.2,
832
+ "completions/mean_length": 24.85,
833
+ "completions/mean_terminated_length": 24.85,
834
+ "completions/min_length": 21.4,
835
+ "completions/min_terminated_length": 21.4,
836
+ "epoch": 0.6888888888888889,
837
  "frac_reward_zero_std": 0.0,
838
+ "grad_norm": 1.101528525352478,
839
+ "kl": 0.0628763473010622,
840
+ "learning_rate": 1.6666666666666667e-06,
841
+ "loss": 0.0025150284171104433,
842
+ "num_tokens": 632434.0,
843
+ "reward": 0.5634499907493591,
844
+ "reward_std": 0.18957532048225403,
845
+ "rewards/reward_fn/mean": 0.5634500086307526,
846
+ "rewards/reward_fn/std": 0.1851608410477638,
847
  "step": 155
848
  },
849
  {
 
854
  "clip_ratio/region_mean": 0.0,
855
  "completion_length": 25.6,
856
  "completions/clipped_ratio": 0.0,
857
+ "completions/max_length": 30.6,
858
+ "completions/max_terminated_length": 30.6,
859
  "completions/mean_length": 25.6,
860
  "completions/mean_terminated_length": 25.6,
861
+ "completions/min_length": 22.4,
862
+ "completions/min_terminated_length": 22.4,
863
+ "epoch": 0.7111111111111111,
864
  "frac_reward_zero_std": 0.0,
865
+ "grad_norm": 0.593997061252594,
866
+ "kl": 0.08342576812719926,
867
+ "learning_rate": 1.5492957746478873e-06,
868
+ "loss": 0.0033370301127433775,
869
+ "num_tokens": 653686.0,
870
+ "reward": 0.5921499967575073,
871
+ "reward_std": 0.20258608162403108,
872
+ "rewards/reward_fn/mean": 0.5921500205993653,
873
+ "rewards/reward_fn/std": 0.20735768973827362,
874
  "step": 160
875
  },
876
  {
 
879
  "clip_ratio/low_mean": 0.0,
880
  "clip_ratio/low_min": 0.0,
881
  "clip_ratio/region_mean": 0.0,
882
+ "completion_length": 25.175,
883
+ "completions/clipped_ratio": 0.0,
884
+ "completions/max_length": 30.4,
885
+ "completions/max_terminated_length": 30.4,
886
+ "completions/mean_length": 25.175,
887
+ "completions/mean_terminated_length": 25.175,
888
+ "completions/min_length": 20.4,
889
+ "completions/min_terminated_length": 20.4,
890
+ "epoch": 0.7333333333333333,
891
  "frac_reward_zero_std": 0.0,
892
+ "grad_norm": 0.8682815432548523,
893
+ "kl": 0.04647499453276396,
894
+ "learning_rate": 1.4319248826291082e-06,
895
+ "loss": 0.0018589019775390625,
896
+ "num_tokens": 673657.0,
897
+ "reward": 0.5476499795913696,
898
+ "reward_std": 0.18038293421268464,
899
+ "rewards/reward_fn/mean": 0.5476500034332276,
900
+ "rewards/reward_fn/std": 0.20103364586830139,
901
  "step": 165
902
  },
903
  {
 
906
  "clip_ratio/low_mean": 0.0,
907
  "clip_ratio/low_min": 0.0,
908
  "clip_ratio/region_mean": 0.0,
909
+ "completion_length": 24.95,
910
  "completions/clipped_ratio": 0.0,
911
+ "completions/max_length": 28.0,
912
+ "completions/max_terminated_length": 28.0,
913
+ "completions/mean_length": 24.95,
914
+ "completions/mean_terminated_length": 24.95,
915
+ "completions/min_length": 23.2,
916
+ "completions/min_terminated_length": 23.2,
917
+ "epoch": 0.7555555555555555,
918
  "frac_reward_zero_std": 0.0,
919
+ "grad_norm": 0.6287415027618408,
920
+ "kl": 0.0606904512271285,
921
+ "learning_rate": 1.3145539906103288e-06,
922
+ "loss": 0.0024275988340377807,
923
+ "num_tokens": 693125.0,
924
+ "reward": 0.6188999772071838,
925
+ "reward_std": 0.20350532233715057,
926
+ "rewards/reward_fn/mean": 0.6188999891281128,
927
+ "rewards/reward_fn/std": 0.2084890365600586,
928
  "step": 170
929
  },
930
  {
 
933
  "clip_ratio/low_mean": 0.0,
934
  "clip_ratio/low_min": 0.0,
935
  "clip_ratio/region_mean": 0.0,
936
+ "completion_length": 25.175,
937
+ "completions/clipped_ratio": 0.0,
938
+ "completions/max_length": 28.8,
939
+ "completions/max_terminated_length": 28.8,
940
+ "completions/mean_length": 25.175,
941
+ "completions/mean_terminated_length": 25.175,
942
+ "completions/min_length": 21.4,
943
+ "completions/min_terminated_length": 21.4,
944
+ "epoch": 0.7777777777777778,
945
  "frac_reward_zero_std": 0.0,
946
+ "grad_norm": 1.0185426473617554,
947
+ "kl": 0.044300994148943576,
948
+ "learning_rate": 1.1971830985915492e-06,
949
+ "loss": 0.001772068440914154,
950
+ "num_tokens": 713356.0,
951
+ "reward": 0.49915000796318054,
952
+ "reward_std": 0.17090770602226257,
953
+ "rewards/reward_fn/mean": 0.4991500020027161,
954
+ "rewards/reward_fn/std": 0.15009358823299407,
955
  "step": 175
956
  },
957
  {
 
960
  "clip_ratio/low_mean": 0.0,
961
  "clip_ratio/low_min": 0.0,
962
  "clip_ratio/region_mean": 0.0,
963
+ "completion_length": 25.825,
964
+ "completions/clipped_ratio": 0.0,
965
+ "completions/max_length": 30.0,
966
+ "completions/max_terminated_length": 30.0,
967
+ "completions/mean_length": 25.825,
968
+ "completions/mean_terminated_length": 25.825,
969
+ "completions/min_length": 22.2,
970
+ "completions/min_terminated_length": 22.2,
971
+ "epoch": 0.8,
972
  "frac_reward_zero_std": 0.0,
973
+ "grad_norm": 0.8517215251922607,
974
+ "kl": 0.07110593506367877,
975
+ "learning_rate": 1.07981220657277e-06,
976
+ "loss": 0.0028442263603210447,
977
+ "num_tokens": 733613.0,
978
+ "reward": 0.5364999890327453,
979
+ "reward_std": 0.17903943061828614,
980
+ "rewards/reward_fn/mean": 0.5364999890327453,
981
+ "rewards/reward_fn/std": 0.1849100574851036,
982
  "step": 180
983
  },
984
  {
 
987
  "clip_ratio/low_mean": 0.0,
988
  "clip_ratio/low_min": 0.0,
989
  "clip_ratio/region_mean": 0.0,
990
+ "completion_length": 25.725,
991
+ "completions/clipped_ratio": 0.0,
992
+ "completions/max_length": 33.2,
993
+ "completions/max_terminated_length": 33.2,
994
+ "completions/mean_length": 25.725,
995
+ "completions/mean_terminated_length": 25.725,
996
+ "completions/min_length": 21.8,
997
+ "completions/min_terminated_length": 21.8,
998
+ "epoch": 0.8222222222222222,
999
  "frac_reward_zero_std": 0.0,
1000
+ "grad_norm": 1.2276585102081299,
1001
+ "kl": 0.06446516590658576,
1002
+ "learning_rate": 9.624413145539907e-07,
1003
+ "loss": 0.002578553557395935,
1004
+ "num_tokens": 753214.0,
1005
+ "reward": 0.5435999751091003,
1006
+ "reward_std": 0.1755038946866989,
1007
+ "rewards/reward_fn/mean": 0.5436000108718873,
1008
+ "rewards/reward_fn/std": 0.1861019790172577,
1009
  "step": 185
1010
  },
1011
  {
 
1014
  "clip_ratio/low_mean": 0.0,
1015
  "clip_ratio/low_min": 0.0,
1016
  "clip_ratio/region_mean": 0.0,
1017
+ "completion_length": 24.6,
1018
  "completions/clipped_ratio": 0.0,
1019
+ "completions/max_length": 27.4,
1020
+ "completions/max_terminated_length": 27.4,
1021
+ "completions/mean_length": 24.6,
1022
+ "completions/mean_terminated_length": 24.6,
1023
+ "completions/min_length": 20.8,
1024
+ "completions/min_terminated_length": 20.8,
1025
+ "epoch": 0.8444444444444444,
1026
  "frac_reward_zero_std": 0.0,
1027
+ "grad_norm": 0.9992023706436157,
1028
+ "kl": 0.05620210377383046,
1029
+ "learning_rate": 8.450704225352114e-07,
1030
+ "loss": 0.0022480204701423646,
1031
+ "num_tokens": 774816.0,
1032
+ "reward": 0.5924999952316284,
1033
+ "reward_std": 0.1981313169002533,
1034
+ "rewards/reward_fn/mean": 0.5924999952316284,
1035
+ "rewards/reward_fn/std": 0.19494172781705857,
1036
  "step": 190
1037
  },
1038
  {
 
1041
  "clip_ratio/low_mean": 0.0,
1042
  "clip_ratio/low_min": 0.0,
1043
  "clip_ratio/region_mean": 0.0,
1044
+ "completion_length": 25.925,
1045
+ "completions/clipped_ratio": 0.0,
1046
+ "completions/max_length": 32.8,
1047
+ "completions/max_terminated_length": 32.8,
1048
+ "completions/mean_length": 25.925,
1049
+ "completions/mean_terminated_length": 25.925,
1050
+ "completions/min_length": 21.6,
1051
+ "completions/min_terminated_length": 21.6,
1052
+ "epoch": 0.8666666666666667,
1053
  "frac_reward_zero_std": 0.0,
1054
+ "grad_norm": 0.6905612349510193,
1055
+ "kl": 0.061457820073701444,
1056
+ "learning_rate": 7.27699530516432e-07,
1057
+ "loss": 0.0024582624435424806,
1058
+ "num_tokens": 794225.0,
1059
+ "reward": 0.5802499890327454,
1060
+ "reward_std": 0.17005917578935623,
1061
+ "rewards/reward_fn/mean": 0.5802499890327454,
1062
+ "rewards/reward_fn/std": 0.18120778799057008,
1063
  "step": 195
1064
  },
1065
  {
 
1068
  "clip_ratio/low_mean": 0.0,
1069
  "clip_ratio/low_min": 0.0,
1070
  "clip_ratio/region_mean": 0.0,
1071
+ "completion_length": 25.975,
1072
  "completions/clipped_ratio": 0.0,
1073
+ "completions/max_length": 31.6,
1074
+ "completions/max_terminated_length": 31.6,
1075
+ "completions/mean_length": 25.975,
1076
+ "completions/mean_terminated_length": 25.975,
1077
+ "completions/min_length": 23.6,
1078
+ "completions/min_terminated_length": 23.6,
1079
+ "epoch": 0.8888888888888888,
1080
  "frac_reward_zero_std": 0.0,
1081
+ "grad_norm": 0.7173504829406738,
1082
+ "kl": 0.06312856795266271,
1083
+ "learning_rate": 6.103286384976526e-07,
1084
+ "loss": 0.0025250956416130064,
1085
+ "num_tokens": 814240.0,
1086
+ "reward": 0.5612499833106994,
1087
+ "reward_std": 0.20499025285243988,
1088
+ "rewards/reward_fn/mean": 0.561249989271164,
1089
+ "rewards/reward_fn/std": 0.2024638831615448,
1090
  "step": 200
1091
  }
1092
  ],
1093
  "logging_steps": 5,
1094
+ "max_steps": 225,
1095
+ "num_input_tokens_seen": 814240,
1096
  "num_train_epochs": 1,
1097
  "save_steps": 50,
1098
  "stateful_callbacks": {
results/phase1/checkpoint-225/README.md ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
+ - grpo
8
+ - lora
9
+ - sft
10
+ - transformers
11
+ - trl
12
+ - unsloth
13
+ ---
14
+
15
+ # Model Card for Model ID
16
+
17
+ <!-- Provide a quick summary of what the model is/does. -->
18
+
19
+
20
+
21
+ ## Model Details
22
+
23
+ ### Model Description
24
+
25
+ <!-- Provide a longer summary of what this model is. -->
26
+
27
+
28
+
29
+ - **Developed by:** [More Information Needed]
30
+ - **Funded by [optional]:** [More Information Needed]
31
+ - **Shared by [optional]:** [More Information Needed]
32
+ - **Model type:** [More Information Needed]
33
+ - **Language(s) (NLP):** [More Information Needed]
34
+ - **License:** [More Information Needed]
35
+ - **Finetuned from model [optional]:** [More Information Needed]
36
+
37
+ ### Model Sources [optional]
38
+
39
+ <!-- Provide the basic links for the model. -->
40
+
41
+ - **Repository:** [More Information Needed]
42
+ - **Paper [optional]:** [More Information Needed]
43
+ - **Demo [optional]:** [More Information Needed]
44
+
45
+ ## Uses
46
+
47
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
48
+
49
+ ### Direct Use
50
+
51
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
52
+
53
+ [More Information Needed]
54
+
55
+ ### Downstream Use [optional]
56
+
57
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
58
+
59
+ [More Information Needed]
60
+
61
+ ### Out-of-Scope Use
62
+
63
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
64
+
65
+ [More Information Needed]
66
+
67
+ ## Bias, Risks, and Limitations
68
+
69
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
70
+
71
+ [More Information Needed]
72
+
73
+ ### Recommendations
74
+
75
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
76
+
77
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
78
+
79
+ ## How to Get Started with the Model
80
+
81
+ Use the code below to get started with the model.
82
+
83
+ [More Information Needed]
84
+
85
+ ## Training Details
86
+
87
+ ### Training Data
88
+
89
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
90
+
91
+ [More Information Needed]
92
+
93
+ ### Training Procedure
94
+
95
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
96
+
97
+ #### Preprocessing [optional]
98
+
99
+ [More Information Needed]
100
+
101
+
102
+ #### Training Hyperparameters
103
+
104
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
105
+
106
+ #### Speeds, Sizes, Times [optional]
107
+
108
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
109
+
110
+ [More Information Needed]
111
+
112
+ ## Evaluation
113
+
114
+ <!-- This section describes the evaluation protocols and provides the results. -->
115
+
116
+ ### Testing Data, Factors & Metrics
117
+
118
+ #### Testing Data
119
+
120
+ <!-- This should link to a Dataset Card if possible. -->
121
+
122
+ [More Information Needed]
123
+
124
+ #### Factors
125
+
126
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
127
+
128
+ [More Information Needed]
129
+
130
+ #### Metrics
131
+
132
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
133
+
134
+ [More Information Needed]
135
+
136
+ ### Results
137
+
138
+ [More Information Needed]
139
+
140
+ #### Summary
141
+
142
+
143
+
144
+ ## Model Examination [optional]
145
+
146
+ <!-- Relevant interpretability work for the model goes here -->
147
+
148
+ [More Information Needed]
149
+
150
+ ## Environmental Impact
151
+
152
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
153
+
154
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
155
+
156
+ - **Hardware Type:** [More Information Needed]
157
+ - **Hours used:** [More Information Needed]
158
+ - **Cloud Provider:** [More Information Needed]
159
+ - **Compute Region:** [More Information Needed]
160
+ - **Carbon Emitted:** [More Information Needed]
161
+
162
+ ## Technical Specifications [optional]
163
+
164
+ ### Model Architecture and Objective
165
+
166
+ [More Information Needed]
167
+
168
+ ### Compute Infrastructure
169
+
170
+ [More Information Needed]
171
+
172
+ #### Hardware
173
+
174
+ [More Information Needed]
175
+
176
+ #### Software
177
+
178
+ [More Information Needed]
179
+
180
+ ## Citation [optional]
181
+
182
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
183
+
184
+ **BibTeX:**
185
+
186
+ [More Information Needed]
187
+
188
+ **APA:**
189
+
190
+ [More Information Needed]
191
+
192
+ ## Glossary [optional]
193
+
194
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
195
+
196
+ [More Information Needed]
197
+
198
+ ## More Information [optional]
199
+
200
+ [More Information Needed]
201
+
202
+ ## Model Card Authors [optional]
203
+
204
+ [More Information Needed]
205
+
206
+ ## Model Card Contact
207
+
208
+ [More Information Needed]
209
+ ### Framework versions
210
+
211
+ - PEFT 0.18.1
results/phase1/checkpoint-225/adapter_config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": {
6
+ "base_model_class": "Qwen2ForCausalLM",
7
+ "parent_library": "transformers.models.qwen2.modeling_qwen2",
8
+ "unsloth_fixed": true
9
+ },
10
+ "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit",
11
+ "bias": "none",
12
+ "corda_config": null,
13
+ "ensure_weight_tying": false,
14
+ "eva_config": null,
15
+ "exclude_modules": null,
16
+ "fan_in_fan_out": false,
17
+ "inference_mode": true,
18
+ "init_lora_weights": true,
19
+ "layer_replication": null,
20
+ "layers_pattern": null,
21
+ "layers_to_transform": null,
22
+ "loftq_config": {},
23
+ "lora_alpha": 32,
24
+ "lora_bias": false,
25
+ "lora_dropout": 0.05,
26
+ "megatron_config": null,
27
+ "megatron_core": "megatron.core",
28
+ "modules_to_save": null,
29
+ "peft_type": "LORA",
30
+ "peft_version": "0.18.1",
31
+ "qalora_group_size": 16,
32
+ "r": 16,
33
+ "rank_pattern": {},
34
+ "revision": null,
35
+ "target_modules": [
36
+ "gate_proj",
37
+ "q_proj",
38
+ "k_proj",
39
+ "v_proj",
40
+ "down_proj",
41
+ "o_proj",
42
+ "up_proj"
43
+ ],
44
+ "target_parameters": null,
45
+ "task_type": "CAUSAL_LM",
46
+ "trainable_token_indices": null,
47
+ "use_dora": false,
48
+ "use_qalora": false,
49
+ "use_rslora": false
50
+ }
results/phase1/checkpoint-225/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe1beb57ab89a2aeb9d572e64c9dedcaba34759e2e289c74a9a5464cb62a56c2
3
+ size 73911112
results/phase1/checkpoint-225/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
results/phase1/checkpoint-225/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdbc4306df8ca94e7edeb1d150377cede486bf781c6dbed18072b6de15d3becd
3
+ size 37969669
results/phase1/checkpoint-225/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:598d859fd8ba3e6ff2f91b82e9538778a790e49919047c7acc5dac0e0a10b77c
3
+ size 14645
results/phase1/checkpoint-225/scaler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:702984c680199a0e28a46b8d953b4d6a6c82386d777f96e7c9bdcfa92e49f34f
3
+ size 1383
results/phase1/checkpoint-225/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2432e75157bae001c5086109cac2c381c97313a80d5b81c47178c2570034aa82
3
+ size 1465
results/phase1/checkpoint-225/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af5891a15588546db1ac7f2baf8fa94835a51a85c032c39793a55bb048b47446
3
+ size 11422523
results/phase1/checkpoint-225/tokenizer_config.json ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "is_local": false,
9
+ "model_max_length": 32768,
10
+ "pad_token": "<|PAD_TOKEN|>",
11
+ "padding_side": "right",
12
+ "split_special_tokens": false,
13
+ "tokenizer_class": "Qwen2Tokenizer",
14
+ "unk_token": null,
15
+ "added_tokens_decoder": {
16
+ "151643": {
17
+ "content": "<|endoftext|>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ "151644": {
25
+ "content": "<|im_start|>",
26
+ "single_word": false,
27
+ "lstrip": false,
28
+ "rstrip": false,
29
+ "normalized": false,
30
+ "special": true
31
+ },
32
+ "151645": {
33
+ "content": "<|im_end|>",
34
+ "single_word": false,
35
+ "lstrip": false,
36
+ "rstrip": false,
37
+ "normalized": false,
38
+ "special": true
39
+ },
40
+ "151646": {
41
+ "content": "<|object_ref_start|>",
42
+ "single_word": false,
43
+ "lstrip": false,
44
+ "rstrip": false,
45
+ "normalized": false,
46
+ "special": true
47
+ },
48
+ "151647": {
49
+ "content": "<|object_ref_end|>",
50
+ "single_word": false,
51
+ "lstrip": false,
52
+ "rstrip": false,
53
+ "normalized": false,
54
+ "special": true
55
+ },
56
+ "151648": {
57
+ "content": "<|box_start|>",
58
+ "single_word": false,
59
+ "lstrip": false,
60
+ "rstrip": false,
61
+ "normalized": false,
62
+ "special": true
63
+ },
64
+ "151649": {
65
+ "content": "<|box_end|>",
66
+ "single_word": false,
67
+ "lstrip": false,
68
+ "rstrip": false,
69
+ "normalized": false,
70
+ "special": true
71
+ },
72
+ "151650": {
73
+ "content": "<|quad_start|>",
74
+ "single_word": false,
75
+ "lstrip": false,
76
+ "rstrip": false,
77
+ "normalized": false,
78
+ "special": true
79
+ },
80
+ "151651": {
81
+ "content": "<|quad_end|>",
82
+ "single_word": false,
83
+ "lstrip": false,
84
+ "rstrip": false,
85
+ "normalized": false,
86
+ "special": true
87
+ },
88
+ "151652": {
89
+ "content": "<|vision_start|>",
90
+ "single_word": false,
91
+ "lstrip": false,
92
+ "rstrip": false,
93
+ "normalized": false,
94
+ "special": true
95
+ },
96
+ "151653": {
97
+ "content": "<|vision_end|>",
98
+ "single_word": false,
99
+ "lstrip": false,
100
+ "rstrip": false,
101
+ "normalized": false,
102
+ "special": true
103
+ },
104
+ "151654": {
105
+ "content": "<|vision_pad|>",
106
+ "single_word": false,
107
+ "lstrip": false,
108
+ "rstrip": false,
109
+ "normalized": false,
110
+ "special": true
111
+ },
112
+ "151655": {
113
+ "content": "<|image_pad|>",
114
+ "single_word": false,
115
+ "lstrip": false,
116
+ "rstrip": false,
117
+ "normalized": false,
118
+ "special": true
119
+ },
120
+ "151656": {
121
+ "content": "<|video_pad|>",
122
+ "single_word": false,
123
+ "lstrip": false,
124
+ "rstrip": false,
125
+ "normalized": false,
126
+ "special": true
127
+ },
128
+ "151657": {
129
+ "content": "<tool_call>",
130
+ "single_word": false,
131
+ "lstrip": false,
132
+ "rstrip": false,
133
+ "normalized": false,
134
+ "special": false
135
+ },
136
+ "151658": {
137
+ "content": "</tool_call>",
138
+ "single_word": false,
139
+ "lstrip": false,
140
+ "rstrip": false,
141
+ "normalized": false,
142
+ "special": false
143
+ },
144
+ "151659": {
145
+ "content": "<|fim_prefix|>",
146
+ "single_word": false,
147
+ "lstrip": false,
148
+ "rstrip": false,
149
+ "normalized": false,
150
+ "special": false
151
+ },
152
+ "151660": {
153
+ "content": "<|fim_middle|>",
154
+ "single_word": false,
155
+ "lstrip": false,
156
+ "rstrip": false,
157
+ "normalized": false,
158
+ "special": false
159
+ },
160
+ "151661": {
161
+ "content": "<|fim_suffix|>",
162
+ "single_word": false,
163
+ "lstrip": false,
164
+ "rstrip": false,
165
+ "normalized": false,
166
+ "special": false
167
+ },
168
+ "151662": {
169
+ "content": "<|fim_pad|>",
170
+ "single_word": false,
171
+ "lstrip": false,
172
+ "rstrip": false,
173
+ "normalized": false,
174
+ "special": false
175
+ },
176
+ "151663": {
177
+ "content": "<|repo_name|>",
178
+ "single_word": false,
179
+ "lstrip": false,
180
+ "rstrip": false,
181
+ "normalized": false,
182
+ "special": false
183
+ },
184
+ "151664": {
185
+ "content": "<|file_sep|>",
186
+ "single_word": false,
187
+ "lstrip": false,
188
+ "rstrip": false,
189
+ "normalized": false,
190
+ "special": false
191
+ },
192
+ "151665": {
193
+ "content": "<|PAD_TOKEN|>",
194
+ "single_word": false,
195
+ "lstrip": false,
196
+ "rstrip": false,
197
+ "normalized": false,
198
+ "special": true
199
+ }
200
+ }
201
+ }
results/phase1/checkpoint-225/trainer_state.json ADDED
@@ -0,0 +1,1249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 1.0,
6
+ "eval_steps": 500,
7
+ "global_step": 225,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completion_length": 25.075,
19
+ "completions/clipped_ratio": 0.0,
20
+ "completions/max_length": 30.8,
21
+ "completions/max_terminated_length": 30.8,
22
+ "completions/mean_length": 25.075,
23
+ "completions/mean_terminated_length": 25.075,
24
+ "completions/min_length": 19.4,
25
+ "completions/min_terminated_length": 19.4,
26
+ "epoch": 0.022222222222222223,
27
+ "frac_reward_zero_std": 0.6,
28
+ "grad_norm": 1.3092279434204102,
29
+ "kl": 0.03290070100920275,
30
+ "learning_rate": 1.6666666666666667e-06,
31
+ "loss": 0.0013160213828086853,
32
+ "num_tokens": 20275.0,
33
+ "reward": 0.48624999523162843,
34
+ "reward_std": 0.05833630859851837,
35
+ "rewards/reward_fn/mean": 0.4862500011920929,
36
+ "rewards/reward_fn/std": 0.04605271518230438,
37
+ "step": 5
38
+ },
39
+ {
40
+ "clip_ratio/high_max": 0.0,
41
+ "clip_ratio/high_mean": 0.0,
42
+ "clip_ratio/low_mean": 0.0,
43
+ "clip_ratio/low_min": 0.0,
44
+ "clip_ratio/region_mean": 0.0,
45
+ "completion_length": 25.925,
46
+ "completions/clipped_ratio": 0.0,
47
+ "completions/max_length": 33.0,
48
+ "completions/max_terminated_length": 33.0,
49
+ "completions/mean_length": 25.925,
50
+ "completions/mean_terminated_length": 25.925,
51
+ "completions/min_length": 19.4,
52
+ "completions/min_terminated_length": 19.4,
53
+ "epoch": 0.044444444444444446,
54
+ "frac_reward_zero_std": 0.0,
55
+ "grad_norm": 1.4204450845718384,
56
+ "kl": 0.040759827199508436,
57
+ "learning_rate": 3.7500000000000005e-06,
58
+ "loss": 0.0016303554177284241,
59
+ "num_tokens": 41560.0,
60
+ "reward": 0.5283999860286712,
61
+ "reward_std": 0.17366542518138886,
62
+ "rewards/reward_fn/mean": 0.5283999919891358,
63
+ "rewards/reward_fn/std": 0.16809422075748442,
64
+ "step": 10
65
+ },
66
+ {
67
+ "clip_ratio/high_max": 0.0,
68
+ "clip_ratio/high_mean": 0.0,
69
+ "clip_ratio/low_mean": 0.0,
70
+ "clip_ratio/low_min": 0.0,
71
+ "clip_ratio/region_mean": 0.0,
72
+ "completion_length": 27.45,
73
+ "completions/clipped_ratio": 0.0,
74
+ "completions/max_length": 33.6,
75
+ "completions/max_terminated_length": 33.6,
76
+ "completions/mean_length": 27.45,
77
+ "completions/mean_terminated_length": 27.45,
78
+ "completions/min_length": 24.0,
79
+ "completions/min_terminated_length": 24.0,
80
+ "epoch": 0.06666666666666667,
81
+ "frac_reward_zero_std": 0.0,
82
+ "grad_norm": 0.9515441656112671,
83
+ "kl": 0.049849181214813146,
84
+ "learning_rate": 4.953051643192488e-06,
85
+ "loss": 0.0019938603043556215,
86
+ "num_tokens": 62246.0,
87
+ "reward": 0.4975499987602234,
88
+ "reward_std": 0.1564827263355255,
89
+ "rewards/reward_fn/mean": 0.49755001068115234,
90
+ "rewards/reward_fn/std": 0.14618260115385057,
91
+ "step": 15
92
+ },
93
+ {
94
+ "clip_ratio/high_max": 0.0,
95
+ "clip_ratio/high_mean": 0.0,
96
+ "clip_ratio/low_mean": 0.0,
97
+ "clip_ratio/low_min": 0.0,
98
+ "clip_ratio/region_mean": 0.0,
99
+ "completion_length": 26.525,
100
+ "completions/clipped_ratio": 0.0,
101
+ "completions/max_length": 33.0,
102
+ "completions/max_terminated_length": 33.0,
103
+ "completions/mean_length": 26.525,
104
+ "completions/mean_terminated_length": 26.525,
105
+ "completions/min_length": 22.2,
106
+ "completions/min_terminated_length": 22.2,
107
+ "epoch": 0.08888888888888889,
108
+ "frac_reward_zero_std": 0.0,
109
+ "grad_norm": 0.6862999200820923,
110
+ "kl": 0.04350169296376407,
111
+ "learning_rate": 4.835680751173709e-06,
112
+ "loss": 0.0017400771379470826,
113
+ "num_tokens": 81479.0,
114
+ "reward": 0.5159000098705292,
115
+ "reward_std": 0.17465537190437316,
116
+ "rewards/reward_fn/mean": 0.5158999919891357,
117
+ "rewards/reward_fn/std": 0.16455023884773254,
118
+ "step": 20
119
+ },
120
+ {
121
+ "clip_ratio/high_max": 0.0,
122
+ "clip_ratio/high_mean": 0.0,
123
+ "clip_ratio/low_mean": 0.0,
124
+ "clip_ratio/low_min": 0.0,
125
+ "clip_ratio/region_mean": 0.0,
126
+ "completion_length": 28.0,
127
+ "completions/clipped_ratio": 0.0,
128
+ "completions/max_length": 40.6,
129
+ "completions/max_terminated_length": 40.6,
130
+ "completions/mean_length": 28.0,
131
+ "completions/mean_terminated_length": 28.0,
132
+ "completions/min_length": 22.6,
133
+ "completions/min_terminated_length": 22.6,
134
+ "epoch": 0.1111111111111111,
135
+ "frac_reward_zero_std": 0.0,
136
+ "grad_norm": 1.7516958713531494,
137
+ "kl": 0.05824573401478119,
138
+ "learning_rate": 4.71830985915493e-06,
139
+ "loss": 0.002329717576503754,
140
+ "num_tokens": 102833.0,
141
+ "reward": 0.5547999978065491,
142
+ "reward_std": 0.18328206837177277,
143
+ "rewards/reward_fn/mean": 0.5547999858856201,
144
+ "rewards/reward_fn/std": 0.1974634051322937,
145
+ "step": 25
146
+ },
147
+ {
148
+ "clip_ratio/high_max": 0.0,
149
+ "clip_ratio/high_mean": 0.0,
150
+ "clip_ratio/low_mean": 0.0,
151
+ "clip_ratio/low_min": 0.0,
152
+ "clip_ratio/region_mean": 0.0,
153
+ "completion_length": 28.675,
154
+ "completions/clipped_ratio": 0.0,
155
+ "completions/max_length": 37.4,
156
+ "completions/max_terminated_length": 37.4,
157
+ "completions/mean_length": 28.675,
158
+ "completions/mean_terminated_length": 28.675,
159
+ "completions/min_length": 22.2,
160
+ "completions/min_terminated_length": 22.2,
161
+ "epoch": 0.13333333333333333,
162
+ "frac_reward_zero_std": 0.0,
163
+ "grad_norm": 1.5611271858215332,
164
+ "kl": 0.09752920938190072,
165
+ "learning_rate": 4.60093896713615e-06,
166
+ "loss": 0.0039011374115943908,
167
+ "num_tokens": 123876.0,
168
+ "reward": 0.5165499806404114,
169
+ "reward_std": 0.16765501499176025,
170
+ "rewards/reward_fn/mean": 0.5165499806404114,
171
+ "rewards/reward_fn/std": 0.16748485416173936,
172
+ "step": 30
173
+ },
174
+ {
175
+ "clip_ratio/high_max": 0.0,
176
+ "clip_ratio/high_mean": 0.0,
177
+ "clip_ratio/low_mean": 0.0,
178
+ "clip_ratio/low_min": 0.0,
179
+ "clip_ratio/region_mean": 0.0,
180
+ "completion_length": 27.475,
181
+ "completions/clipped_ratio": 0.0,
182
+ "completions/max_length": 36.8,
183
+ "completions/max_terminated_length": 36.8,
184
+ "completions/mean_length": 27.475,
185
+ "completions/mean_terminated_length": 27.475,
186
+ "completions/min_length": 20.6,
187
+ "completions/min_terminated_length": 20.6,
188
+ "epoch": 0.15555555555555556,
189
+ "frac_reward_zero_std": 0.0,
190
+ "grad_norm": 0.9550014138221741,
191
+ "kl": 0.08032013729680329,
192
+ "learning_rate": 4.483568075117371e-06,
193
+ "loss": 0.003212757408618927,
194
+ "num_tokens": 145019.0,
195
+ "reward": 0.5004499793052674,
196
+ "reward_std": 0.18391846716403962,
197
+ "rewards/reward_fn/mean": 0.5004500031471253,
198
+ "rewards/reward_fn/std": 0.16948954164981841,
199
+ "step": 35
200
+ },
201
+ {
202
+ "clip_ratio/high_max": 0.0,
203
+ "clip_ratio/high_mean": 0.0,
204
+ "clip_ratio/low_mean": 0.0,
205
+ "clip_ratio/low_min": 0.0,
206
+ "clip_ratio/region_mean": 0.0,
207
+ "completion_length": 32.125,
208
+ "completions/clipped_ratio": 0.025,
209
+ "completions/max_length": 54.6,
210
+ "completions/max_terminated_length": 46.0,
211
+ "completions/mean_length": 32.125,
212
+ "completions/mean_terminated_length": 30.521428680419923,
213
+ "completions/min_length": 23.2,
214
+ "completions/min_terminated_length": 23.2,
215
+ "epoch": 0.17777777777777778,
216
+ "frac_reward_zero_std": 0.0,
217
+ "grad_norm": 1.1203869581222534,
218
+ "kl": 0.08563535290304572,
219
+ "learning_rate": 4.3661971830985915e-06,
220
+ "loss": 0.003425435721874237,
221
+ "num_tokens": 164800.0,
222
+ "reward": 0.5472500026226044,
223
+ "reward_std": 0.19579786211252212,
224
+ "rewards/reward_fn/mean": 0.5472500085830688,
225
+ "rewards/reward_fn/std": 0.19857290089130403,
226
+ "step": 40
227
+ },
228
+ {
229
+ "clip_ratio/high_max": 0.0,
230
+ "clip_ratio/high_mean": 0.0,
231
+ "clip_ratio/low_mean": 0.0,
232
+ "clip_ratio/low_min": 0.0,
233
+ "clip_ratio/region_mean": 0.0,
234
+ "completion_length": 27.875,
235
+ "completions/clipped_ratio": 0.0,
236
+ "completions/max_length": 35.0,
237
+ "completions/max_terminated_length": 35.0,
238
+ "completions/mean_length": 27.875,
239
+ "completions/mean_terminated_length": 27.875,
240
+ "completions/min_length": 21.4,
241
+ "completions/min_terminated_length": 21.4,
242
+ "epoch": 0.2,
243
+ "frac_reward_zero_std": 0.0,
244
+ "grad_norm": 0.95042884349823,
245
+ "kl": 0.07295196709455923,
246
+ "learning_rate": 4.248826291079813e-06,
247
+ "loss": 0.0029180020093917845,
248
+ "num_tokens": 185761.0,
249
+ "reward": 0.5225999832153321,
250
+ "reward_std": 0.16489729881286622,
251
+ "rewards/reward_fn/mean": 0.5226000070571899,
252
+ "rewards/reward_fn/std": 0.16541497856378556,
253
+ "step": 45
254
+ },
255
+ {
256
+ "clip_ratio/high_max": 0.0,
257
+ "clip_ratio/high_mean": 0.0,
258
+ "clip_ratio/low_mean": 0.0,
259
+ "clip_ratio/low_min": 0.0,
260
+ "clip_ratio/region_mean": 0.0,
261
+ "completion_length": 27.875,
262
+ "completions/clipped_ratio": 0.0,
263
+ "completions/max_length": 35.2,
264
+ "completions/max_terminated_length": 35.2,
265
+ "completions/mean_length": 27.875,
266
+ "completions/mean_terminated_length": 27.875,
267
+ "completions/min_length": 21.4,
268
+ "completions/min_terminated_length": 21.4,
269
+ "epoch": 0.2222222222222222,
270
+ "frac_reward_zero_std": 0.0,
271
+ "grad_norm": 1.1037548780441284,
272
+ "kl": 0.08246680488809943,
273
+ "learning_rate": 4.131455399061034e-06,
274
+ "loss": 0.0032985761761665346,
275
+ "num_tokens": 206674.0,
276
+ "reward": 0.5245000004768372,
277
+ "reward_std": 0.1653215616941452,
278
+ "rewards/reward_fn/mean": 0.5245000004768372,
279
+ "rewards/reward_fn/std": 0.16877340227365495,
280
+ "step": 50
281
+ },
282
+ {
283
+ "clip_ratio/high_max": 0.0,
284
+ "clip_ratio/high_mean": 0.0,
285
+ "clip_ratio/low_mean": 0.0,
286
+ "clip_ratio/low_min": 0.0,
287
+ "clip_ratio/region_mean": 0.0,
288
+ "completion_length": 29.225,
289
+ "completions/clipped_ratio": 0.0,
290
+ "completions/max_length": 47.4,
291
+ "completions/max_terminated_length": 47.4,
292
+ "completions/mean_length": 29.225,
293
+ "completions/mean_terminated_length": 29.225,
294
+ "completions/min_length": 20.2,
295
+ "completions/min_terminated_length": 20.2,
296
+ "epoch": 0.24444444444444444,
297
+ "frac_reward_zero_std": 0.0,
298
+ "grad_norm": 0.8115681409835815,
299
+ "kl": 0.08297618771903217,
300
+ "learning_rate": 4.014084507042254e-06,
301
+ "loss": 0.0033189669251441956,
302
+ "num_tokens": 227351.0,
303
+ "reward": 0.5301500022411346,
304
+ "reward_std": 0.20612163245677947,
305
+ "rewards/reward_fn/mean": 0.5301500082015991,
306
+ "rewards/reward_fn/std": 0.19000594317913055,
307
+ "step": 55
308
+ },
309
+ {
310
+ "clip_ratio/high_max": 0.0,
311
+ "clip_ratio/high_mean": 0.0,
312
+ "clip_ratio/low_mean": 0.0,
313
+ "clip_ratio/low_min": 0.0,
314
+ "clip_ratio/region_mean": 0.0,
315
+ "completion_length": 27.475,
316
+ "completions/clipped_ratio": 0.025,
317
+ "completions/max_length": 45.0,
318
+ "completions/max_terminated_length": 31.8,
319
+ "completions/mean_length": 27.475,
320
+ "completions/mean_terminated_length": 25.646428680419923,
321
+ "completions/min_length": 18.8,
322
+ "completions/min_terminated_length": 18.8,
323
+ "epoch": 0.26666666666666666,
324
+ "frac_reward_zero_std": 0.0,
325
+ "grad_norm": 1.1722970008850098,
326
+ "kl": 0.06283304298995063,
327
+ "learning_rate": 3.896713615023475e-06,
328
+ "loss": 0.002513286471366882,
329
+ "num_tokens": 246464.0,
330
+ "reward": 0.513349997997284,
331
+ "reward_std": 0.18999958634376526,
332
+ "rewards/reward_fn/mean": 0.5133499920368194,
333
+ "rewards/reward_fn/std": 0.1748345360159874,
334
+ "step": 60
335
+ },
336
+ {
337
+ "clip_ratio/high_max": 0.0,
338
+ "clip_ratio/high_mean": 0.0,
339
+ "clip_ratio/low_mean": 0.0,
340
+ "clip_ratio/low_min": 0.0,
341
+ "clip_ratio/region_mean": 0.0,
342
+ "completion_length": 26.075,
343
+ "completions/clipped_ratio": 0.0,
344
+ "completions/max_length": 34.2,
345
+ "completions/max_terminated_length": 34.2,
346
+ "completions/mean_length": 26.075,
347
+ "completions/mean_terminated_length": 26.075,
348
+ "completions/min_length": 19.4,
349
+ "completions/min_terminated_length": 19.4,
350
+ "epoch": 0.28888888888888886,
351
+ "frac_reward_zero_std": 0.0,
352
+ "grad_norm": 1.2825119495391846,
353
+ "kl": 0.07132846353342756,
354
+ "learning_rate": 3.779342723004695e-06,
355
+ "loss": 0.0028530970215797425,
356
+ "num_tokens": 267279.0,
357
+ "reward": 0.5383000016212464,
358
+ "reward_std": 0.20746512711048126,
359
+ "rewards/reward_fn/mean": 0.5383000195026397,
360
+ "rewards/reward_fn/std": 0.1835445523262024,
361
+ "step": 65
362
+ },
363
+ {
364
+ "clip_ratio/high_max": 0.0,
365
+ "clip_ratio/high_mean": 0.0,
366
+ "clip_ratio/low_mean": 0.0,
367
+ "clip_ratio/low_min": 0.0,
368
+ "clip_ratio/region_mean": 0.0,
369
+ "completion_length": 27.925,
370
+ "completions/clipped_ratio": 0.0,
371
+ "completions/max_length": 44.0,
372
+ "completions/max_terminated_length": 44.0,
373
+ "completions/mean_length": 27.925,
374
+ "completions/mean_terminated_length": 27.925,
375
+ "completions/min_length": 22.0,
376
+ "completions/min_terminated_length": 22.0,
377
+ "epoch": 0.3111111111111111,
378
+ "frac_reward_zero_std": 0.0,
379
+ "grad_norm": 0.5832120776176453,
380
+ "kl": 0.05197672065114602,
381
+ "learning_rate": 3.6619718309859158e-06,
382
+ "loss": 0.0020790368318557738,
383
+ "num_tokens": 287984.0,
384
+ "reward": 0.5424999892711639,
385
+ "reward_std": 0.18002938330173493,
386
+ "rewards/reward_fn/mean": 0.5424999952316284,
387
+ "rewards/reward_fn/std": 0.18769851326942444,
388
+ "step": 70
389
+ },
390
+ {
391
+ "clip_ratio/high_max": 0.0,
392
+ "clip_ratio/high_mean": 0.0,
393
+ "clip_ratio/low_mean": 0.0,
394
+ "clip_ratio/low_min": 0.0,
395
+ "clip_ratio/region_mean": 0.0,
396
+ "completion_length": 27.6,
397
+ "completions/clipped_ratio": 0.025,
398
+ "completions/max_length": 44.6,
399
+ "completions/max_terminated_length": 30.6,
400
+ "completions/mean_length": 27.6,
401
+ "completions/mean_terminated_length": 25.807143020629884,
402
+ "completions/min_length": 22.2,
403
+ "completions/min_terminated_length": 22.2,
404
+ "epoch": 0.3333333333333333,
405
+ "frac_reward_zero_std": 0.0,
406
+ "grad_norm": 0.6677760481834412,
407
+ "kl": 0.03671608620206825,
408
+ "learning_rate": 3.5446009389671364e-06,
409
+ "loss": 0.0014685407280921937,
410
+ "num_tokens": 307800.0,
411
+ "reward": 0.5715999841690064,
412
+ "reward_std": 0.18639334440231323,
413
+ "rewards/reward_fn/mean": 0.5715999960899353,
414
+ "rewards/reward_fn/std": 0.19429495334625244,
415
+ "step": 75
416
+ },
417
+ {
418
+ "clip_ratio/high_max": 0.0,
419
+ "clip_ratio/high_mean": 0.0,
420
+ "clip_ratio/low_mean": 0.0,
421
+ "clip_ratio/low_min": 0.0,
422
+ "clip_ratio/region_mean": 0.0,
423
+ "completion_length": 25.6,
424
+ "completions/clipped_ratio": 0.0,
425
+ "completions/max_length": 30.2,
426
+ "completions/max_terminated_length": 30.2,
427
+ "completions/mean_length": 25.6,
428
+ "completions/mean_terminated_length": 25.6,
429
+ "completions/min_length": 21.4,
430
+ "completions/min_terminated_length": 21.4,
431
+ "epoch": 0.35555555555555557,
432
+ "frac_reward_zero_std": 0.0,
433
+ "grad_norm": 0.7261527180671692,
434
+ "kl": 0.05616153636947274,
435
+ "learning_rate": 3.427230046948357e-06,
436
+ "loss": 0.0022463813424110413,
437
+ "num_tokens": 327794.0,
438
+ "reward": 0.5464499950408935,
439
+ "reward_std": 0.18264567852020264,
440
+ "rewards/reward_fn/mean": 0.5464499950408935,
441
+ "rewards/reward_fn/std": 0.17480863779783248,
442
+ "step": 80
443
+ },
444
+ {
445
+ "clip_ratio/high_max": 0.0,
446
+ "clip_ratio/high_mean": 0.0,
447
+ "clip_ratio/low_mean": 0.0,
448
+ "clip_ratio/low_min": 0.0,
449
+ "clip_ratio/region_mean": 0.0,
450
+ "completion_length": 24.075,
451
+ "completions/clipped_ratio": 0.0,
452
+ "completions/max_length": 29.6,
453
+ "completions/max_terminated_length": 29.6,
454
+ "completions/mean_length": 24.075,
455
+ "completions/mean_terminated_length": 24.075,
456
+ "completions/min_length": 16.6,
457
+ "completions/min_terminated_length": 16.6,
458
+ "epoch": 0.37777777777777777,
459
+ "frac_reward_zero_std": 0.0,
460
+ "grad_norm": 1.1086981296539307,
461
+ "kl": 0.07743949705036357,
462
+ "learning_rate": 3.3098591549295777e-06,
463
+ "loss": 0.0030974715948104857,
464
+ "num_tokens": 348233.0,
465
+ "reward": 0.5536999821662902,
466
+ "reward_std": 0.18073648810386658,
467
+ "rewards/reward_fn/mean": 0.5536999821662902,
468
+ "rewards/reward_fn/std": 0.1820658951997757,
469
+ "step": 85
470
+ },
471
+ {
472
+ "clip_ratio/high_max": 0.0,
473
+ "clip_ratio/high_mean": 0.0,
474
+ "clip_ratio/low_mean": 0.0,
475
+ "clip_ratio/low_min": 0.0,
476
+ "clip_ratio/region_mean": 0.0,
477
+ "completion_length": 25.375,
478
+ "completions/clipped_ratio": 0.0,
479
+ "completions/max_length": 34.4,
480
+ "completions/max_terminated_length": 34.4,
481
+ "completions/mean_length": 25.375,
482
+ "completions/mean_terminated_length": 25.375,
483
+ "completions/min_length": 18.6,
484
+ "completions/min_terminated_length": 18.6,
485
+ "epoch": 0.4,
486
+ "frac_reward_zero_std": 0.0,
487
+ "grad_norm": 1.1903071403503418,
488
+ "kl": 0.058934826811309904,
489
+ "learning_rate": 3.1924882629107983e-06,
490
+ "loss": 0.002357317507266998,
491
+ "num_tokens": 367732.0,
492
+ "reward": 0.5361499905586242,
493
+ "reward_std": 0.179675829410553,
494
+ "rewards/reward_fn/mean": 0.5361500144004822,
495
+ "rewards/reward_fn/std": 0.18574500381946563,
496
+ "step": 90
497
+ },
498
+ {
499
+ "clip_ratio/high_max": 0.0,
500
+ "clip_ratio/high_mean": 0.0,
501
+ "clip_ratio/low_mean": 0.0,
502
+ "clip_ratio/low_min": 0.0,
503
+ "clip_ratio/region_mean": 0.0,
504
+ "completion_length": 26.825,
505
+ "completions/clipped_ratio": 0.0,
506
+ "completions/max_length": 37.2,
507
+ "completions/max_terminated_length": 37.2,
508
+ "completions/mean_length": 26.825,
509
+ "completions/mean_terminated_length": 26.825,
510
+ "completions/min_length": 20.8,
511
+ "completions/min_terminated_length": 20.8,
512
+ "epoch": 0.4222222222222222,
513
+ "frac_reward_zero_std": 0.0,
514
+ "grad_norm": 1.0952492952346802,
515
+ "kl": 0.040232469444163144,
516
+ "learning_rate": 3.075117370892019e-06,
517
+ "loss": 0.0016092658042907715,
518
+ "num_tokens": 387075.0,
519
+ "reward": 0.5278500020503998,
520
+ "reward_std": 0.1932522773742676,
521
+ "rewards/reward_fn/mean": 0.5278499901294709,
522
+ "rewards/reward_fn/std": 0.1796583503484726,
523
+ "step": 95
524
+ },
525
+ {
526
+ "clip_ratio/high_max": 0.0,
527
+ "clip_ratio/high_mean": 0.0,
528
+ "clip_ratio/low_mean": 0.0,
529
+ "clip_ratio/low_min": 0.0,
530
+ "clip_ratio/region_mean": 0.0,
531
+ "completion_length": 26.125,
532
+ "completions/clipped_ratio": 0.0,
533
+ "completions/max_length": 35.6,
534
+ "completions/max_terminated_length": 35.6,
535
+ "completions/mean_length": 26.125,
536
+ "completions/mean_terminated_length": 26.125,
537
+ "completions/min_length": 22.6,
538
+ "completions/min_terminated_length": 22.6,
539
+ "epoch": 0.4444444444444444,
540
+ "frac_reward_zero_std": 0.0,
541
+ "grad_norm": 0.8072858452796936,
542
+ "kl": 0.06188266044482589,
543
+ "learning_rate": 2.9577464788732396e-06,
544
+ "loss": 0.0024752289056777952,
545
+ "num_tokens": 406884.0,
546
+ "reward": 0.5849499881267548,
547
+ "reward_std": 0.20499025285243988,
548
+ "rewards/reward_fn/mean": 0.5849500000476837,
549
+ "rewards/reward_fn/std": 0.2056175708770752,
550
+ "step": 100
551
+ },
552
+ {
553
+ "clip_ratio/high_max": 0.0,
554
+ "clip_ratio/high_mean": 0.0,
555
+ "clip_ratio/low_mean": 0.0,
556
+ "clip_ratio/low_min": 0.0,
557
+ "clip_ratio/region_mean": 0.0,
558
+ "completion_length": 26.45,
559
+ "completions/clipped_ratio": 0.025,
560
+ "completions/max_length": 43.2,
561
+ "completions/max_terminated_length": 29.0,
562
+ "completions/mean_length": 26.45,
563
+ "completions/mean_terminated_length": 24.639286041259766,
564
+ "completions/min_length": 20.4,
565
+ "completions/min_terminated_length": 20.4,
566
+ "epoch": 0.4666666666666667,
567
+ "frac_reward_zero_std": 0.0,
568
+ "grad_norm": 0.9750375747680664,
569
+ "kl": 0.0710214663646184,
570
+ "learning_rate": 2.8403755868544603e-06,
571
+ "loss": 0.0028407976031303407,
572
+ "num_tokens": 427518.0,
573
+ "reward": 0.5496499896049499,
574
+ "reward_std": 0.16298811435699462,
575
+ "rewards/reward_fn/mean": 0.5496499896049499,
576
+ "rewards/reward_fn/std": 0.15672676265239716,
577
+ "step": 105
578
+ },
579
+ {
580
+ "clip_ratio/high_max": 0.0,
581
+ "clip_ratio/high_mean": 0.0,
582
+ "clip_ratio/low_mean": 0.0,
583
+ "clip_ratio/low_min": 0.0,
584
+ "clip_ratio/region_mean": 0.0,
585
+ "completion_length": 25.925,
586
+ "completions/clipped_ratio": 0.0,
587
+ "completions/max_length": 31.8,
588
+ "completions/max_terminated_length": 31.8,
589
+ "completions/mean_length": 25.925,
590
+ "completions/mean_terminated_length": 25.925,
591
+ "completions/min_length": 21.6,
592
+ "completions/min_terminated_length": 21.6,
593
+ "epoch": 0.4888888888888889,
594
+ "frac_reward_zero_std": 0.0,
595
+ "grad_norm": 1.0184260606765747,
596
+ "kl": 0.0396142341895029,
597
+ "learning_rate": 2.723004694835681e-06,
598
+ "loss": 0.0015845373272895813,
599
+ "num_tokens": 447807.0,
600
+ "reward": 0.5718499898910523,
601
+ "reward_std": 0.20725298821926116,
602
+ "rewards/reward_fn/mean": 0.5718500018119812,
603
+ "rewards/reward_fn/std": 0.214348441362381,
604
+ "step": 110
605
+ },
606
+ {
607
+ "clip_ratio/high_max": 0.0,
608
+ "clip_ratio/high_mean": 0.0,
609
+ "clip_ratio/low_mean": 0.0,
610
+ "clip_ratio/low_min": 0.0,
611
+ "clip_ratio/region_mean": 0.0,
612
+ "completion_length": 26.225,
613
+ "completions/clipped_ratio": 0.0,
614
+ "completions/max_length": 31.2,
615
+ "completions/max_terminated_length": 31.2,
616
+ "completions/mean_length": 26.225,
617
+ "completions/mean_terminated_length": 26.225,
618
+ "completions/min_length": 22.4,
619
+ "completions/min_terminated_length": 22.4,
620
+ "epoch": 0.5111111111111111,
621
+ "frac_reward_zero_std": 0.0,
622
+ "grad_norm": 0.8158187866210938,
623
+ "kl": 0.048210305260727185,
624
+ "learning_rate": 2.6056338028169015e-06,
625
+ "loss": 0.0019283831119537354,
626
+ "num_tokens": 468650.0,
627
+ "reward": 0.5343500077724457,
628
+ "reward_std": 0.1919794887304306,
629
+ "rewards/reward_fn/mean": 0.5343500018119812,
630
+ "rewards/reward_fn/std": 0.17689327299594879,
631
+ "step": 115
632
+ },
633
+ {
634
+ "clip_ratio/high_max": 0.0,
635
+ "clip_ratio/high_mean": 0.0,
636
+ "clip_ratio/low_mean": 0.0,
637
+ "clip_ratio/low_min": 0.0,
638
+ "clip_ratio/region_mean": 0.0,
639
+ "completion_length": 25.875,
640
+ "completions/clipped_ratio": 0.0,
641
+ "completions/max_length": 30.0,
642
+ "completions/max_terminated_length": 30.0,
643
+ "completions/mean_length": 25.875,
644
+ "completions/mean_terminated_length": 25.875,
645
+ "completions/min_length": 24.2,
646
+ "completions/min_terminated_length": 24.2,
647
+ "epoch": 0.5333333333333333,
648
+ "frac_reward_zero_std": 0.05,
649
+ "grad_norm": 0.8932302594184875,
650
+ "kl": 0.05402324852766469,
651
+ "learning_rate": 2.488262910798122e-06,
652
+ "loss": 0.0021608427166938783,
653
+ "num_tokens": 489051.0,
654
+ "reward": 0.6121499896049499,
655
+ "reward_std": 0.16680648624897004,
656
+ "rewards/reward_fn/mean": 0.612150001525879,
657
+ "rewards/reward_fn/std": 0.18321824073791504,
658
+ "step": 120
659
+ },
660
+ {
661
+ "clip_ratio/high_max": 0.0,
662
+ "clip_ratio/high_mean": 0.0,
663
+ "clip_ratio/low_mean": 0.0,
664
+ "clip_ratio/low_min": 0.0,
665
+ "clip_ratio/region_mean": 0.0,
666
+ "completion_length": 27.3,
667
+ "completions/clipped_ratio": 0.025,
668
+ "completions/max_length": 42.6,
669
+ "completions/max_terminated_length": 29.8,
670
+ "completions/mean_length": 27.3,
671
+ "completions/mean_terminated_length": 25.514286041259766,
672
+ "completions/min_length": 21.4,
673
+ "completions/min_terminated_length": 21.4,
674
+ "epoch": 0.5555555555555556,
675
+ "frac_reward_zero_std": 0.0,
676
+ "grad_norm": 0.922471821308136,
677
+ "kl": 0.04857689954806119,
678
+ "learning_rate": 2.370892018779343e-06,
679
+ "loss": 0.0019433587789535523,
680
+ "num_tokens": 508231.0,
681
+ "reward": 0.5884499907493591,
682
+ "reward_std": 0.17119054943323136,
683
+ "rewards/reward_fn/mean": 0.588450014591217,
684
+ "rewards/reward_fn/std": 0.18389662504196166,
685
+ "step": 125
686
+ },
687
+ {
688
+ "clip_ratio/high_max": 0.0,
689
+ "clip_ratio/high_mean": 0.0,
690
+ "clip_ratio/low_mean": 0.0,
691
+ "clip_ratio/low_min": 0.0,
692
+ "clip_ratio/region_mean": 0.0,
693
+ "completion_length": 26.05,
694
+ "completions/clipped_ratio": 0.0,
695
+ "completions/max_length": 31.0,
696
+ "completions/max_terminated_length": 31.0,
697
+ "completions/mean_length": 26.05,
698
+ "completions/mean_terminated_length": 26.05,
699
+ "completions/min_length": 22.0,
700
+ "completions/min_terminated_length": 22.0,
701
+ "epoch": 0.5777777777777777,
702
+ "frac_reward_zero_std": 0.0,
703
+ "grad_norm": 0.7222557067871094,
704
+ "kl": 0.06997429557377473,
705
+ "learning_rate": 2.2535211267605635e-06,
706
+ "loss": 0.0027989834547042848,
707
+ "num_tokens": 528729.0,
708
+ "reward": 0.5927999973297119,
709
+ "reward_std": 0.22429427206516267,
710
+ "rewards/reward_fn/mean": 0.5928000092506409,
711
+ "rewards/reward_fn/std": 0.21314262747764587,
712
+ "step": 130
713
+ },
714
+ {
715
+ "clip_ratio/high_max": 0.0,
716
+ "clip_ratio/high_mean": 0.0,
717
+ "clip_ratio/low_mean": 0.0,
718
+ "clip_ratio/low_min": 0.0,
719
+ "clip_ratio/region_mean": 0.0,
720
+ "completion_length": 25.925,
721
+ "completions/clipped_ratio": 0.0,
722
+ "completions/max_length": 35.6,
723
+ "completions/max_terminated_length": 35.6,
724
+ "completions/mean_length": 25.925,
725
+ "completions/mean_terminated_length": 25.925,
726
+ "completions/min_length": 18.8,
727
+ "completions/min_terminated_length": 18.8,
728
+ "epoch": 0.6,
729
+ "frac_reward_zero_std": 0.0,
730
+ "grad_norm": 1.2586678266525269,
731
+ "kl": 0.08584307442652062,
732
+ "learning_rate": 2.136150234741784e-06,
733
+ "loss": 0.0034336388111114503,
734
+ "num_tokens": 548748.0,
735
+ "reward": 0.5738499760627747,
736
+ "reward_std": 0.17571603059768676,
737
+ "rewards/reward_fn/mean": 0.5738499999046326,
738
+ "rewards/reward_fn/std": 0.19411510229110718,
739
+ "step": 135
740
+ },
741
+ {
742
+ "clip_ratio/high_max": 0.0,
743
+ "clip_ratio/high_mean": 0.0,
744
+ "clip_ratio/low_mean": 0.0,
745
+ "clip_ratio/low_min": 0.0,
746
+ "clip_ratio/region_mean": 0.0,
747
+ "completion_length": 25.9,
748
+ "completions/clipped_ratio": 0.0,
749
+ "completions/max_length": 29.4,
750
+ "completions/max_terminated_length": 29.4,
751
+ "completions/mean_length": 25.9,
752
+ "completions/mean_terminated_length": 25.9,
753
+ "completions/min_length": 23.4,
754
+ "completions/min_terminated_length": 23.4,
755
+ "epoch": 0.6222222222222222,
756
+ "frac_reward_zero_std": 0.0,
757
+ "grad_norm": 1.0420219898223877,
758
+ "kl": 0.06210445412434638,
759
+ "learning_rate": 2.0187793427230047e-06,
760
+ "loss": 0.0024841248989105223,
761
+ "num_tokens": 569094.0,
762
+ "reward": 0.6095999956130982,
763
+ "reward_std": 0.2001112163066864,
764
+ "rewards/reward_fn/mean": 0.6095999956130982,
765
+ "rewards/reward_fn/std": 0.18984024077653885,
766
+ "step": 140
767
+ },
768
+ {
769
+ "clip_ratio/high_max": 0.0,
770
+ "clip_ratio/high_mean": 0.0,
771
+ "clip_ratio/low_mean": 0.0,
772
+ "clip_ratio/low_min": 0.0,
773
+ "clip_ratio/region_mean": 0.0,
774
+ "completion_length": 27.85,
775
+ "completions/clipped_ratio": 0.025,
776
+ "completions/max_length": 46.0,
777
+ "completions/max_terminated_length": 34.8,
778
+ "completions/mean_length": 27.85,
779
+ "completions/mean_terminated_length": 26.164286041259764,
780
+ "completions/min_length": 21.2,
781
+ "completions/min_terminated_length": 21.2,
782
+ "epoch": 0.6444444444444445,
783
+ "frac_reward_zero_std": 0.0,
784
+ "grad_norm": 0.9585386514663696,
785
+ "kl": 0.04591481959214434,
786
+ "learning_rate": 1.9014084507042254e-06,
787
+ "loss": 0.0018365621566772462,
788
+ "num_tokens": 589932.0,
789
+ "reward": 0.5361499905586242,
790
+ "reward_std": 0.20909147560596467,
791
+ "rewards/reward_fn/mean": 0.5361500024795532,
792
+ "rewards/reward_fn/std": 0.1902428910136223,
793
+ "step": 145
794
+ },
795
+ {
796
+ "clip_ratio/high_max": 0.0,
797
+ "clip_ratio/high_mean": 0.0,
798
+ "clip_ratio/low_mean": 0.0,
799
+ "clip_ratio/low_min": 0.0,
800
+ "clip_ratio/region_mean": 0.0,
801
+ "completion_length": 25.25,
802
+ "completions/clipped_ratio": 0.0,
803
+ "completions/max_length": 29.6,
804
+ "completions/max_terminated_length": 29.6,
805
+ "completions/mean_length": 25.25,
806
+ "completions/mean_terminated_length": 25.25,
807
+ "completions/min_length": 21.6,
808
+ "completions/min_terminated_length": 21.6,
809
+ "epoch": 0.6666666666666666,
810
+ "frac_reward_zero_std": 0.0,
811
+ "grad_norm": 0.45393237471580505,
812
+ "kl": 0.04076897802297026,
813
+ "learning_rate": 1.784037558685446e-06,
814
+ "loss": 0.001630684733390808,
815
+ "num_tokens": 610754.0,
816
+ "reward": 0.5666499972343445,
817
+ "reward_std": 0.17826161682605743,
818
+ "rewards/reward_fn/mean": 0.5666500210762024,
819
+ "rewards/reward_fn/std": 0.1894826263189316,
820
+ "step": 150
821
+ },
822
+ {
823
+ "clip_ratio/high_max": 0.0,
824
+ "clip_ratio/high_mean": 0.0,
825
+ "clip_ratio/low_mean": 0.0,
826
+ "clip_ratio/low_min": 0.0,
827
+ "clip_ratio/region_mean": 0.0,
828
+ "completion_length": 24.85,
829
+ "completions/clipped_ratio": 0.0,
830
+ "completions/max_length": 29.2,
831
+ "completions/max_terminated_length": 29.2,
832
+ "completions/mean_length": 24.85,
833
+ "completions/mean_terminated_length": 24.85,
834
+ "completions/min_length": 21.4,
835
+ "completions/min_terminated_length": 21.4,
836
+ "epoch": 0.6888888888888889,
837
+ "frac_reward_zero_std": 0.0,
838
+ "grad_norm": 1.101528525352478,
839
+ "kl": 0.0628763473010622,
840
+ "learning_rate": 1.6666666666666667e-06,
841
+ "loss": 0.0025150284171104433,
842
+ "num_tokens": 632434.0,
843
+ "reward": 0.5634499907493591,
844
+ "reward_std": 0.18957532048225403,
845
+ "rewards/reward_fn/mean": 0.5634500086307526,
846
+ "rewards/reward_fn/std": 0.1851608410477638,
847
+ "step": 155
848
+ },
849
+ {
850
+ "clip_ratio/high_max": 0.0,
851
+ "clip_ratio/high_mean": 0.0,
852
+ "clip_ratio/low_mean": 0.0,
853
+ "clip_ratio/low_min": 0.0,
854
+ "clip_ratio/region_mean": 0.0,
855
+ "completion_length": 25.6,
856
+ "completions/clipped_ratio": 0.0,
857
+ "completions/max_length": 30.6,
858
+ "completions/max_terminated_length": 30.6,
859
+ "completions/mean_length": 25.6,
860
+ "completions/mean_terminated_length": 25.6,
861
+ "completions/min_length": 22.4,
862
+ "completions/min_terminated_length": 22.4,
863
+ "epoch": 0.7111111111111111,
864
+ "frac_reward_zero_std": 0.0,
865
+ "grad_norm": 0.593997061252594,
866
+ "kl": 0.08342576812719926,
867
+ "learning_rate": 1.5492957746478873e-06,
868
+ "loss": 0.0033370301127433775,
869
+ "num_tokens": 653686.0,
870
+ "reward": 0.5921499967575073,
871
+ "reward_std": 0.20258608162403108,
872
+ "rewards/reward_fn/mean": 0.5921500205993653,
873
+ "rewards/reward_fn/std": 0.20735768973827362,
874
+ "step": 160
875
+ },
876
+ {
877
+ "clip_ratio/high_max": 0.0,
878
+ "clip_ratio/high_mean": 0.0,
879
+ "clip_ratio/low_mean": 0.0,
880
+ "clip_ratio/low_min": 0.0,
881
+ "clip_ratio/region_mean": 0.0,
882
+ "completion_length": 25.175,
883
+ "completions/clipped_ratio": 0.0,
884
+ "completions/max_length": 30.4,
885
+ "completions/max_terminated_length": 30.4,
886
+ "completions/mean_length": 25.175,
887
+ "completions/mean_terminated_length": 25.175,
888
+ "completions/min_length": 20.4,
889
+ "completions/min_terminated_length": 20.4,
890
+ "epoch": 0.7333333333333333,
891
+ "frac_reward_zero_std": 0.0,
892
+ "grad_norm": 0.8682815432548523,
893
+ "kl": 0.04647499453276396,
894
+ "learning_rate": 1.4319248826291082e-06,
895
+ "loss": 0.0018589019775390625,
896
+ "num_tokens": 673657.0,
897
+ "reward": 0.5476499795913696,
898
+ "reward_std": 0.18038293421268464,
899
+ "rewards/reward_fn/mean": 0.5476500034332276,
900
+ "rewards/reward_fn/std": 0.20103364586830139,
901
+ "step": 165
902
+ },
903
+ {
904
+ "clip_ratio/high_max": 0.0,
905
+ "clip_ratio/high_mean": 0.0,
906
+ "clip_ratio/low_mean": 0.0,
907
+ "clip_ratio/low_min": 0.0,
908
+ "clip_ratio/region_mean": 0.0,
909
+ "completion_length": 24.95,
910
+ "completions/clipped_ratio": 0.0,
911
+ "completions/max_length": 28.0,
912
+ "completions/max_terminated_length": 28.0,
913
+ "completions/mean_length": 24.95,
914
+ "completions/mean_terminated_length": 24.95,
915
+ "completions/min_length": 23.2,
916
+ "completions/min_terminated_length": 23.2,
917
+ "epoch": 0.7555555555555555,
918
+ "frac_reward_zero_std": 0.0,
919
+ "grad_norm": 0.6287415027618408,
920
+ "kl": 0.0606904512271285,
921
+ "learning_rate": 1.3145539906103288e-06,
922
+ "loss": 0.0024275988340377807,
923
+ "num_tokens": 693125.0,
924
+ "reward": 0.6188999772071838,
925
+ "reward_std": 0.20350532233715057,
926
+ "rewards/reward_fn/mean": 0.6188999891281128,
927
+ "rewards/reward_fn/std": 0.2084890365600586,
928
+ "step": 170
929
+ },
930
+ {
931
+ "clip_ratio/high_max": 0.0,
932
+ "clip_ratio/high_mean": 0.0,
933
+ "clip_ratio/low_mean": 0.0,
934
+ "clip_ratio/low_min": 0.0,
935
+ "clip_ratio/region_mean": 0.0,
936
+ "completion_length": 25.175,
937
+ "completions/clipped_ratio": 0.0,
938
+ "completions/max_length": 28.8,
939
+ "completions/max_terminated_length": 28.8,
940
+ "completions/mean_length": 25.175,
941
+ "completions/mean_terminated_length": 25.175,
942
+ "completions/min_length": 21.4,
943
+ "completions/min_terminated_length": 21.4,
944
+ "epoch": 0.7777777777777778,
945
+ "frac_reward_zero_std": 0.0,
946
+ "grad_norm": 1.0185426473617554,
947
+ "kl": 0.044300994148943576,
948
+ "learning_rate": 1.1971830985915492e-06,
949
+ "loss": 0.001772068440914154,
950
+ "num_tokens": 713356.0,
951
+ "reward": 0.49915000796318054,
952
+ "reward_std": 0.17090770602226257,
953
+ "rewards/reward_fn/mean": 0.4991500020027161,
954
+ "rewards/reward_fn/std": 0.15009358823299407,
955
+ "step": 175
956
+ },
957
+ {
958
+ "clip_ratio/high_max": 0.0,
959
+ "clip_ratio/high_mean": 0.0,
960
+ "clip_ratio/low_mean": 0.0,
961
+ "clip_ratio/low_min": 0.0,
962
+ "clip_ratio/region_mean": 0.0,
963
+ "completion_length": 25.825,
964
+ "completions/clipped_ratio": 0.0,
965
+ "completions/max_length": 30.0,
966
+ "completions/max_terminated_length": 30.0,
967
+ "completions/mean_length": 25.825,
968
+ "completions/mean_terminated_length": 25.825,
969
+ "completions/min_length": 22.2,
970
+ "completions/min_terminated_length": 22.2,
971
+ "epoch": 0.8,
972
+ "frac_reward_zero_std": 0.0,
973
+ "grad_norm": 0.8517215251922607,
974
+ "kl": 0.07110593506367877,
975
+ "learning_rate": 1.07981220657277e-06,
976
+ "loss": 0.0028442263603210447,
977
+ "num_tokens": 733613.0,
978
+ "reward": 0.5364999890327453,
979
+ "reward_std": 0.17903943061828614,
980
+ "rewards/reward_fn/mean": 0.5364999890327453,
981
+ "rewards/reward_fn/std": 0.1849100574851036,
982
+ "step": 180
983
+ },
984
+ {
985
+ "clip_ratio/high_max": 0.0,
986
+ "clip_ratio/high_mean": 0.0,
987
+ "clip_ratio/low_mean": 0.0,
988
+ "clip_ratio/low_min": 0.0,
989
+ "clip_ratio/region_mean": 0.0,
990
+ "completion_length": 25.725,
991
+ "completions/clipped_ratio": 0.0,
992
+ "completions/max_length": 33.2,
993
+ "completions/max_terminated_length": 33.2,
994
+ "completions/mean_length": 25.725,
995
+ "completions/mean_terminated_length": 25.725,
996
+ "completions/min_length": 21.8,
997
+ "completions/min_terminated_length": 21.8,
998
+ "epoch": 0.8222222222222222,
999
+ "frac_reward_zero_std": 0.0,
1000
+ "grad_norm": 1.2276585102081299,
1001
+ "kl": 0.06446516590658576,
1002
+ "learning_rate": 9.624413145539907e-07,
1003
+ "loss": 0.002578553557395935,
1004
+ "num_tokens": 753214.0,
1005
+ "reward": 0.5435999751091003,
1006
+ "reward_std": 0.1755038946866989,
1007
+ "rewards/reward_fn/mean": 0.5436000108718873,
1008
+ "rewards/reward_fn/std": 0.1861019790172577,
1009
+ "step": 185
1010
+ },
1011
+ {
1012
+ "clip_ratio/high_max": 0.0,
1013
+ "clip_ratio/high_mean": 0.0,
1014
+ "clip_ratio/low_mean": 0.0,
1015
+ "clip_ratio/low_min": 0.0,
1016
+ "clip_ratio/region_mean": 0.0,
1017
+ "completion_length": 24.6,
1018
+ "completions/clipped_ratio": 0.0,
1019
+ "completions/max_length": 27.4,
1020
+ "completions/max_terminated_length": 27.4,
1021
+ "completions/mean_length": 24.6,
1022
+ "completions/mean_terminated_length": 24.6,
1023
+ "completions/min_length": 20.8,
1024
+ "completions/min_terminated_length": 20.8,
1025
+ "epoch": 0.8444444444444444,
1026
+ "frac_reward_zero_std": 0.0,
1027
+ "grad_norm": 0.9992023706436157,
1028
+ "kl": 0.05620210377383046,
1029
+ "learning_rate": 8.450704225352114e-07,
1030
+ "loss": 0.0022480204701423646,
1031
+ "num_tokens": 774816.0,
1032
+ "reward": 0.5924999952316284,
1033
+ "reward_std": 0.1981313169002533,
1034
+ "rewards/reward_fn/mean": 0.5924999952316284,
1035
+ "rewards/reward_fn/std": 0.19494172781705857,
1036
+ "step": 190
1037
+ },
1038
+ {
1039
+ "clip_ratio/high_max": 0.0,
1040
+ "clip_ratio/high_mean": 0.0,
1041
+ "clip_ratio/low_mean": 0.0,
1042
+ "clip_ratio/low_min": 0.0,
1043
+ "clip_ratio/region_mean": 0.0,
1044
+ "completion_length": 25.925,
1045
+ "completions/clipped_ratio": 0.0,
1046
+ "completions/max_length": 32.8,
1047
+ "completions/max_terminated_length": 32.8,
1048
+ "completions/mean_length": 25.925,
1049
+ "completions/mean_terminated_length": 25.925,
1050
+ "completions/min_length": 21.6,
1051
+ "completions/min_terminated_length": 21.6,
1052
+ "epoch": 0.8666666666666667,
1053
+ "frac_reward_zero_std": 0.0,
1054
+ "grad_norm": 0.6905612349510193,
1055
+ "kl": 0.061457820073701444,
1056
+ "learning_rate": 7.27699530516432e-07,
1057
+ "loss": 0.0024582624435424806,
1058
+ "num_tokens": 794225.0,
1059
+ "reward": 0.5802499890327454,
1060
+ "reward_std": 0.17005917578935623,
1061
+ "rewards/reward_fn/mean": 0.5802499890327454,
1062
+ "rewards/reward_fn/std": 0.18120778799057008,
1063
+ "step": 195
1064
+ },
1065
+ {
1066
+ "clip_ratio/high_max": 0.0,
1067
+ "clip_ratio/high_mean": 0.0,
1068
+ "clip_ratio/low_mean": 0.0,
1069
+ "clip_ratio/low_min": 0.0,
1070
+ "clip_ratio/region_mean": 0.0,
1071
+ "completion_length": 25.975,
1072
+ "completions/clipped_ratio": 0.0,
1073
+ "completions/max_length": 31.6,
1074
+ "completions/max_terminated_length": 31.6,
1075
+ "completions/mean_length": 25.975,
1076
+ "completions/mean_terminated_length": 25.975,
1077
+ "completions/min_length": 23.6,
1078
+ "completions/min_terminated_length": 23.6,
1079
+ "epoch": 0.8888888888888888,
1080
+ "frac_reward_zero_std": 0.0,
1081
+ "grad_norm": 0.7173504829406738,
1082
+ "kl": 0.06312856795266271,
1083
+ "learning_rate": 6.103286384976526e-07,
1084
+ "loss": 0.0025250956416130064,
1085
+ "num_tokens": 814240.0,
1086
+ "reward": 0.5612499833106994,
1087
+ "reward_std": 0.20499025285243988,
1088
+ "rewards/reward_fn/mean": 0.561249989271164,
1089
+ "rewards/reward_fn/std": 0.2024638831615448,
1090
+ "step": 200
1091
+ },
1092
+ {
1093
+ "clip_ratio/high_max": 0.0,
1094
+ "clip_ratio/high_mean": 0.0,
1095
+ "clip_ratio/low_mean": 0.0,
1096
+ "clip_ratio/low_min": 0.0,
1097
+ "clip_ratio/region_mean": 0.0,
1098
+ "completion_length": 24.75,
1099
+ "completions/clipped_ratio": 0.0,
1100
+ "completions/max_length": 28.8,
1101
+ "completions/max_terminated_length": 28.8,
1102
+ "completions/mean_length": 24.75,
1103
+ "completions/mean_terminated_length": 24.75,
1104
+ "completions/min_length": 20.8,
1105
+ "completions/min_terminated_length": 20.8,
1106
+ "epoch": 0.9111111111111111,
1107
+ "frac_reward_zero_std": 0.0,
1108
+ "grad_norm": 0.7710515260696411,
1109
+ "kl": 0.07052509421482682,
1110
+ "learning_rate": 4.929577464788733e-07,
1111
+ "loss": 0.0028209388256073,
1112
+ "num_tokens": 834334.0,
1113
+ "reward": 0.5948500037193298,
1114
+ "reward_std": 0.17953440248966218,
1115
+ "rewards/reward_fn/mean": 0.5948499917984009,
1116
+ "rewards/reward_fn/std": 0.19131502211093904,
1117
+ "step": 205
1118
+ },
1119
+ {
1120
+ "clip_ratio/high_max": 0.0,
1121
+ "clip_ratio/high_mean": 0.0,
1122
+ "clip_ratio/low_mean": 0.0,
1123
+ "clip_ratio/low_min": 0.0,
1124
+ "clip_ratio/region_mean": 0.0,
1125
+ "completion_length": 25.8,
1126
+ "completions/clipped_ratio": 0.0,
1127
+ "completions/max_length": 30.6,
1128
+ "completions/max_terminated_length": 30.6,
1129
+ "completions/mean_length": 25.8,
1130
+ "completions/mean_terminated_length": 25.8,
1131
+ "completions/min_length": 23.4,
1132
+ "completions/min_terminated_length": 23.4,
1133
+ "epoch": 0.9333333333333333,
1134
+ "frac_reward_zero_std": 0.0,
1135
+ "grad_norm": 0.8554002046585083,
1136
+ "kl": 0.05014841896481812,
1137
+ "learning_rate": 3.755868544600939e-07,
1138
+ "loss": 0.0020059287548065186,
1139
+ "num_tokens": 854900.0,
1140
+ "reward": 0.5546000003814697,
1141
+ "reward_std": 0.21057639718055726,
1142
+ "rewards/reward_fn/mean": 0.5545999944210053,
1143
+ "rewards/reward_fn/std": 0.20256412923336028,
1144
+ "step": 210
1145
+ },
1146
+ {
1147
+ "clip_ratio/high_max": 0.0,
1148
+ "clip_ratio/high_mean": 0.0,
1149
+ "clip_ratio/low_mean": 0.0,
1150
+ "clip_ratio/low_min": 0.0,
1151
+ "clip_ratio/region_mean": 0.0,
1152
+ "completion_length": 25.475,
1153
+ "completions/clipped_ratio": 0.0,
1154
+ "completions/max_length": 29.4,
1155
+ "completions/max_terminated_length": 29.4,
1156
+ "completions/mean_length": 25.475,
1157
+ "completions/mean_terminated_length": 25.475,
1158
+ "completions/min_length": 23.2,
1159
+ "completions/min_terminated_length": 23.2,
1160
+ "epoch": 0.9555555555555556,
1161
+ "frac_reward_zero_std": 0.0,
1162
+ "grad_norm": 0.8410727381706238,
1163
+ "kl": 0.06370279549155385,
1164
+ "learning_rate": 2.582159624413146e-07,
1165
+ "loss": 0.0025480970740318297,
1166
+ "num_tokens": 874953.0,
1167
+ "reward": 0.5734000027179718,
1168
+ "reward_std": 0.2160918265581131,
1169
+ "rewards/reward_fn/mean": 0.5733999907970428,
1170
+ "rewards/reward_fn/std": 0.20869273394346238,
1171
+ "step": 215
1172
+ },
1173
+ {
1174
+ "clip_ratio/high_max": 0.0,
1175
+ "clip_ratio/high_mean": 0.0,
1176
+ "clip_ratio/low_mean": 0.0,
1177
+ "clip_ratio/low_min": 0.0,
1178
+ "clip_ratio/region_mean": 0.0,
1179
+ "completion_length": 25.9,
1180
+ "completions/clipped_ratio": 0.0,
1181
+ "completions/max_length": 34.8,
1182
+ "completions/max_terminated_length": 34.8,
1183
+ "completions/mean_length": 25.9,
1184
+ "completions/mean_terminated_length": 25.9,
1185
+ "completions/min_length": 20.8,
1186
+ "completions/min_terminated_length": 20.8,
1187
+ "epoch": 0.9777777777777777,
1188
+ "frac_reward_zero_std": 0.0,
1189
+ "grad_norm": 0.9725803136825562,
1190
+ "kl": 0.075754539296031,
1191
+ "learning_rate": 1.4084507042253522e-07,
1192
+ "loss": 0.0030301779508590697,
1193
+ "num_tokens": 894699.0,
1194
+ "reward": 0.5753999829292298,
1195
+ "reward_std": 0.20675801634788513,
1196
+ "rewards/reward_fn/mean": 0.5753999948501587,
1197
+ "rewards/reward_fn/std": 0.2071171909570694,
1198
+ "step": 220
1199
+ },
1200
+ {
1201
+ "clip_ratio/high_max": 0.0,
1202
+ "clip_ratio/high_mean": 0.0,
1203
+ "clip_ratio/low_mean": 0.0,
1204
+ "clip_ratio/low_min": 0.0,
1205
+ "clip_ratio/region_mean": 0.0,
1206
+ "completion_length": 25.4,
1207
+ "completions/clipped_ratio": 0.0,
1208
+ "completions/max_length": 30.2,
1209
+ "completions/max_terminated_length": 30.2,
1210
+ "completions/mean_length": 25.4,
1211
+ "completions/mean_terminated_length": 25.4,
1212
+ "completions/min_length": 21.8,
1213
+ "completions/min_terminated_length": 21.8,
1214
+ "epoch": 1.0,
1215
+ "frac_reward_zero_std": 0.0,
1216
+ "grad_norm": 0.6214760541915894,
1217
+ "kl": 0.0598387235251721,
1218
+ "learning_rate": 2.347417840375587e-08,
1219
+ "loss": 0.0023935258388519285,
1220
+ "num_tokens": 914809.0,
1221
+ "reward": 0.599399995803833,
1222
+ "reward_std": 0.20831365883350372,
1223
+ "rewards/reward_fn/mean": 0.599399995803833,
1224
+ "rewards/reward_fn/std": 0.20224641859531403,
1225
+ "step": 225
1226
+ }
1227
+ ],
1228
+ "logging_steps": 5,
1229
+ "max_steps": 225,
1230
+ "num_input_tokens_seen": 914809,
1231
+ "num_train_epochs": 1,
1232
+ "save_steps": 50,
1233
+ "stateful_callbacks": {
1234
+ "TrainerControl": {
1235
+ "args": {
1236
+ "should_epoch_stop": false,
1237
+ "should_evaluate": false,
1238
+ "should_log": false,
1239
+ "should_save": true,
1240
+ "should_training_stop": true
1241
+ },
1242
+ "attributes": {}
1243
+ }
1244
+ },
1245
+ "total_flos": 0.0,
1246
+ "train_batch_size": 1,
1247
+ "trial_name": null,
1248
+ "trial_params": null
1249
+ }
results/phase1/checkpoint-225/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b53420ca071212a25df4a51528d52e34338221d4319e6e32ff795209b21413d
3
+ size 6673
results/phase1/checkpoint-50/README.md CHANGED
@@ -6,6 +6,7 @@ tags:
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
 
9
  - transformers
10
  - trl
11
  - unsloth
 
6
  - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
  - grpo
8
  - lora
9
+ - sft
10
  - transformers
11
  - trl
12
  - unsloth
results/phase1/checkpoint-50/adapter_config.json CHANGED
@@ -33,13 +33,13 @@
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
 
 
36
  "k_proj",
37
- "up_proj",
38
  "down_proj",
39
  "o_proj",
40
- "v_proj",
41
- "gate_proj",
42
- "q_proj"
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
 
33
  "rank_pattern": {},
34
  "revision": null,
35
  "target_modules": [
36
+ "gate_proj",
37
+ "q_proj",
38
  "k_proj",
39
+ "v_proj",
40
  "down_proj",
41
  "o_proj",
42
+ "up_proj"
 
 
43
  ],
44
  "target_parameters": null,
45
  "task_type": "CAUSAL_LM",
results/phase1/checkpoint-50/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0adbf9d63bca21b84d7699d12cc495ff6d76158cfc43146af81ec0b5b93d22e8
3
  size 73911112
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:810a279575518f5aa4d0cab9e75ceddaa94cf06e33533008d364a47d79267e69
3
  size 73911112
results/phase1/checkpoint-50/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ed241d586f6e676b800ea6c9ca0946e4b50e6c56b18b8f340ed3c8b51991a66d
3
  size 37969669
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9bbcbd4511a82c9c6bdb43bad61356c0af918216f63f91e10fa7ed7e0488ef08
3
  size 37969669
results/phase1/checkpoint-50/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:cd7a6e62201dd7f462f420cdacde3f7d6295b8e73708203688b8af13c5d4f5c2
3
  size 14645
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06dea39c5c2b8f2bd1c6a68214429e67c2ef23cf860ac79996068119077e351f
3
  size 14645
results/phase1/checkpoint-50/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:823af046a154dcc15b21be64915c4a2d28a7165ca91fe16175e2f7897dc126b8
3
  size 1465
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:123e450a8ef68e6d25122f141b1c70f4e336373980eca07d10a278f228dc2c80
3
  size 1465
results/phase1/checkpoint-50/trainer_state.json CHANGED
@@ -2,7 +2,7 @@
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
- "epoch": 0.22522522522522523,
6
  "eval_steps": 500,
7
  "global_step": 50,
8
  "is_hyper_param_search": false,
@@ -15,25 +15,25 @@
15
  "clip_ratio/low_mean": 0.0,
16
  "clip_ratio/low_min": 0.0,
17
  "clip_ratio/region_mean": 0.0,
18
- "completion_length": 29.05,
19
- "completions/clipped_ratio": 0.05,
20
- "completions/max_length": 44.2,
21
- "completions/max_terminated_length": 31.0,
22
- "completions/mean_length": 29.05,
23
- "completions/mean_terminated_length": 25.583333587646486,
24
- "completions/min_length": 21.4,
25
- "completions/min_terminated_length": 21.4,
26
- "epoch": 0.02252252252252252,
27
- "frac_reward_zero_std": 0.0,
28
- "grad_norm": 1.2577366828918457,
29
- "kl": 7.414049605358741e-06,
30
  "learning_rate": 1.6666666666666667e-06,
31
- "loss": 2.8312206268310547e-07,
32
- "num_tokens": 21378.0,
33
- "reward": 0.4944999933242798,
34
- "reward_std": 0.15089658200740813,
35
- "rewards/reward_fn/mean": 0.49449999928474425,
36
- "rewards/reward_fn/std": 0.12999328523874282,
37
  "step": 5
38
  },
39
  {
@@ -42,25 +42,25 @@
42
  "clip_ratio/low_mean": 0.0,
43
  "clip_ratio/low_min": 0.0,
44
  "clip_ratio/region_mean": 0.0,
45
- "completion_length": 29.175,
46
  "completions/clipped_ratio": 0.0,
47
- "completions/max_length": 41.8,
48
- "completions/max_terminated_length": 41.8,
49
- "completions/mean_length": 29.175,
50
- "completions/mean_terminated_length": 29.175,
51
- "completions/min_length": 19.2,
52
- "completions/min_terminated_length": 19.2,
53
- "epoch": 0.04504504504504504,
54
  "frac_reward_zero_std": 0.0,
55
- "grad_norm": 1.148110032081604,
56
- "kl": 2.58529256058182e-05,
57
  "learning_rate": 3.7500000000000005e-06,
58
- "loss": 9.343028068542481e-07,
59
- "num_tokens": 42709.0,
60
- "reward": 0.48589999675750734,
61
- "reward_std": 0.14184561967849732,
62
- "rewards/reward_fn/mean": 0.4859000027179718,
63
- "rewards/reward_fn/std": 0.12539554834365846,
64
  "step": 10
65
  },
66
  {
@@ -69,25 +69,25 @@
69
  "clip_ratio/low_mean": 0.0,
70
  "clip_ratio/low_min": 0.0,
71
  "clip_ratio/region_mean": 0.0,
72
- "completion_length": 28.825,
73
  "completions/clipped_ratio": 0.0,
74
- "completions/max_length": 44.2,
75
- "completions/max_terminated_length": 44.2,
76
- "completions/mean_length": 28.825,
77
- "completions/mean_terminated_length": 28.825,
78
- "completions/min_length": 21.2,
79
- "completions/min_terminated_length": 21.2,
80
- "epoch": 0.06756756756756757,
81
  "frac_reward_zero_std": 0.0,
82
- "grad_norm": 1.1520825624465942,
83
- "kl": 0.0001522944080875277,
84
- "learning_rate": 4.952380952380953e-06,
85
- "loss": 6.105005741119385e-06,
86
- "num_tokens": 63008.0,
87
- "reward": 0.4894999861717224,
88
- "reward_std": 0.15768481492996217,
89
- "rewards/reward_fn/mean": 0.48950000405311583,
90
- "rewards/reward_fn/std": 0.14185291826725005,
91
  "step": 15
92
  },
93
  {
@@ -96,25 +96,25 @@
96
  "clip_ratio/low_mean": 0.0,
97
  "clip_ratio/low_min": 0.0,
98
  "clip_ratio/region_mean": 0.0,
99
- "completion_length": 28.25,
100
  "completions/clipped_ratio": 0.0,
101
- "completions/max_length": 43.2,
102
- "completions/max_terminated_length": 43.2,
103
- "completions/mean_length": 28.25,
104
- "completions/mean_terminated_length": 28.25,
105
- "completions/min_length": 21.6,
106
- "completions/min_terminated_length": 21.6,
107
- "epoch": 0.09009009009009009,
108
  "frac_reward_zero_std": 0.0,
109
- "grad_norm": 1.0063226222991943,
110
- "kl": 0.0008983879492006963,
111
- "learning_rate": 4.833333333333333e-06,
112
- "loss": 3.593862056732178e-05,
113
- "num_tokens": 82836.0,
114
- "reward": 0.4881500005722046,
115
- "reward_std": 0.14941166192293168,
116
- "rewards/reward_fn/mean": 0.48814999461174013,
117
- "rewards/reward_fn/std": 0.12791907638311387,
118
  "step": 20
119
  },
120
  {
@@ -123,25 +123,25 @@
123
  "clip_ratio/low_mean": 0.0,
124
  "clip_ratio/low_min": 0.0,
125
  "clip_ratio/region_mean": 0.0,
126
- "completion_length": 27.25,
127
  "completions/clipped_ratio": 0.0,
128
- "completions/max_length": 45.6,
129
- "completions/max_terminated_length": 45.6,
130
- "completions/mean_length": 27.25,
131
- "completions/mean_terminated_length": 27.25,
132
- "completions/min_length": 18.8,
133
- "completions/min_terminated_length": 18.8,
134
- "epoch": 0.11261261261261261,
135
  "frac_reward_zero_std": 0.0,
136
- "grad_norm": 1.3132057189941406,
137
- "kl": 0.003781939600594342,
138
- "learning_rate": 4.714285714285715e-06,
139
- "loss": 0.00015127062797546387,
140
- "num_tokens": 103596.0,
141
- "reward": 0.4680500030517578,
142
- "reward_std": 0.17189764976501465,
143
- "rewards/reward_fn/mean": 0.46804999113082885,
144
- "rewards/reward_fn/std": 0.17421672195196153,
145
  "step": 25
146
  },
147
  {
@@ -150,25 +150,25 @@
150
  "clip_ratio/low_mean": 0.0,
151
  "clip_ratio/low_min": 0.0,
152
  "clip_ratio/region_mean": 0.0,
153
- "completion_length": 25.225,
154
  "completions/clipped_ratio": 0.0,
155
- "completions/max_length": 34.6,
156
- "completions/max_terminated_length": 34.6,
157
- "completions/mean_length": 25.225,
158
- "completions/mean_terminated_length": 25.225,
159
- "completions/min_length": 19.2,
160
- "completions/min_terminated_length": 19.2,
161
- "epoch": 0.13513513513513514,
162
  "frac_reward_zero_std": 0.0,
163
- "grad_norm": 1.5750545263290405,
164
- "kl": 0.006673775642411783,
165
- "learning_rate": 4.595238095238095e-06,
166
- "loss": 0.0002669498324394226,
167
- "num_tokens": 123651.0,
168
- "reward": 0.48049998879432676,
169
- "reward_std": 0.1656044065952301,
170
- "rewards/reward_fn/mean": 0.48049998879432676,
171
- "rewards/reward_fn/std": 0.13908967524766921,
172
  "step": 30
173
  },
174
  {
@@ -177,25 +177,25 @@
177
  "clip_ratio/low_mean": 0.0,
178
  "clip_ratio/low_min": 0.0,
179
  "clip_ratio/region_mean": 0.0,
180
- "completion_length": 28.775,
181
- "completions/clipped_ratio": 0.025,
182
- "completions/max_length": 54.8,
183
- "completions/max_terminated_length": 41.8,
184
- "completions/mean_length": 28.775,
185
- "completions/mean_terminated_length": 26.967857360839844,
186
- "completions/min_length": 15.4,
187
- "completions/min_terminated_length": 15.4,
188
- "epoch": 0.15765765765765766,
189
  "frac_reward_zero_std": 0.0,
190
- "grad_norm": 1.1105402708053589,
191
- "kl": 0.005106111426721327,
192
- "learning_rate": 4.476190476190477e-06,
193
- "loss": 0.0002042025327682495,
194
- "num_tokens": 144316.0,
195
- "reward": 0.4778999924659729,
196
- "reward_std": 0.17012988924980163,
197
- "rewards/reward_fn/mean": 0.47790001034736634,
198
- "rewards/reward_fn/std": 0.14147266000509262,
199
  "step": 35
200
  },
201
  {
@@ -204,25 +204,25 @@
204
  "clip_ratio/low_mean": 0.0,
205
  "clip_ratio/low_min": 0.0,
206
  "clip_ratio/region_mean": 0.0,
207
- "completion_length": 24.4,
208
- "completions/clipped_ratio": 0.0,
209
- "completions/max_length": 33.4,
210
- "completions/max_terminated_length": 33.4,
211
- "completions/mean_length": 24.4,
212
- "completions/mean_terminated_length": 24.4,
213
- "completions/min_length": 18.8,
214
- "completions/min_terminated_length": 18.8,
215
- "epoch": 0.18018018018018017,
216
  "frac_reward_zero_std": 0.0,
217
- "grad_norm": 1.0676746368408203,
218
- "kl": 0.006288998411764624,
219
- "learning_rate": 4.357142857142857e-06,
220
- "loss": 0.00025154203176498414,
221
- "num_tokens": 164334.0,
222
- "reward": 0.5014999985694886,
223
- "reward_std": 0.15358359068632127,
224
- "rewards/reward_fn/mean": 0.5014999985694886,
225
- "rewards/reward_fn/std": 0.13824734836816788,
226
  "step": 40
227
  },
228
  {
@@ -231,25 +231,25 @@
231
  "clip_ratio/low_mean": 0.0,
232
  "clip_ratio/low_min": 0.0,
233
  "clip_ratio/region_mean": 0.0,
234
- "completion_length": 27.35,
235
  "completions/clipped_ratio": 0.0,
236
- "completions/max_length": 43.6,
237
- "completions/max_terminated_length": 43.6,
238
- "completions/mean_length": 27.35,
239
- "completions/mean_terminated_length": 27.35,
240
- "completions/min_length": 19.6,
241
- "completions/min_terminated_length": 19.6,
242
- "epoch": 0.20270270270270271,
243
  "frac_reward_zero_std": 0.0,
244
- "grad_norm": 1.0749404430389404,
245
- "kl": 0.010629063473606948,
246
- "learning_rate": 4.238095238095239e-06,
247
- "loss": 0.0004251018166542053,
248
- "num_tokens": 185140.0,
249
- "reward": 0.524949985742569,
250
- "reward_std": 0.14630039632320405,
251
- "rewards/reward_fn/mean": 0.5249499917030335,
252
- "rewards/reward_fn/std": 0.1460244983434677,
253
  "step": 45
254
  },
255
  {
@@ -258,31 +258,31 @@
258
  "clip_ratio/low_mean": 0.0,
259
  "clip_ratio/low_min": 0.0,
260
  "clip_ratio/region_mean": 0.0,
261
- "completion_length": 25.425,
262
  "completions/clipped_ratio": 0.0,
263
  "completions/max_length": 35.2,
264
  "completions/max_terminated_length": 35.2,
265
- "completions/mean_length": 25.425,
266
- "completions/mean_terminated_length": 25.425,
267
- "completions/min_length": 17.6,
268
- "completions/min_terminated_length": 17.6,
269
- "epoch": 0.22522522522522523,
270
  "frac_reward_zero_std": 0.0,
271
- "grad_norm": 1.4353065490722656,
272
- "kl": 0.008290678875346203,
273
- "learning_rate": 4.119047619047619e-06,
274
- "loss": 0.0003315746784210205,
275
- "num_tokens": 206569.0,
276
- "reward": 0.4908999800682068,
277
- "reward_std": 0.17055415213108063,
278
- "rewards/reward_fn/mean": 0.49089999198913575,
279
- "rewards/reward_fn/std": 0.154354290664196,
280
  "step": 50
281
  }
282
  ],
283
  "logging_steps": 5,
284
- "max_steps": 222,
285
- "num_input_tokens_seen": 206569,
286
  "num_train_epochs": 1,
287
  "save_steps": 50,
288
  "stateful_callbacks": {
 
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
+ "epoch": 0.2222222222222222,
6
  "eval_steps": 500,
7
  "global_step": 50,
8
  "is_hyper_param_search": false,
 
15
  "clip_ratio/low_mean": 0.0,
16
  "clip_ratio/low_min": 0.0,
17
  "clip_ratio/region_mean": 0.0,
18
+ "completion_length": 25.075,
19
+ "completions/clipped_ratio": 0.0,
20
+ "completions/max_length": 30.8,
21
+ "completions/max_terminated_length": 30.8,
22
+ "completions/mean_length": 25.075,
23
+ "completions/mean_terminated_length": 25.075,
24
+ "completions/min_length": 19.4,
25
+ "completions/min_terminated_length": 19.4,
26
+ "epoch": 0.022222222222222223,
27
+ "frac_reward_zero_std": 0.6,
28
+ "grad_norm": 1.3092279434204102,
29
+ "kl": 0.03290070100920275,
30
  "learning_rate": 1.6666666666666667e-06,
31
+ "loss": 0.0013160213828086853,
32
+ "num_tokens": 20275.0,
33
+ "reward": 0.48624999523162843,
34
+ "reward_std": 0.05833630859851837,
35
+ "rewards/reward_fn/mean": 0.4862500011920929,
36
+ "rewards/reward_fn/std": 0.04605271518230438,
37
  "step": 5
38
  },
39
  {
 
42
  "clip_ratio/low_mean": 0.0,
43
  "clip_ratio/low_min": 0.0,
44
  "clip_ratio/region_mean": 0.0,
45
+ "completion_length": 25.925,
46
  "completions/clipped_ratio": 0.0,
47
+ "completions/max_length": 33.0,
48
+ "completions/max_terminated_length": 33.0,
49
+ "completions/mean_length": 25.925,
50
+ "completions/mean_terminated_length": 25.925,
51
+ "completions/min_length": 19.4,
52
+ "completions/min_terminated_length": 19.4,
53
+ "epoch": 0.044444444444444446,
54
  "frac_reward_zero_std": 0.0,
55
+ "grad_norm": 1.4204450845718384,
56
+ "kl": 0.040759827199508436,
57
  "learning_rate": 3.7500000000000005e-06,
58
+ "loss": 0.0016303554177284241,
59
+ "num_tokens": 41560.0,
60
+ "reward": 0.5283999860286712,
61
+ "reward_std": 0.17366542518138886,
62
+ "rewards/reward_fn/mean": 0.5283999919891358,
63
+ "rewards/reward_fn/std": 0.16809422075748442,
64
  "step": 10
65
  },
66
  {
 
69
  "clip_ratio/low_mean": 0.0,
70
  "clip_ratio/low_min": 0.0,
71
  "clip_ratio/region_mean": 0.0,
72
+ "completion_length": 27.45,
73
  "completions/clipped_ratio": 0.0,
74
+ "completions/max_length": 33.6,
75
+ "completions/max_terminated_length": 33.6,
76
+ "completions/mean_length": 27.45,
77
+ "completions/mean_terminated_length": 27.45,
78
+ "completions/min_length": 24.0,
79
+ "completions/min_terminated_length": 24.0,
80
+ "epoch": 0.06666666666666667,
81
  "frac_reward_zero_std": 0.0,
82
+ "grad_norm": 0.9515441656112671,
83
+ "kl": 0.049849181214813146,
84
+ "learning_rate": 4.953051643192488e-06,
85
+ "loss": 0.0019938603043556215,
86
+ "num_tokens": 62246.0,
87
+ "reward": 0.4975499987602234,
88
+ "reward_std": 0.1564827263355255,
89
+ "rewards/reward_fn/mean": 0.49755001068115234,
90
+ "rewards/reward_fn/std": 0.14618260115385057,
91
  "step": 15
92
  },
93
  {
 
96
  "clip_ratio/low_mean": 0.0,
97
  "clip_ratio/low_min": 0.0,
98
  "clip_ratio/region_mean": 0.0,
99
+ "completion_length": 26.525,
100
  "completions/clipped_ratio": 0.0,
101
+ "completions/max_length": 33.0,
102
+ "completions/max_terminated_length": 33.0,
103
+ "completions/mean_length": 26.525,
104
+ "completions/mean_terminated_length": 26.525,
105
+ "completions/min_length": 22.2,
106
+ "completions/min_terminated_length": 22.2,
107
+ "epoch": 0.08888888888888889,
108
  "frac_reward_zero_std": 0.0,
109
+ "grad_norm": 0.6862999200820923,
110
+ "kl": 0.04350169296376407,
111
+ "learning_rate": 4.835680751173709e-06,
112
+ "loss": 0.0017400771379470826,
113
+ "num_tokens": 81479.0,
114
+ "reward": 0.5159000098705292,
115
+ "reward_std": 0.17465537190437316,
116
+ "rewards/reward_fn/mean": 0.5158999919891357,
117
+ "rewards/reward_fn/std": 0.16455023884773254,
118
  "step": 20
119
  },
120
  {
 
123
  "clip_ratio/low_mean": 0.0,
124
  "clip_ratio/low_min": 0.0,
125
  "clip_ratio/region_mean": 0.0,
126
+ "completion_length": 28.0,
127
  "completions/clipped_ratio": 0.0,
128
+ "completions/max_length": 40.6,
129
+ "completions/max_terminated_length": 40.6,
130
+ "completions/mean_length": 28.0,
131
+ "completions/mean_terminated_length": 28.0,
132
+ "completions/min_length": 22.6,
133
+ "completions/min_terminated_length": 22.6,
134
+ "epoch": 0.1111111111111111,
135
  "frac_reward_zero_std": 0.0,
136
+ "grad_norm": 1.7516958713531494,
137
+ "kl": 0.05824573401478119,
138
+ "learning_rate": 4.71830985915493e-06,
139
+ "loss": 0.002329717576503754,
140
+ "num_tokens": 102833.0,
141
+ "reward": 0.5547999978065491,
142
+ "reward_std": 0.18328206837177277,
143
+ "rewards/reward_fn/mean": 0.5547999858856201,
144
+ "rewards/reward_fn/std": 0.1974634051322937,
145
  "step": 25
146
  },
147
  {
 
150
  "clip_ratio/low_mean": 0.0,
151
  "clip_ratio/low_min": 0.0,
152
  "clip_ratio/region_mean": 0.0,
153
+ "completion_length": 28.675,
154
  "completions/clipped_ratio": 0.0,
155
+ "completions/max_length": 37.4,
156
+ "completions/max_terminated_length": 37.4,
157
+ "completions/mean_length": 28.675,
158
+ "completions/mean_terminated_length": 28.675,
159
+ "completions/min_length": 22.2,
160
+ "completions/min_terminated_length": 22.2,
161
+ "epoch": 0.13333333333333333,
162
  "frac_reward_zero_std": 0.0,
163
+ "grad_norm": 1.5611271858215332,
164
+ "kl": 0.09752920938190072,
165
+ "learning_rate": 4.60093896713615e-06,
166
+ "loss": 0.0039011374115943908,
167
+ "num_tokens": 123876.0,
168
+ "reward": 0.5165499806404114,
169
+ "reward_std": 0.16765501499176025,
170
+ "rewards/reward_fn/mean": 0.5165499806404114,
171
+ "rewards/reward_fn/std": 0.16748485416173936,
172
  "step": 30
173
  },
174
  {
 
177
  "clip_ratio/low_mean": 0.0,
178
  "clip_ratio/low_min": 0.0,
179
  "clip_ratio/region_mean": 0.0,
180
+ "completion_length": 27.475,
181
+ "completions/clipped_ratio": 0.0,
182
+ "completions/max_length": 36.8,
183
+ "completions/max_terminated_length": 36.8,
184
+ "completions/mean_length": 27.475,
185
+ "completions/mean_terminated_length": 27.475,
186
+ "completions/min_length": 20.6,
187
+ "completions/min_terminated_length": 20.6,
188
+ "epoch": 0.15555555555555556,
189
  "frac_reward_zero_std": 0.0,
190
+ "grad_norm": 0.9550014138221741,
191
+ "kl": 0.08032013729680329,
192
+ "learning_rate": 4.483568075117371e-06,
193
+ "loss": 0.003212757408618927,
194
+ "num_tokens": 145019.0,
195
+ "reward": 0.5004499793052674,
196
+ "reward_std": 0.18391846716403962,
197
+ "rewards/reward_fn/mean": 0.5004500031471253,
198
+ "rewards/reward_fn/std": 0.16948954164981841,
199
  "step": 35
200
  },
201
  {
 
204
  "clip_ratio/low_mean": 0.0,
205
  "clip_ratio/low_min": 0.0,
206
  "clip_ratio/region_mean": 0.0,
207
+ "completion_length": 32.125,
208
+ "completions/clipped_ratio": 0.025,
209
+ "completions/max_length": 54.6,
210
+ "completions/max_terminated_length": 46.0,
211
+ "completions/mean_length": 32.125,
212
+ "completions/mean_terminated_length": 30.521428680419923,
213
+ "completions/min_length": 23.2,
214
+ "completions/min_terminated_length": 23.2,
215
+ "epoch": 0.17777777777777778,
216
  "frac_reward_zero_std": 0.0,
217
+ "grad_norm": 1.1203869581222534,
218
+ "kl": 0.08563535290304572,
219
+ "learning_rate": 4.3661971830985915e-06,
220
+ "loss": 0.003425435721874237,
221
+ "num_tokens": 164800.0,
222
+ "reward": 0.5472500026226044,
223
+ "reward_std": 0.19579786211252212,
224
+ "rewards/reward_fn/mean": 0.5472500085830688,
225
+ "rewards/reward_fn/std": 0.19857290089130403,
226
  "step": 40
227
  },
228
  {
 
231
  "clip_ratio/low_mean": 0.0,
232
  "clip_ratio/low_min": 0.0,
233
  "clip_ratio/region_mean": 0.0,
234
+ "completion_length": 27.875,
235
  "completions/clipped_ratio": 0.0,
236
+ "completions/max_length": 35.0,
237
+ "completions/max_terminated_length": 35.0,
238
+ "completions/mean_length": 27.875,
239
+ "completions/mean_terminated_length": 27.875,
240
+ "completions/min_length": 21.4,
241
+ "completions/min_terminated_length": 21.4,
242
+ "epoch": 0.2,
243
  "frac_reward_zero_std": 0.0,
244
+ "grad_norm": 0.95042884349823,
245
+ "kl": 0.07295196709455923,
246
+ "learning_rate": 4.248826291079813e-06,
247
+ "loss": 0.0029180020093917845,
248
+ "num_tokens": 185761.0,
249
+ "reward": 0.5225999832153321,
250
+ "reward_std": 0.16489729881286622,
251
+ "rewards/reward_fn/mean": 0.5226000070571899,
252
+ "rewards/reward_fn/std": 0.16541497856378556,
253
  "step": 45
254
  },
255
  {
 
258
  "clip_ratio/low_mean": 0.0,
259
  "clip_ratio/low_min": 0.0,
260
  "clip_ratio/region_mean": 0.0,
261
+ "completion_length": 27.875,
262
  "completions/clipped_ratio": 0.0,
263
  "completions/max_length": 35.2,
264
  "completions/max_terminated_length": 35.2,
265
+ "completions/mean_length": 27.875,
266
+ "completions/mean_terminated_length": 27.875,
267
+ "completions/min_length": 21.4,
268
+ "completions/min_terminated_length": 21.4,
269
+ "epoch": 0.2222222222222222,
270
  "frac_reward_zero_std": 0.0,
271
+ "grad_norm": 1.1037548780441284,
272
+ "kl": 0.08246680488809943,
273
+ "learning_rate": 4.131455399061034e-06,
274
+ "loss": 0.0032985761761665346,
275
+ "num_tokens": 206674.0,
276
+ "reward": 0.5245000004768372,
277
+ "reward_std": 0.1653215616941452,
278
+ "rewards/reward_fn/mean": 0.5245000004768372,
279
+ "rewards/reward_fn/std": 0.16877340227365495,
280
  "step": 50
281
  }
282
  ],
283
  "logging_steps": 5,
284
+ "max_steps": 225,
285
+ "num_input_tokens_seen": 206674,
286
  "num_train_epochs": 1,
287
  "save_steps": 50,
288
  "stateful_callbacks": {
results/phase1/sft_warmup/README.md ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
3
+ library_name: transformers
4
+ model_name: sft_warmup
5
+ tags:
6
+ - generated_from_trainer
7
+ - unsloth
8
+ - trl
9
+ - sft
10
+ licence: license
11
+ ---
12
+
13
+ # Model Card for sft_warmup
14
+
15
+ This model is a fine-tuned version of [unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit](https://huggingface.co/unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit).
16
+ It has been trained using [TRL](https://github.com/huggingface/trl).
17
+
18
+ ## Quick start
19
+
20
+ ```python
21
+ from transformers import pipeline
22
+
23
+ question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
24
+ generator = pipeline("text-generation", model="None", device="cuda")
25
+ output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
26
+ print(output["generated_text"])
27
+ ```
28
+
29
+ ## Training procedure
30
+
31
+
32
+
33
+
34
+ This model was trained with SFT.
35
+
36
+ ### Framework versions
37
+
38
+ - TRL: 0.24.0
39
+ - Transformers: 5.5.0
40
+ - Pytorch: 2.10.0+cu128
41
+ - Datasets: 4.3.0
42
+ - Tokenizers: 0.22.2
43
+
44
+ ## Citations
45
+
46
+
47
+
48
+ Cite TRL as:
49
+
50
+ ```bibtex
51
+ @misc{vonwerra2022trl,
52
+ title = {{TRL: Transformer Reinforcement Learning}},
53
+ author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
54
+ year = 2020,
55
+ journal = {GitHub repository},
56
+ publisher = {GitHub},
57
+ howpublished = {\url{https://github.com/huggingface/trl}}
58
+ }
59
+ ```
results/phase1/sft_warmup/checkpoint-48/README.md ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
7
+ - lora
8
+ - sft
9
+ - transformers
10
+ - trl
11
+ - unsloth
12
+ ---
13
+
14
+ # Model Card for Model ID
15
+
16
+ <!-- Provide a quick summary of what the model is/does. -->
17
+
18
+
19
+
20
+ ## Model Details
21
+
22
+ ### Model Description
23
+
24
+ <!-- Provide a longer summary of what this model is. -->
25
+
26
+
27
+
28
+ - **Developed by:** [More Information Needed]
29
+ - **Funded by [optional]:** [More Information Needed]
30
+ - **Shared by [optional]:** [More Information Needed]
31
+ - **Model type:** [More Information Needed]
32
+ - **Language(s) (NLP):** [More Information Needed]
33
+ - **License:** [More Information Needed]
34
+ - **Finetuned from model [optional]:** [More Information Needed]
35
+
36
+ ### Model Sources [optional]
37
+
38
+ <!-- Provide the basic links for the model. -->
39
+
40
+ - **Repository:** [More Information Needed]
41
+ - **Paper [optional]:** [More Information Needed]
42
+ - **Demo [optional]:** [More Information Needed]
43
+
44
+ ## Uses
45
+
46
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
47
+
48
+ ### Direct Use
49
+
50
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
51
+
52
+ [More Information Needed]
53
+
54
+ ### Downstream Use [optional]
55
+
56
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
57
+
58
+ [More Information Needed]
59
+
60
+ ### Out-of-Scope Use
61
+
62
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
63
+
64
+ [More Information Needed]
65
+
66
+ ## Bias, Risks, and Limitations
67
+
68
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
69
+
70
+ [More Information Needed]
71
+
72
+ ### Recommendations
73
+
74
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
75
+
76
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
77
+
78
+ ## How to Get Started with the Model
79
+
80
+ Use the code below to get started with the model.
81
+
82
+ [More Information Needed]
83
+
84
+ ## Training Details
85
+
86
+ ### Training Data
87
+
88
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
89
+
90
+ [More Information Needed]
91
+
92
+ ### Training Procedure
93
+
94
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
95
+
96
+ #### Preprocessing [optional]
97
+
98
+ [More Information Needed]
99
+
100
+
101
+ #### Training Hyperparameters
102
+
103
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
104
+
105
+ #### Speeds, Sizes, Times [optional]
106
+
107
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
108
+
109
+ [More Information Needed]
110
+
111
+ ## Evaluation
112
+
113
+ <!-- This section describes the evaluation protocols and provides the results. -->
114
+
115
+ ### Testing Data, Factors & Metrics
116
+
117
+ #### Testing Data
118
+
119
+ <!-- This should link to a Dataset Card if possible. -->
120
+
121
+ [More Information Needed]
122
+
123
+ #### Factors
124
+
125
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
126
+
127
+ [More Information Needed]
128
+
129
+ #### Metrics
130
+
131
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
132
+
133
+ [More Information Needed]
134
+
135
+ ### Results
136
+
137
+ [More Information Needed]
138
+
139
+ #### Summary
140
+
141
+
142
+
143
+ ## Model Examination [optional]
144
+
145
+ <!-- Relevant interpretability work for the model goes here -->
146
+
147
+ [More Information Needed]
148
+
149
+ ## Environmental Impact
150
+
151
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
152
+
153
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
154
+
155
+ - **Hardware Type:** [More Information Needed]
156
+ - **Hours used:** [More Information Needed]
157
+ - **Cloud Provider:** [More Information Needed]
158
+ - **Compute Region:** [More Information Needed]
159
+ - **Carbon Emitted:** [More Information Needed]
160
+
161
+ ## Technical Specifications [optional]
162
+
163
+ ### Model Architecture and Objective
164
+
165
+ [More Information Needed]
166
+
167
+ ### Compute Infrastructure
168
+
169
+ [More Information Needed]
170
+
171
+ #### Hardware
172
+
173
+ [More Information Needed]
174
+
175
+ #### Software
176
+
177
+ [More Information Needed]
178
+
179
+ ## Citation [optional]
180
+
181
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
182
+
183
+ **BibTeX:**
184
+
185
+ [More Information Needed]
186
+
187
+ **APA:**
188
+
189
+ [More Information Needed]
190
+
191
+ ## Glossary [optional]
192
+
193
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
194
+
195
+ [More Information Needed]
196
+
197
+ ## More Information [optional]
198
+
199
+ [More Information Needed]
200
+
201
+ ## Model Card Authors [optional]
202
+
203
+ [More Information Needed]
204
+
205
+ ## Model Card Contact
206
+
207
+ [More Information Needed]
208
+ ### Framework versions
209
+
210
+ - PEFT 0.18.1
results/phase1/sft_warmup/checkpoint-48/adapter_config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": {
6
+ "base_model_class": "Qwen2ForCausalLM",
7
+ "parent_library": "transformers.models.qwen2.modeling_qwen2",
8
+ "unsloth_fixed": true
9
+ },
10
+ "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit",
11
+ "bias": "none",
12
+ "corda_config": null,
13
+ "ensure_weight_tying": false,
14
+ "eva_config": null,
15
+ "exclude_modules": null,
16
+ "fan_in_fan_out": false,
17
+ "inference_mode": true,
18
+ "init_lora_weights": true,
19
+ "layer_replication": null,
20
+ "layers_pattern": null,
21
+ "layers_to_transform": null,
22
+ "loftq_config": {},
23
+ "lora_alpha": 32,
24
+ "lora_bias": false,
25
+ "lora_dropout": 0.05,
26
+ "megatron_config": null,
27
+ "megatron_core": "megatron.core",
28
+ "modules_to_save": null,
29
+ "peft_type": "LORA",
30
+ "peft_version": "0.18.1",
31
+ "qalora_group_size": 16,
32
+ "r": 16,
33
+ "rank_pattern": {},
34
+ "revision": null,
35
+ "target_modules": [
36
+ "gate_proj",
37
+ "q_proj",
38
+ "k_proj",
39
+ "v_proj",
40
+ "down_proj",
41
+ "o_proj",
42
+ "up_proj"
43
+ ],
44
+ "target_parameters": null,
45
+ "task_type": "CAUSAL_LM",
46
+ "trainable_token_indices": null,
47
+ "use_dora": false,
48
+ "use_qalora": false,
49
+ "use_rslora": false
50
+ }
results/phase1/sft_warmup/checkpoint-48/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58986d6fe9c326b9899b549ea777f5f19819c15ce46e32313d9421d1af860335
3
+ size 73911112
results/phase1/sft_warmup/checkpoint-48/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
results/phase1/sft_warmup/checkpoint-48/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8612906f7146fa5cd7415c37909b407936414060b71ebc534a3f5ffaf7f3d53d
3
+ size 37969669