File size: 12,090 Bytes
f78519d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 | {
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.00024,
"eval_steps": 500,
"global_step": 12,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
"completions/max_length": 1928.0,
"completions/max_terminated_length": 1928.0,
"completions/mean_length": 1755.5,
"completions/mean_terminated_length": 1755.5,
"completions/min_length": 1363.0,
"completions/min_terminated_length": 1363.0,
"entropy": 0.5008892454206944,
"epoch": 2e-05,
"frac_reward_zero_std": 0.375,
"grad_norm": 1.3115488290786743,
"kl": 0.0,
"learning_rate": 0.0,
"loss": -0.013,
"num_tokens": 79465.0,
"reward": -0.06562499701976776,
"reward_std": 0.11540064215660095,
"rewards/rollout_reward_func/mean": -0.06562499701976776,
"rewards/rollout_reward_func/std": 0.19278831779956818,
"sampling/importance_sampling_ratio/max": 1.7688987255096436,
"sampling/importance_sampling_ratio/mean": 1.0469263792037964,
"sampling/importance_sampling_ratio/min": 0.6919283866882324,
"sampling/sampling_logp_difference/max": 0.3354456424713135,
"sampling/sampling_logp_difference/mean": 0.018323732540011406,
"step": 1,
"step_time": 37.61465689600027
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"entropy": 0.5008892454206944,
"epoch": 4e-05,
"grad_norm": 1.3185971975326538,
"kl": 0.0,
"learning_rate": 2.8571428571428575e-07,
"loss": -0.013,
"step": 2,
"step_time": 6.174051868999413
},
{
"clip_ratio/high_max": 0.00390625,
"clip_ratio/high_mean": 0.001953125,
"clip_ratio/low_mean": 0.00390625,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.005859375,
"completions/clipped_ratio": 0.0,
"completions/max_length": 1859.0,
"completions/max_terminated_length": 1859.0,
"completions/mean_length": 1702.625,
"completions/mean_terminated_length": 1702.625,
"completions/min_length": 1369.0,
"completions/min_terminated_length": 1369.0,
"entropy": 0.5293242037296295,
"epoch": 6e-05,
"frac_reward_zero_std": 0.375,
"grad_norm": 2.0585877895355225,
"kl": 0.0012533214612631127,
"learning_rate": 5.714285714285715e-07,
"loss": 0.0512,
"num_tokens": 157637.0,
"reward": -0.06562500447034836,
"reward_std": 0.07261751592159271,
"rewards/rollout_reward_func/mean": -0.06562500447034836,
"rewards/rollout_reward_func/std": 0.11807426810264587,
"sampling/importance_sampling_ratio/max": 2.1121888160705566,
"sampling/importance_sampling_ratio/mean": 1.0355395078659058,
"sampling/importance_sampling_ratio/min": 0.5390263199806213,
"sampling/sampling_logp_difference/max": 0.683211088180542,
"sampling/sampling_logp_difference/mean": 0.02265790104866028,
"step": 3,
"step_time": 34.844595972002026
},
{
"clip_ratio/high_max": 0.00390625,
"clip_ratio/high_mean": 0.001953125,
"clip_ratio/low_mean": 0.0035556890070438385,
"clip_ratio/low_min": 0.0032051282469183207,
"clip_ratio/region_mean": 0.0055088140070438385,
"entropy": 0.5292382091283798,
"epoch": 8e-05,
"grad_norm": 2.174487829208374,
"kl": 0.0010874079816858284,
"learning_rate": 8.571428571428572e-07,
"loss": 0.0533,
"step": 4,
"step_time": 6.182152846000463
},
{
"clip_ratio/high_max": 0.0032051282469183207,
"clip_ratio/high_mean": 0.0016025641234591603,
"clip_ratio/low_mean": 0.001953125,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0035556891234591603,
"completions/clipped_ratio": 0.0,
"completions/max_length": 1825.0,
"completions/max_terminated_length": 1825.0,
"completions/mean_length": 1647.34375,
"completions/mean_terminated_length": 1647.34375,
"completions/min_length": 1082.0,
"completions/min_terminated_length": 1082.0,
"entropy": 0.5518431439995766,
"epoch": 0.0001,
"frac_reward_zero_std": 0.125,
"grad_norm": 2.0179476737976074,
"kl": 0.0010270353013765998,
"learning_rate": 1.142857142857143e-06,
"loss": 0.0455,
"num_tokens": 233699.0,
"reward": -0.07500000298023224,
"reward_std": 0.1056123897433281,
"rewards/rollout_reward_func/mean": -0.07500000298023224,
"rewards/rollout_reward_func/std": 0.11359237134456635,
"sampling/importance_sampling_ratio/max": 1.5899720191955566,
"sampling/importance_sampling_ratio/mean": 1.0574636459350586,
"sampling/importance_sampling_ratio/min": 0.7474462389945984,
"sampling/sampling_logp_difference/max": 0.23842597007751465,
"sampling/sampling_logp_difference/mean": 0.02176746539771557,
"step": 5,
"step_time": 36.2267263499989
},
{
"clip_ratio/high_max": 0.0032051282469183207,
"clip_ratio/high_mean": 0.0016025641234591603,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0016025641234591603,
"entropy": 0.551122885197401,
"epoch": 0.00012,
"grad_norm": 2.1906440258026123,
"kl": 0.0010457161733938847,
"learning_rate": 1.4285714285714286e-06,
"loss": 0.0423,
"step": 6,
"step_time": 6.1078073650005535
},
{
"clip_ratio/high_max": 0.005681818351149559,
"clip_ratio/high_mean": 0.0028409091755747795,
"clip_ratio/low_mean": 0.0026041667442768812,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.005445075919851661,
"completions/clipped_ratio": 0.0,
"completions/max_length": 1803.0,
"completions/max_terminated_length": 1803.0,
"completions/mean_length": 1562.1875,
"completions/mean_terminated_length": 1562.1875,
"completions/min_length": 359.0,
"completions/min_terminated_length": 359.0,
"entropy": 0.48404254019260406,
"epoch": 0.00014,
"frac_reward_zero_std": 0.25,
"grad_norm": 1.9169107675552368,
"kl": 0.001029455175739713,
"learning_rate": 1.7142857142857145e-06,
"loss": -0.0132,
"num_tokens": 307662.0,
"reward": -0.06875000894069672,
"reward_std": 0.20966878533363342,
"rewards/rollout_reward_func/mean": -0.06875000894069672,
"rewards/rollout_reward_func/std": 0.3440156579017639,
"sampling/importance_sampling_ratio/max": 1.3860647678375244,
"sampling/importance_sampling_ratio/mean": 1.0004303455352783,
"sampling/importance_sampling_ratio/min": 0.6425995826721191,
"sampling/sampling_logp_difference/max": 0.30062544345855713,
"sampling/sampling_logp_difference/mean": 0.02074645832180977,
"step": 7,
"step_time": 35.09266174299955
},
{
"clip_ratio/high_max": 0.0037878789007663727,
"clip_ratio/high_mean": 0.0018939394503831863,
"clip_ratio/low_mean": 0.0026041667442768812,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.004498106427490711,
"entropy": 0.48381197452545166,
"epoch": 0.00016,
"grad_norm": 1.8268851041793823,
"kl": 0.001109405180613976,
"learning_rate": 2.0000000000000003e-06,
"loss": -0.0145,
"step": 8,
"step_time": 6.09429400800218
},
{
"clip_ratio/high_max": 0.0078125,
"clip_ratio/high_mean": 0.005800189450383186,
"clip_ratio/low_mean": 0.003791360300965607,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.009591549751348794,
"completions/clipped_ratio": 0.0,
"completions/max_length": 1877.0,
"completions/max_terminated_length": 1877.0,
"completions/mean_length": 1641.03125,
"completions/mean_terminated_length": 1641.03125,
"completions/min_length": 294.0,
"completions/min_terminated_length": 294.0,
"entropy": 0.5420897267758846,
"epoch": 0.00018,
"frac_reward_zero_std": 0.0,
"grad_norm": 2.5005621910095215,
"kl": 0.0013470195408444852,
"learning_rate": 2.285714285714286e-06,
"loss": -0.0267,
"num_tokens": 383747.0,
"reward": -0.10625000298023224,
"reward_std": 0.16220411658287048,
"rewards/rollout_reward_func/mean": -0.10625000298023224,
"rewards/rollout_reward_func/std": 0.19499793648719788,
"sampling/importance_sampling_ratio/max": 1.596044898033142,
"sampling/importance_sampling_ratio/mean": 0.9659844636917114,
"sampling/importance_sampling_ratio/min": 0.42962342500686646,
"sampling/sampling_logp_difference/max": 0.3873096704483032,
"sampling/sampling_logp_difference/mean": 0.02420671284198761,
"step": 9,
"step_time": 35.05370295100056
},
{
"clip_ratio/high_max": 0.00390625,
"clip_ratio/high_mean": 0.001953125,
"clip_ratio/low_mean": 0.001953125,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.00390625,
"entropy": 0.5415336266160011,
"epoch": 0.0002,
"grad_norm": 2.5184950828552246,
"kl": 0.0014528355168295093,
"learning_rate": 2.571428571428571e-06,
"loss": -0.0249,
"step": 10,
"step_time": 6.594039571999929
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
"completions/max_length": 1835.0,
"completions/max_terminated_length": 1835.0,
"completions/mean_length": 1630.1875,
"completions/mean_terminated_length": 1630.1875,
"completions/min_length": 1176.0,
"completions/min_terminated_length": 1176.0,
"entropy": 0.5399914123117924,
"epoch": 0.00022,
"frac_reward_zero_std": 0.125,
"grad_norm": 1.7063910961151123,
"kl": 0.0010663218854460865,
"learning_rate": 2.8571428571428573e-06,
"loss": -0.0121,
"num_tokens": 459548.0,
"reward": -0.08124999701976776,
"reward_std": 0.07863743603229523,
"rewards/rollout_reward_func/mean": -0.08124999701976776,
"rewards/rollout_reward_func/std": 0.11198358237743378,
"sampling/importance_sampling_ratio/max": 1.3067278861999512,
"sampling/importance_sampling_ratio/mean": 0.980239748954773,
"sampling/importance_sampling_ratio/min": 0.6372140645980835,
"sampling/sampling_logp_difference/max": 0.36971473693847656,
"sampling/sampling_logp_difference/mean": 0.021189481019973755,
"step": 11,
"step_time": 34.695202157001404
},
{
"clip_ratio/high_max": 0.00390625,
"clip_ratio/high_mean": 0.001953125,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.001953125,
"entropy": 0.5411706231534481,
"epoch": 0.00024,
"grad_norm": 1.7831242084503174,
"kl": 0.0010139764053747058,
"learning_rate": 3.142857142857143e-06,
"loss": -0.0092,
"step": 12,
"step_time": 6.109187065001606
}
],
"logging_steps": 1.0,
"max_steps": 100000,
"num_input_tokens_seen": 459548,
"num_train_epochs": 2,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}
|