File size: 15,046 Bytes
fe1c2de | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 | {
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.002535004852157725,
"eval_steps": 500,
"global_step": 16,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0001584378032598578,
"grad_norm": 4.8242447519442067e-05,
"learning_rate": 0.0029998415465061004,
"loss": 0.0,
"loss/policy_avg": 7.299240678548813e-08,
"objective/entropy": 67.43842315673828,
"objective/kl": 0.0,
"objective/rlhf_reward": 1.5219240188598633,
"objective/scores": 1.52203369140625,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 1,
"timer/active_model_forward": 0.7594873309135437,
"timer/calc_advantages": 31.33674430847168,
"timer/calc_loss": 0.774461030960083,
"timer/generate": 28.869869232177734,
"timer/get_reward": 0.5197552442550659,
"timer/model_forward": 0.7690438628196716,
"timer/ref_model_forward": 0.7777600884437561,
"timer/training_step": 34.545597076416016,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0003168756065197156,
"grad_norm": 0.00026670348597690463,
"learning_rate": 0.002999683093012201,
"loss": -0.0,
"loss/policy_avg": -1.7043203115463257e-07,
"objective/entropy": 118.22715759277344,
"objective/kl": 0.0,
"objective/rlhf_reward": -3.824268341064453,
"objective/scores": -3.82421875,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 2,
"timer/active_model_forward": 0.6569761037826538,
"timer/calc_advantages": 30.551788330078125,
"timer/calc_loss": 0.6658673286437988,
"timer/generate": 28.41728973388672,
"timer/get_reward": 0.43157756328582764,
"timer/model_forward": 0.6641299724578857,
"timer/ref_model_forward": 0.67349773645401,
"timer/training_step": 33.37031936645508,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0004753134097795734,
"grad_norm": 0.00013700829003937542,
"learning_rate": 0.0029995246395183015,
"loss": -0.0,
"loss/policy_avg": -2.9243528842926025e-07,
"objective/entropy": 1081.7860107421875,
"objective/kl": 0.0,
"objective/rlhf_reward": -5.476104736328125,
"objective/scores": -5.476806640625,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 3,
"timer/active_model_forward": 0.9020705223083496,
"timer/calc_advantages": 32.11817169189453,
"timer/calc_loss": 0.9118615984916687,
"timer/generate": 29.300296783447266,
"timer/get_reward": 0.5959209203720093,
"timer/model_forward": 0.9100804924964905,
"timer/ref_model_forward": 0.9205952882766724,
"timer/training_step": 35.83306121826172,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0006337512130394313,
"grad_norm": 7.692693907301873e-05,
"learning_rate": 0.002999366186024402,
"loss": -0.0,
"loss/policy_avg": -2.0489096641540527e-07,
"objective/entropy": 1181.2274169921875,
"objective/kl": 0.0,
"objective/rlhf_reward": -5.0478363037109375,
"objective/scores": -5.04833984375,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 4,
"timer/active_model_forward": 0.8427364826202393,
"timer/calc_advantages": 31.744182586669922,
"timer/calc_loss": 0.8533791899681091,
"timer/generate": 29.096416473388672,
"timer/get_reward": 0.5531487464904785,
"timer/model_forward": 0.851642906665802,
"timer/ref_model_forward": 0.8617820143699646,
"timer/training_step": 35.25868225097656,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.000792189016299289,
"grad_norm": 3.911158273695037e-05,
"learning_rate": 0.0029992077325305022,
"loss": -0.0,
"loss/policy_avg": -2.1513551473617554e-07,
"objective/entropy": 1185.438232421875,
"objective/kl": 0.0,
"objective/rlhf_reward": -4.198909759521484,
"objective/scores": -4.198486328125,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 5,
"timer/active_model_forward": 0.791987955570221,
"timer/calc_advantages": 31.324127197265625,
"timer/calc_loss": 0.8030194044113159,
"timer/generate": 28.832427978515625,
"timer/get_reward": 0.5202938914299011,
"timer/model_forward": 0.8012962937355042,
"timer/ref_model_forward": 0.8093616366386414,
"timer/training_step": 34.65721130371094,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0009506268195591468,
"grad_norm": 2.5278823159169406e-05,
"learning_rate": 0.002999049279036603,
"loss": -0.0,
"loss/policy_avg": -1.2619420886039734e-07,
"objective/entropy": 528.8932495117188,
"objective/kl": 0.0,
"objective/rlhf_reward": -3.1182708740234375,
"objective/scores": -3.1180419921875,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 6,
"timer/active_model_forward": 0.7996934652328491,
"timer/calc_advantages": 31.30746078491211,
"timer/calc_loss": 0.8107499480247498,
"timer/generate": 28.788341522216797,
"timer/get_reward": 0.5261460542678833,
"timer/model_forward": 0.8090219497680664,
"timer/ref_model_forward": 0.8171602487564087,
"timer/training_step": 34.65579605102539,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0011090646228190046,
"grad_norm": 1.0207173772869282e-06,
"learning_rate": 0.0029988908255427034,
"loss": -0.0,
"loss/policy_avg": -3.1781382858753204e-08,
"objective/entropy": 1.5396817922592163,
"objective/kl": 0.0,
"objective/rlhf_reward": -1.2192144393920898,
"objective/scores": -1.2193603515625,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 7,
"timer/active_model_forward": 1.0039308071136475,
"timer/calc_advantages": 32.610069274902344,
"timer/calc_loss": 1.0154139995574951,
"timer/generate": 29.52051544189453,
"timer/get_reward": 0.6604768633842468,
"timer/model_forward": 1.0136762857437134,
"timer/ref_model_forward": 1.0217972993850708,
"timer/training_step": 36.71036911010742,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0012675024260788625,
"grad_norm": 1.4357982536239433e-06,
"learning_rate": 0.0029987323720488037,
"loss": -0.0,
"loss/policy_avg": -7.380731403827667e-08,
"objective/entropy": 2.126983404159546,
"objective/kl": 0.0,
"objective/rlhf_reward": -1.3588581085205078,
"objective/scores": -1.35894775390625,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 8,
"timer/active_model_forward": 1.0782428979873657,
"timer/calc_advantages": 33.10707092285156,
"timer/calc_loss": 1.0924396514892578,
"timer/generate": 29.794893264770508,
"timer/get_reward": 0.7108417749404907,
"timer/model_forward": 1.0906702280044556,
"timer/ref_model_forward": 1.0941966772079468,
"timer/training_step": 37.46529769897461,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0014259402293387202,
"grad_norm": 1.58626687607466e-06,
"learning_rate": 0.002998573918554904,
"loss": -0.0,
"loss/policy_avg": -5.851325113326311e-08,
"objective/entropy": 5.8357086181640625,
"objective/kl": 0.0,
"objective/rlhf_reward": -1.3656387329101562,
"objective/scores": -1.36566162109375,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 9,
"timer/active_model_forward": 0.9783480167388916,
"timer/calc_advantages": 32.70989990234375,
"timer/calc_loss": 0.9892486333847046,
"timer/generate": 29.658161163330078,
"timer/get_reward": 0.6437245011329651,
"timer/model_forward": 0.9874842166900635,
"timer/ref_model_forward": 0.9964737892150879,
"timer/training_step": 36.70344924926758,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.001584378032598578,
"grad_norm": 1.8057213537758798e-06,
"learning_rate": 0.002998415465061005,
"loss": -0.0,
"loss/policy_avg": -4.391768015921116e-08,
"objective/entropy": 37.01044464111328,
"objective/kl": 0.0,
"objective/rlhf_reward": -1.1269941329956055,
"objective/scores": -1.126983642578125,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 10,
"timer/active_model_forward": 0.7927595973014832,
"timer/calc_advantages": 31.501224517822266,
"timer/calc_loss": 0.8029042482376099,
"timer/generate": 28.97046661376953,
"timer/get_reward": 0.5210747718811035,
"timer/model_forward": 0.8011468052864075,
"timer/ref_model_forward": 0.8100461363792419,
"timer/training_step": 34.83835983276367,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0017428158358584358,
"grad_norm": 1.3606485254058498e-06,
"learning_rate": 0.002998257011567105,
"loss": -0.0,
"loss/policy_avg": -6.2282197177410126e-09,
"objective/entropy": 82.50379943847656,
"objective/kl": 0.0,
"objective/rlhf_reward": -0.20352506637573242,
"objective/scores": -0.20351409912109375,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 11,
"timer/active_model_forward": 0.8447611331939697,
"timer/calc_advantages": 31.68113899230957,
"timer/calc_loss": 0.8556842803955078,
"timer/generate": 29.030838012695312,
"timer/get_reward": 0.5556706786155701,
"timer/model_forward": 0.853936493396759,
"timer/ref_model_forward": 0.8623838424682617,
"timer/training_step": 35.189205169677734,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0019012536391182935,
"grad_norm": 1.5168055824688054e-06,
"learning_rate": 0.0029980985580732055,
"loss": -0.0,
"loss/policy_avg": -5.324545782059431e-08,
"objective/entropy": 43.37924575805664,
"objective/kl": 0.0,
"objective/rlhf_reward": -1.273842453956604,
"objective/scores": -1.2738037109375,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 12,
"timer/active_model_forward": 0.6771436929702759,
"timer/calc_advantages": 30.566415786743164,
"timer/calc_loss": 0.6872404217720032,
"timer/generate": 28.381263732910156,
"timer/get_reward": 0.4441335201263428,
"timer/model_forward": 0.6854845881462097,
"timer/ref_model_forward": 0.6946568489074707,
"timer/training_step": 33.479454040527344,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.0020596914423781513,
"grad_norm": 9.032095817929076e-07,
"learning_rate": 0.0029979401045793063,
"loss": -0.0,
"loss/policy_avg": -5.2401446737349033e-08,
"objective/entropy": 43.8469123840332,
"objective/kl": 0.0,
"objective/rlhf_reward": -1.2306889295578003,
"objective/scores": -1.230804443359375,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 13,
"timer/active_model_forward": 0.8871802091598511,
"timer/calc_advantages": 31.831775665283203,
"timer/calc_loss": 0.8963775634765625,
"timer/generate": 29.063201904296875,
"timer/get_reward": 0.5812451839447021,
"timer/model_forward": 0.8946273326873779,
"timer/ref_model_forward": 0.9052593111991882,
"timer/training_step": 35.49396514892578,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.002218129245638009,
"grad_norm": 1.8163491404266097e-06,
"learning_rate": 0.0029977816510854067,
"loss": -0.0,
"loss/policy_avg": -6.629852578043938e-08,
"objective/entropy": 46.85642623901367,
"objective/kl": 0.0,
"objective/rlhf_reward": -1.4318056106567383,
"objective/scores": -1.431884765625,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 14,
"timer/active_model_forward": 0.7645549774169922,
"timer/calc_advantages": 31.06265640258789,
"timer/calc_loss": 0.775362491607666,
"timer/generate": 28.632740020751953,
"timer/get_reward": 0.5032914876937866,
"timer/model_forward": 0.7736347317695618,
"timer/ref_model_forward": 0.7821637988090515,
"timer/training_step": 34.282867431640625,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.002376567048897867,
"grad_norm": 1.1021771797459223e-06,
"learning_rate": 0.002997623197591507,
"loss": -0.0,
"loss/policy_avg": -4.4819898903369904e-08,
"objective/entropy": 83.53251647949219,
"objective/kl": 0.0,
"objective/rlhf_reward": -0.6715226173400879,
"objective/scores": -0.671539306640625,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 15,
"timer/active_model_forward": 1.1367318630218506,
"timer/calc_advantages": 33.59859848022461,
"timer/calc_loss": 1.2175476551055908,
"timer/generate": 30.091690063476562,
"timer/get_reward": 0.7473501563072205,
"timer/model_forward": 1.2155828475952148,
"timer/ref_model_forward": 1.1532230377197266,
"timer/training_step": 38.26678466796875,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
},
{
"epoch": 0.002535004852157725,
"grad_norm": 1.2331845482549397e-06,
"learning_rate": 0.0029974647440976073,
"loss": -0.0,
"loss/policy_avg": -1.6909325495362282e-08,
"objective/entropy": 119.47362518310547,
"objective/kl": 0.0,
"objective/rlhf_reward": -0.5125551223754883,
"objective/scores": -0.5126724243164062,
"policy/approxkl_avg": 0.0,
"policy/clipfrac_avg": 0.0,
"step": 16,
"timer/active_model_forward": 0.9046459197998047,
"timer/calc_advantages": 32.02373123168945,
"timer/calc_loss": 0.914932906627655,
"timer/generate": 29.190690994262695,
"timer/get_reward": 0.5946121215820312,
"timer/model_forward": 0.9131810665130615,
"timer/ref_model_forward": 0.922136127948761,
"timer/training_step": 35.75535202026367,
"val/num_eos_tokens": 0.0,
"val/ratio": 1.0
}
],
"logging_steps": 1,
"max_steps": 18933,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 8,
"total_flos": 8202719718014976.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
|