| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 1407, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.010666666666666666, |
| "grad_norm": 3.0879628658294678, |
| "learning_rate": 4.651162790697675e-07, |
| "logits/chosen": -1.4615767002105713, |
| "logits/rejected": -1.345485806465149, |
| "logps/chosen": -254.444580078125, |
| "logps/rejected": -294.7290954589844, |
| "loss": 0.693, |
| "rewards/accuracies": 0.33125001192092896, |
| "rewards/chosen": -0.000815639563370496, |
| "rewards/margins": 0.0002851820026990026, |
| "rewards/rejected": -0.0011008214205503464, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.021333333333333333, |
| "grad_norm": 3.0283706188201904, |
| "learning_rate": 1.0465116279069768e-06, |
| "logits/chosen": -1.4140113592147827, |
| "logits/rejected": -1.314051866531372, |
| "logps/chosen": -235.2375030517578, |
| "logps/rejected": -272.03533935546875, |
| "loss": 0.6916, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.0029421807266771793, |
| "rewards/margins": 0.0031576156616210938, |
| "rewards/rejected": -0.00609979685395956, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.032, |
| "grad_norm": 3.4339029788970947, |
| "learning_rate": 1.6279069767441862e-06, |
| "logits/chosen": -1.3767629861831665, |
| "logits/rejected": -1.2262457609176636, |
| "logps/chosen": -244.1561279296875, |
| "logps/rejected": -282.74468994140625, |
| "loss": 0.6856, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.0028314352966845036, |
| "rewards/margins": 0.015238702297210693, |
| "rewards/rejected": -0.01807013712823391, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.042666666666666665, |
| "grad_norm": 2.816466808319092, |
| "learning_rate": 2.2093023255813954e-06, |
| "logits/chosen": -1.368130087852478, |
| "logits/rejected": -1.2854630947113037, |
| "logps/chosen": -240.41079711914062, |
| "logps/rejected": -290.0007629394531, |
| "loss": 0.6771, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.005986953154206276, |
| "rewards/margins": 0.03268471360206604, |
| "rewards/rejected": -0.03867166489362717, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.05333333333333334, |
| "grad_norm": 3.2854762077331543, |
| "learning_rate": 2.790697674418605e-06, |
| "logits/chosen": -1.3610224723815918, |
| "logits/rejected": -1.2579267024993896, |
| "logps/chosen": -228.66049194335938, |
| "logps/rejected": -317.1540832519531, |
| "loss": 0.6559, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.01977255381643772, |
| "rewards/margins": 0.07735180109739304, |
| "rewards/rejected": -0.09712435305118561, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.064, |
| "grad_norm": 3.386852264404297, |
| "learning_rate": 3.372093023255814e-06, |
| "logits/chosen": -1.5247161388397217, |
| "logits/rejected": -1.3776860237121582, |
| "logps/chosen": -228.644287109375, |
| "logps/rejected": -294.73724365234375, |
| "loss": 0.6424, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.03603464737534523, |
| "rewards/margins": 0.10792265087366104, |
| "rewards/rejected": -0.14395730197429657, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07466666666666667, |
| "grad_norm": 3.72318434715271, |
| "learning_rate": 3.953488372093024e-06, |
| "logits/chosen": -1.5082950592041016, |
| "logits/rejected": -1.4147998094558716, |
| "logps/chosen": -243.84384155273438, |
| "logps/rejected": -301.76507568359375, |
| "loss": 0.6181, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.06065545603632927, |
| "rewards/margins": 0.16406485438346863, |
| "rewards/rejected": -0.2247203290462494, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.08533333333333333, |
| "grad_norm": 2.8949575424194336, |
| "learning_rate": 4.5348837209302326e-06, |
| "logits/chosen": -1.4104528427124023, |
| "logits/rejected": -1.2926199436187744, |
| "logps/chosen": -242.8151397705078, |
| "logps/rejected": -290.7886047363281, |
| "loss": 0.6012, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.08680661767721176, |
| "rewards/margins": 0.20925834774971008, |
| "rewards/rejected": -0.29606494307518005, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.096, |
| "grad_norm": 3.3057022094726562, |
| "learning_rate": 4.99999336897255e-06, |
| "logits/chosen": -1.4417957067489624, |
| "logits/rejected": -1.2875239849090576, |
| "logps/chosen": -255.4944610595703, |
| "logps/rejected": -300.8448486328125, |
| "loss": 0.5696, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.15032631158828735, |
| "rewards/margins": 0.29992568492889404, |
| "rewards/rejected": -0.4502519965171814, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.10666666666666667, |
| "grad_norm": 3.007528781890869, |
| "learning_rate": 4.999761286705285e-06, |
| "logits/chosen": -1.4402955770492554, |
| "logits/rejected": -1.3425570726394653, |
| "logps/chosen": -252.9389190673828, |
| "logps/rejected": -297.7711486816406, |
| "loss": 0.5204, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.1917630285024643, |
| "rewards/margins": 0.45220041275024414, |
| "rewards/rejected": -0.6439634561538696, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11733333333333333, |
| "grad_norm": 4.166290760040283, |
| "learning_rate": 4.999197688241076e-06, |
| "logits/chosen": -1.4534099102020264, |
| "logits/rejected": -1.3092278242111206, |
| "logps/chosen": -246.3505859375, |
| "logps/rejected": -299.24774169921875, |
| "loss": 0.4838, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.2579441964626312, |
| "rewards/margins": 0.5496667623519897, |
| "rewards/rejected": -0.8076111078262329, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.128, |
| "grad_norm": 3.144472599029541, |
| "learning_rate": 4.9983026483238685e-06, |
| "logits/chosen": -1.5001187324523926, |
| "logits/rejected": -1.3482980728149414, |
| "logps/chosen": -234.2502899169922, |
| "logps/rejected": -308.27545166015625, |
| "loss": 0.4739, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.2955595850944519, |
| "rewards/margins": 0.6387100219726562, |
| "rewards/rejected": -0.9342697262763977, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.13866666666666666, |
| "grad_norm": 3.791512966156006, |
| "learning_rate": 4.997076285653087e-06, |
| "logits/chosen": -1.572967290878296, |
| "logits/rejected": -1.4168083667755127, |
| "logps/chosen": -252.80081176757812, |
| "logps/rejected": -302.7495422363281, |
| "loss": 0.4457, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.3314843773841858, |
| "rewards/margins": 0.7851157188415527, |
| "rewards/rejected": -1.1166000366210938, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.14933333333333335, |
| "grad_norm": 2.9347736835479736, |
| "learning_rate": 4.995518762867897e-06, |
| "logits/chosen": -1.5696239471435547, |
| "logits/rejected": -1.4655719995498657, |
| "logps/chosen": -224.55050659179688, |
| "logps/rejected": -306.6766052246094, |
| "loss": 0.4296, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.4790423512458801, |
| "rewards/margins": 0.930472195148468, |
| "rewards/rejected": -1.4095146656036377, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 2.7714455127716064, |
| "learning_rate": 4.993630286525634e-06, |
| "logits/chosen": -1.5546661615371704, |
| "logits/rejected": -1.4705660343170166, |
| "logps/chosen": -223.55191040039062, |
| "logps/rejected": -324.2339172363281, |
| "loss": 0.408, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.4930631220340729, |
| "rewards/margins": 1.1179817914962769, |
| "rewards/rejected": -1.6110451221466064, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.17066666666666666, |
| "grad_norm": 2.880370616912842, |
| "learning_rate": 4.99141110707441e-06, |
| "logits/chosen": -1.5705974102020264, |
| "logits/rejected": -1.4546583890914917, |
| "logps/chosen": -251.497314453125, |
| "logps/rejected": -295.24920654296875, |
| "loss": 0.4206, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -0.48983249068260193, |
| "rewards/margins": 1.0172946453094482, |
| "rewards/rejected": -1.507127046585083, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18133333333333335, |
| "grad_norm": 2.310981273651123, |
| "learning_rate": 4.9888615188199e-06, |
| "logits/chosen": -1.649092435836792, |
| "logits/rejected": -1.4132144451141357, |
| "logps/chosen": -237.09988403320312, |
| "logps/rejected": -305.7721252441406, |
| "loss": 0.3328, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.4604329466819763, |
| "rewards/margins": 1.4129736423492432, |
| "rewards/rejected": -1.8734064102172852, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.192, |
| "grad_norm": 3.936634063720703, |
| "learning_rate": 4.9859818598863104e-06, |
| "logits/chosen": -1.5428638458251953, |
| "logits/rejected": -1.3714148998260498, |
| "logps/chosen": -253.7353973388672, |
| "logps/rejected": -307.34625244140625, |
| "loss": 0.4253, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.6021460294723511, |
| "rewards/margins": 1.2255902290344238, |
| "rewards/rejected": -1.827736496925354, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.20266666666666666, |
| "grad_norm": 2.6783416271209717, |
| "learning_rate": 4.982772512171538e-06, |
| "logits/chosen": -1.6300023794174194, |
| "logits/rejected": -1.4778879880905151, |
| "logps/chosen": -233.5361328125, |
| "logps/rejected": -318.7696838378906, |
| "loss": 0.3059, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.39014920592308044, |
| "rewards/margins": 1.6632559299468994, |
| "rewards/rejected": -2.053405284881592, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.21333333333333335, |
| "grad_norm": 2.8961541652679443, |
| "learning_rate": 4.979233901296523e-06, |
| "logits/chosen": -1.707256555557251, |
| "logits/rejected": -1.5723159313201904, |
| "logps/chosen": -225.2086181640625, |
| "logps/rejected": -306.49444580078125, |
| "loss": 0.297, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.455028235912323, |
| "rewards/margins": 1.6697375774383545, |
| "rewards/rejected": -2.124765634536743, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.224, |
| "grad_norm": 3.754955530166626, |
| "learning_rate": 4.975366496548804e-06, |
| "logits/chosen": -1.6757465600967407, |
| "logits/rejected": -1.4587806463241577, |
| "logps/chosen": -262.8829345703125, |
| "logps/rejected": -323.92864990234375, |
| "loss": 0.37, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.7448396682739258, |
| "rewards/margins": 1.6061832904815674, |
| "rewards/rejected": -2.351022720336914, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.23466666666666666, |
| "grad_norm": 2.480962038040161, |
| "learning_rate": 4.971170810820279e-06, |
| "logits/chosen": -1.6342111825942993, |
| "logits/rejected": -1.4541571140289307, |
| "logps/chosen": -237.62551879882812, |
| "logps/rejected": -295.17742919921875, |
| "loss": 0.2926, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.5155616998672485, |
| "rewards/margins": 1.8526878356933594, |
| "rewards/rejected": -2.3682494163513184, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.24533333333333332, |
| "grad_norm": 3.0674545764923096, |
| "learning_rate": 4.96664740053919e-06, |
| "logits/chosen": -1.6263113021850586, |
| "logits/rejected": -1.4940664768218994, |
| "logps/chosen": -238.77255249023438, |
| "logps/rejected": -313.794189453125, |
| "loss": 0.2864, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.5373059511184692, |
| "rewards/margins": 1.9535423517227173, |
| "rewards/rejected": -2.4908483028411865, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.256, |
| "grad_norm": 2.636179208755493, |
| "learning_rate": 4.961796865596328e-06, |
| "logits/chosen": -1.698141098022461, |
| "logits/rejected": -1.5799622535705566, |
| "logps/chosen": -229.77719116210938, |
| "logps/rejected": -291.76953125, |
| "loss": 0.3145, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.5624663233757019, |
| "rewards/margins": 1.8811826705932617, |
| "rewards/rejected": -2.4436488151550293, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.26666666666666666, |
| "grad_norm": 2.3341727256774902, |
| "learning_rate": 4.956619849265473e-06, |
| "logits/chosen": -1.6567869186401367, |
| "logits/rejected": -1.4683877229690552, |
| "logps/chosen": -228.9341583251953, |
| "logps/rejected": -299.1568603515625, |
| "loss": 0.2756, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.6057113409042358, |
| "rewards/margins": 2.1040005683898926, |
| "rewards/rejected": -2.709711790084839, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.2773333333333333, |
| "grad_norm": 2.6476287841796875, |
| "learning_rate": 4.951117038118091e-06, |
| "logits/chosen": -1.7967493534088135, |
| "logits/rejected": -1.561000108718872, |
| "logps/chosen": -254.7824249267578, |
| "logps/rejected": -323.19622802734375, |
| "loss": 0.2694, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.6340410709381104, |
| "rewards/margins": 2.2163166999816895, |
| "rewards/rejected": -2.8503577709198, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.288, |
| "grad_norm": 2.2833471298217773, |
| "learning_rate": 4.945289161932273e-06, |
| "logits/chosen": -1.771174669265747, |
| "logits/rejected": -1.6656172275543213, |
| "logps/chosen": -248.03750610351562, |
| "logps/rejected": -318.04083251953125, |
| "loss": 0.2534, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.5317661762237549, |
| "rewards/margins": 2.3246371746063232, |
| "rewards/rejected": -2.856403350830078, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.2986666666666667, |
| "grad_norm": 3.619351625442505, |
| "learning_rate": 4.93913699359596e-06, |
| "logits/chosen": -1.758262038230896, |
| "logits/rejected": -1.5464653968811035, |
| "logps/chosen": -248.1946563720703, |
| "logps/rejected": -311.05731201171875, |
| "loss": 0.2719, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.5504440069198608, |
| "rewards/margins": 2.0690348148345947, |
| "rewards/rejected": -2.619478702545166, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.30933333333333335, |
| "grad_norm": 2.925778865814209, |
| "learning_rate": 4.9326613490044364e-06, |
| "logits/chosen": -1.6672502756118774, |
| "logits/rejected": -1.4521372318267822, |
| "logps/chosen": -261.5872802734375, |
| "logps/rejected": -328.64715576171875, |
| "loss": 0.2461, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.511759877204895, |
| "rewards/margins": 2.3731701374053955, |
| "rewards/rejected": -2.88493013381958, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 2.139474391937256, |
| "learning_rate": 4.925863086952132e-06, |
| "logits/chosen": -1.646867036819458, |
| "logits/rejected": -1.548393964767456, |
| "logps/chosen": -252.48684692382812, |
| "logps/rejected": -318.29766845703125, |
| "loss": 0.2392, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.4442919194698334, |
| "rewards/margins": 2.4507598876953125, |
| "rewards/rejected": -2.895052194595337, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33066666666666666, |
| "grad_norm": 2.3797218799591064, |
| "learning_rate": 4.918743109018727e-06, |
| "logits/chosen": -1.7487760782241821, |
| "logits/rejected": -1.5734509229660034, |
| "logps/chosen": -227.19985961914062, |
| "logps/rejected": -301.46002197265625, |
| "loss": 0.2587, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.5879919528961182, |
| "rewards/margins": 2.3543381690979004, |
| "rewards/rejected": -2.9423301219940186, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.3413333333333333, |
| "grad_norm": 2.023538112640381, |
| "learning_rate": 4.911302359449585e-06, |
| "logits/chosen": -1.854344129562378, |
| "logits/rejected": -1.6243727207183838, |
| "logps/chosen": -243.7124481201172, |
| "logps/rejected": -324.5124206542969, |
| "loss": 0.2205, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5555624961853027, |
| "rewards/margins": 2.51129412651062, |
| "rewards/rejected": -3.066856861114502, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.352, |
| "grad_norm": 3.3351263999938965, |
| "learning_rate": 4.9035418250305314e-06, |
| "logits/chosen": -1.700575828552246, |
| "logits/rejected": -1.545649766921997, |
| "logps/chosen": -241.32308959960938, |
| "logps/rejected": -319.8808898925781, |
| "loss": 0.2507, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.5143498182296753, |
| "rewards/margins": 2.7291011810302734, |
| "rewards/rejected": -3.243450880050659, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.3626666666666667, |
| "grad_norm": 4.075646877288818, |
| "learning_rate": 4.895462534956981e-06, |
| "logits/chosen": -1.7546123266220093, |
| "logits/rejected": -1.6150308847427368, |
| "logps/chosen": -246.62918090820312, |
| "logps/rejected": -305.169677734375, |
| "loss": 0.2609, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.4843987822532654, |
| "rewards/margins": 2.729264736175537, |
| "rewards/rejected": -3.2136638164520264, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.37333333333333335, |
| "grad_norm": 2.9243505001068115, |
| "learning_rate": 4.887065560697449e-06, |
| "logits/chosen": -1.6826279163360596, |
| "logits/rejected": -1.5745372772216797, |
| "logps/chosen": -251.0845489501953, |
| "logps/rejected": -311.9941711425781, |
| "loss": 0.2163, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.40687593817710876, |
| "rewards/margins": 2.7896533012390137, |
| "rewards/rejected": -3.1965291500091553, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.384, |
| "grad_norm": 2.442293643951416, |
| "learning_rate": 4.878352015851459e-06, |
| "logits/chosen": -1.8285224437713623, |
| "logits/rejected": -1.5991096496582031, |
| "logps/chosen": -239.60757446289062, |
| "logps/rejected": -300.1702880859375, |
| "loss": 0.2069, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.2699597477912903, |
| "rewards/margins": 2.888909101486206, |
| "rewards/rejected": -3.1588692665100098, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39466666666666667, |
| "grad_norm": 2.2139148712158203, |
| "learning_rate": 4.869323056001852e-06, |
| "logits/chosen": -1.6878550052642822, |
| "logits/rejected": -1.532397985458374, |
| "logps/chosen": -234.5260467529297, |
| "logps/rejected": -294.7267150878906, |
| "loss": 0.2396, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.4796174466609955, |
| "rewards/margins": 2.7517666816711426, |
| "rewards/rejected": -3.231383800506592, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.4053333333333333, |
| "grad_norm": 3.6296610832214355, |
| "learning_rate": 4.859979878561536e-06, |
| "logits/chosen": -1.7104793787002563, |
| "logits/rejected": -1.550100564956665, |
| "logps/chosen": -233.6837921142578, |
| "logps/rejected": -302.03216552734375, |
| "loss": 0.2463, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.5261295437812805, |
| "rewards/margins": 2.788874387741089, |
| "rewards/rejected": -3.3150038719177246, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.416, |
| "grad_norm": 4.420226573944092, |
| "learning_rate": 4.850323722614685e-06, |
| "logits/chosen": -1.785644769668579, |
| "logits/rejected": -1.643869161605835, |
| "logps/chosen": -284.42401123046875, |
| "logps/rejected": -325.4669494628906, |
| "loss": 0.2562, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.466386616230011, |
| "rewards/margins": 3.1091275215148926, |
| "rewards/rejected": -3.575514316558838, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.4266666666666667, |
| "grad_norm": 2.722252607345581, |
| "learning_rate": 4.840355868752416e-06, |
| "logits/chosen": -1.6250245571136475, |
| "logits/rejected": -1.4949519634246826, |
| "logps/chosen": -302.90606689453125, |
| "logps/rejected": -318.97576904296875, |
| "loss": 0.276, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.5031349062919617, |
| "rewards/margins": 2.656146764755249, |
| "rewards/rejected": -3.1592819690704346, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.43733333333333335, |
| "grad_norm": 2.4931721687316895, |
| "learning_rate": 4.830077638902955e-06, |
| "logits/chosen": -1.869330644607544, |
| "logits/rejected": -1.6783950328826904, |
| "logps/chosen": -216.135986328125, |
| "logps/rejected": -323.79736328125, |
| "loss": 0.1944, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.6569713950157166, |
| "rewards/margins": 3.1940982341766357, |
| "rewards/rejected": -3.851069927215576, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.448, |
| "grad_norm": 2.207162857055664, |
| "learning_rate": 4.8194903961563255e-06, |
| "logits/chosen": -1.837034821510315, |
| "logits/rejected": -1.7211320400238037, |
| "logps/chosen": -256.60430908203125, |
| "logps/rejected": -348.6403503417969, |
| "loss": 0.1971, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.7174243927001953, |
| "rewards/margins": 3.4919838905334473, |
| "rewards/rejected": -4.209408760070801, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.45866666666666667, |
| "grad_norm": 2.2889108657836914, |
| "learning_rate": 4.808595544583578e-06, |
| "logits/chosen": -1.724928617477417, |
| "logits/rejected": -1.6084390878677368, |
| "logps/chosen": -240.2550048828125, |
| "logps/rejected": -350.73138427734375, |
| "loss": 0.1624, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.8236926198005676, |
| "rewards/margins": 3.7942657470703125, |
| "rewards/rejected": -4.617957592010498, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.4693333333333333, |
| "grad_norm": 5.622860908508301, |
| "learning_rate": 4.797394529050577e-06, |
| "logits/chosen": -1.7057397365570068, |
| "logits/rejected": -1.578086256980896, |
| "logps/chosen": -269.3270263671875, |
| "logps/rejected": -328.9361572265625, |
| "loss": 0.2316, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.7075448632240295, |
| "rewards/margins": 3.286930799484253, |
| "rewards/rejected": -3.9944756031036377, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 1.7874726057052612, |
| "learning_rate": 4.785888835026393e-06, |
| "logits/chosen": -1.8103277683258057, |
| "logits/rejected": -1.6355822086334229, |
| "logps/chosen": -248.0997314453125, |
| "logps/rejected": -305.6097106933594, |
| "loss": 0.1906, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.5874157547950745, |
| "rewards/margins": 3.4188735485076904, |
| "rewards/rejected": -4.006289482116699, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.49066666666666664, |
| "grad_norm": 4.244070053100586, |
| "learning_rate": 4.7740799883862966e-06, |
| "logits/chosen": -1.7845863103866577, |
| "logits/rejected": -1.6956688165664673, |
| "logps/chosen": -234.7997589111328, |
| "logps/rejected": -296.48455810546875, |
| "loss": 0.2286, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.7968846559524536, |
| "rewards/margins": 3.4404284954071045, |
| "rewards/rejected": -4.237313270568848, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5013333333333333, |
| "grad_norm": 1.5574147701263428, |
| "learning_rate": 4.761969555209394e-06, |
| "logits/chosen": -1.7761728763580322, |
| "logits/rejected": -1.6755540370941162, |
| "logps/chosen": -257.5173034667969, |
| "logps/rejected": -344.9666442871094, |
| "loss": 0.2145, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -1.023348093032837, |
| "rewards/margins": 3.6855106353759766, |
| "rewards/rejected": -4.708858489990234, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.512, |
| "grad_norm": 3.5682854652404785, |
| "learning_rate": 4.7495591415709435e-06, |
| "logits/chosen": -1.7692630290985107, |
| "logits/rejected": -1.662489891052246, |
| "logps/chosen": -250.2325897216797, |
| "logps/rejected": -339.6884765625, |
| "loss": 0.1866, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.8209010362625122, |
| "rewards/margins": 3.811253070831299, |
| "rewards/rejected": -4.63215446472168, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5226666666666666, |
| "grad_norm": 2.5317459106445312, |
| "learning_rate": 4.736850393329353e-06, |
| "logits/chosen": -1.8111603260040283, |
| "logits/rejected": -1.6281133890151978, |
| "logps/chosen": -264.32891845703125, |
| "logps/rejected": -339.49945068359375, |
| "loss": 0.1849, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8485740423202515, |
| "rewards/margins": 3.6573715209960938, |
| "rewards/rejected": -4.505945205688477, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5333333333333333, |
| "grad_norm": 4.821799278259277, |
| "learning_rate": 4.7238449959079095e-06, |
| "logits/chosen": -1.8489344120025635, |
| "logits/rejected": -1.737826943397522, |
| "logps/chosen": -252.2685546875, |
| "logps/rejected": -352.93438720703125, |
| "loss": 0.1943, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.8746472597122192, |
| "rewards/margins": 3.947354793548584, |
| "rewards/rejected": -4.822002410888672, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.544, |
| "grad_norm": 1.9329137802124023, |
| "learning_rate": 4.710544674071259e-06, |
| "logits/chosen": -1.7850000858306885, |
| "logits/rejected": -1.655548334121704, |
| "logps/chosen": -264.97052001953125, |
| "logps/rejected": -344.24969482421875, |
| "loss": 0.2013, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.9141705632209778, |
| "rewards/margins": 4.228996753692627, |
| "rewards/rejected": -5.143166542053223, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5546666666666666, |
| "grad_norm": 2.2834794521331787, |
| "learning_rate": 4.696951191696673e-06, |
| "logits/chosen": -1.8504130840301514, |
| "logits/rejected": -1.6865558624267578, |
| "logps/chosen": -246.1381378173828, |
| "logps/rejected": -348.99932861328125, |
| "loss": 0.1407, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8693944811820984, |
| "rewards/margins": 4.502323150634766, |
| "rewards/rejected": -5.371718406677246, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5653333333333334, |
| "grad_norm": 4.157337665557861, |
| "learning_rate": 4.6830663515401196e-06, |
| "logits/chosen": -1.8646320104599, |
| "logits/rejected": -1.66921865940094, |
| "logps/chosen": -252.67324829101562, |
| "logps/rejected": -351.05133056640625, |
| "loss": 0.1778, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.7936631441116333, |
| "rewards/margins": 4.153098106384277, |
| "rewards/rejected": -4.946761608123779, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.576, |
| "grad_norm": 1.9427365064620972, |
| "learning_rate": 4.6688919949971855e-06, |
| "logits/chosen": -1.761182188987732, |
| "logits/rejected": -1.6403383016586304, |
| "logps/chosen": -245.98727416992188, |
| "logps/rejected": -356.08294677734375, |
| "loss": 0.1804, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -1.0275193452835083, |
| "rewards/margins": 3.9113502502441406, |
| "rewards/rejected": -4.938869476318359, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5866666666666667, |
| "grad_norm": 1.9152891635894775, |
| "learning_rate": 4.654430001858874e-06, |
| "logits/chosen": -1.7100435495376587, |
| "logits/rejected": -1.6215158700942993, |
| "logps/chosen": -258.38714599609375, |
| "logps/rejected": -336.82720947265625, |
| "loss": 0.2111, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.7515047788619995, |
| "rewards/margins": 3.80024790763855, |
| "rewards/rejected": -4.551753044128418, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5973333333333334, |
| "grad_norm": 2.2597246170043945, |
| "learning_rate": 4.6396822900623075e-06, |
| "logits/chosen": -1.8039283752441406, |
| "logits/rejected": -1.66970956325531, |
| "logps/chosen": -264.4094543457031, |
| "logps/rejected": -344.8823547363281, |
| "loss": 0.1296, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8846942782402039, |
| "rewards/margins": 4.342747688293457, |
| "rewards/rejected": -5.227442741394043, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.608, |
| "grad_norm": 4.062746047973633, |
| "learning_rate": 4.624650815436368e-06, |
| "logits/chosen": -1.8472976684570312, |
| "logits/rejected": -1.7211005687713623, |
| "logps/chosen": -253.5005645751953, |
| "logps/rejected": -343.5950622558594, |
| "loss": 0.1942, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.9936168789863586, |
| "rewards/margins": 4.138308048248291, |
| "rewards/rejected": -5.131924629211426, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.6186666666666667, |
| "grad_norm": 2.604783535003662, |
| "learning_rate": 4.609337571442324e-06, |
| "logits/chosen": -1.8356084823608398, |
| "logits/rejected": -1.7241547107696533, |
| "logps/chosen": -243.8323974609375, |
| "logps/rejected": -360.04669189453125, |
| "loss": 0.1449, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.9927956461906433, |
| "rewards/margins": 4.568629264831543, |
| "rewards/rejected": -5.561424732208252, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6293333333333333, |
| "grad_norm": 1.896136999130249, |
| "learning_rate": 4.5937445889094535e-06, |
| "logits/chosen": -1.8107223510742188, |
| "logits/rejected": -1.6749213933944702, |
| "logps/chosen": -250.9614715576172, |
| "logps/rejected": -343.70001220703125, |
| "loss": 0.1625, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.9373570680618286, |
| "rewards/margins": 4.263827323913574, |
| "rewards/rejected": -5.201184272766113, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 4.003625392913818, |
| "learning_rate": 4.577873935765722e-06, |
| "logits/chosen": -1.9519007205963135, |
| "logits/rejected": -1.7255446910858154, |
| "logps/chosen": -245.7286376953125, |
| "logps/rejected": -343.4656066894531, |
| "loss": 0.1946, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -1.0532045364379883, |
| "rewards/margins": 4.6587090492248535, |
| "rewards/rejected": -5.711912631988525, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6506666666666666, |
| "grad_norm": 2.2551584243774414, |
| "learning_rate": 4.561727716763532e-06, |
| "logits/chosen": -1.8950430154800415, |
| "logits/rejected": -1.746446967124939, |
| "logps/chosen": -257.13311767578125, |
| "logps/rejected": -330.2293701171875, |
| "loss": 0.2094, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.9331668615341187, |
| "rewards/margins": 3.9878017902374268, |
| "rewards/rejected": -4.920968532562256, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.6613333333333333, |
| "grad_norm": 2.4485409259796143, |
| "learning_rate": 4.545308073200591e-06, |
| "logits/chosen": -1.805546522140503, |
| "logits/rejected": -1.6681687831878662, |
| "logps/chosen": -247.18472290039062, |
| "logps/rejected": -348.66400146484375, |
| "loss": 0.2251, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.8696174621582031, |
| "rewards/margins": 4.181191921234131, |
| "rewards/rejected": -5.050809860229492, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.672, |
| "grad_norm": 4.527316570281982, |
| "learning_rate": 4.528617182635943e-06, |
| "logits/chosen": -1.8156181573867798, |
| "logits/rejected": -1.715304970741272, |
| "logps/chosen": -260.9232177734375, |
| "logps/rejected": -349.6746520996094, |
| "loss": 0.1582, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.2807674407958984, |
| "rewards/margins": 4.808705806732178, |
| "rewards/rejected": -6.089472770690918, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6826666666666666, |
| "grad_norm": 2.6639559268951416, |
| "learning_rate": 4.511657258601171e-06, |
| "logits/chosen": -1.814720869064331, |
| "logits/rejected": -1.5974810123443604, |
| "logps/chosen": -273.97039794921875, |
| "logps/rejected": -347.71075439453125, |
| "loss": 0.1918, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.1705496311187744, |
| "rewards/margins": 4.306663990020752, |
| "rewards/rejected": -5.4772138595581055, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6933333333333334, |
| "grad_norm": 3.535733699798584, |
| "learning_rate": 4.494430550306848e-06, |
| "logits/chosen": -1.6838403940200806, |
| "logits/rejected": -1.6364444494247437, |
| "logps/chosen": -251.2472686767578, |
| "logps/rejected": -342.08489990234375, |
| "loss": 0.2131, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.7756344079971313, |
| "rewards/margins": 4.671334266662598, |
| "rewards/rejected": -5.446969032287598, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.704, |
| "grad_norm": 4.793890476226807, |
| "learning_rate": 4.476939342344246e-06, |
| "logits/chosen": -1.9221420288085938, |
| "logits/rejected": -1.7676732540130615, |
| "logps/chosen": -247.0741424560547, |
| "logps/rejected": -332.93292236328125, |
| "loss": 0.1601, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.9986921548843384, |
| "rewards/margins": 4.544274806976318, |
| "rewards/rejected": -5.542966842651367, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7146666666666667, |
| "grad_norm": 6.12879753112793, |
| "learning_rate": 4.45918595438236e-06, |
| "logits/chosen": -1.7161098718643188, |
| "logits/rejected": -1.6740608215332031, |
| "logps/chosen": -261.7930908203125, |
| "logps/rejected": -353.0909118652344, |
| "loss": 0.1974, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.0057028532028198, |
| "rewards/margins": 4.220341205596924, |
| "rewards/rejected": -5.226044178009033, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7253333333333334, |
| "grad_norm": 3.4764912128448486, |
| "learning_rate": 4.441172740860267e-06, |
| "logits/chosen": -1.812603235244751, |
| "logits/rejected": -1.6671600341796875, |
| "logps/chosen": -256.25775146484375, |
| "logps/rejected": -343.37750244140625, |
| "loss": 0.1811, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.1001125574111938, |
| "rewards/margins": 4.227419853210449, |
| "rewards/rejected": -5.327532768249512, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.736, |
| "grad_norm": 3.187913656234741, |
| "learning_rate": 4.422902090674886e-06, |
| "logits/chosen": -1.6854442358016968, |
| "logits/rejected": -1.6130621433258057, |
| "logps/chosen": -248.1710662841797, |
| "logps/rejected": -364.08953857421875, |
| "loss": 0.1584, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8657025098800659, |
| "rewards/margins": 4.714477062225342, |
| "rewards/rejected": -5.580178737640381, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.7466666666666667, |
| "grad_norm": 2.092165470123291, |
| "learning_rate": 4.4043764268641705e-06, |
| "logits/chosen": -1.7814304828643799, |
| "logits/rejected": -1.6742616891860962, |
| "logps/chosen": -243.86672973632812, |
| "logps/rejected": -343.56060791015625, |
| "loss": 0.183, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.8583084940910339, |
| "rewards/margins": 4.422949314117432, |
| "rewards/rejected": -5.2812581062316895, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7573333333333333, |
| "grad_norm": 2.406649351119995, |
| "learning_rate": 4.385598206285758e-06, |
| "logits/chosen": -1.8445268869400024, |
| "logits/rejected": -1.7397022247314453, |
| "logps/chosen": -274.53399658203125, |
| "logps/rejected": -365.04168701171875, |
| "loss": 0.1659, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.0924501419067383, |
| "rewards/margins": 4.701749324798584, |
| "rewards/rejected": -5.794199466705322, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.768, |
| "grad_norm": 3.1149144172668457, |
| "learning_rate": 4.3665699192911495e-06, |
| "logits/chosen": -1.784285545349121, |
| "logits/rejected": -1.664687156677246, |
| "logps/chosen": -251.3987579345703, |
| "logps/rejected": -350.62103271484375, |
| "loss": 0.1909, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.1020582914352417, |
| "rewards/margins": 4.7440361976623535, |
| "rewards/rejected": -5.846094131469727, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7786666666666666, |
| "grad_norm": 3.1475017070770264, |
| "learning_rate": 4.347294089395438e-06, |
| "logits/chosen": -1.75801682472229, |
| "logits/rejected": -1.629412293434143, |
| "logps/chosen": -267.57562255859375, |
| "logps/rejected": -339.25567626953125, |
| "loss": 0.1784, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.2324341535568237, |
| "rewards/margins": 4.7525458335876465, |
| "rewards/rejected": -5.984980583190918, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7893333333333333, |
| "grad_norm": 5.428934097290039, |
| "learning_rate": 4.327773272942647e-06, |
| "logits/chosen": -1.7988414764404297, |
| "logits/rejected": -1.610979437828064, |
| "logps/chosen": -254.54312133789062, |
| "logps/rejected": -333.9504699707031, |
| "loss": 0.1686, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.1293680667877197, |
| "rewards/margins": 5.0215744972229, |
| "rewards/rejected": -6.150942802429199, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 3.3105201721191406, |
| "learning_rate": 4.308010058766703e-06, |
| "logits/chosen": -1.7741140127182007, |
| "logits/rejected": -1.7098277807235718, |
| "logps/chosen": -270.8016357421875, |
| "logps/rejected": -379.1322326660156, |
| "loss": 0.1397, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.3129689693450928, |
| "rewards/margins": 4.9763569831848145, |
| "rewards/rejected": -6.289326190948486, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.8106666666666666, |
| "grad_norm": 2.7516579627990723, |
| "learning_rate": 4.2880070678481135e-06, |
| "logits/chosen": -1.8347715139389038, |
| "logits/rejected": -1.6875629425048828, |
| "logps/chosen": -234.46694946289062, |
| "logps/rejected": -346.2992248535156, |
| "loss": 0.1281, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.0527235269546509, |
| "rewards/margins": 5.5215559005737305, |
| "rewards/rejected": -6.57427978515625, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8213333333333334, |
| "grad_norm": 3.6830787658691406, |
| "learning_rate": 4.267766952966369e-06, |
| "logits/chosen": -1.76150643825531, |
| "logits/rejected": -1.695257544517517, |
| "logps/chosen": -247.07595825195312, |
| "logps/rejected": -348.04656982421875, |
| "loss": 0.1845, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.2883862257003784, |
| "rewards/margins": 4.600207328796387, |
| "rewards/rejected": -5.8885931968688965, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.832, |
| "grad_norm": 4.31989860534668, |
| "learning_rate": 4.24729239834814e-06, |
| "logits/chosen": -1.7455536127090454, |
| "logits/rejected": -1.6559183597564697, |
| "logps/chosen": -253.7515869140625, |
| "logps/rejected": -348.94537353515625, |
| "loss": 0.21, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.2032108306884766, |
| "rewards/margins": 4.9352593421936035, |
| "rewards/rejected": -6.138470649719238, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.8426666666666667, |
| "grad_norm": 4.060668468475342, |
| "learning_rate": 4.226586119311291e-06, |
| "logits/chosen": -1.775027871131897, |
| "logits/rejected": -1.7871853113174438, |
| "logps/chosen": -264.8758239746094, |
| "logps/rejected": -360.35614013671875, |
| "loss": 0.1818, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.096254825592041, |
| "rewards/margins": 5.317919731140137, |
| "rewards/rejected": -6.414175510406494, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8533333333333334, |
| "grad_norm": 3.2177236080169678, |
| "learning_rate": 4.205650861904782e-06, |
| "logits/chosen": -1.8364406824111938, |
| "logits/rejected": -1.6941852569580078, |
| "logps/chosen": -226.8248291015625, |
| "logps/rejected": -345.60479736328125, |
| "loss": 0.1841, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -1.129191517829895, |
| "rewards/margins": 4.975957870483398, |
| "rewards/rejected": -6.105149269104004, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.864, |
| "grad_norm": 3.1651129722595215, |
| "learning_rate": 4.184489402544486e-06, |
| "logits/chosen": -1.8180274963378906, |
| "logits/rejected": -1.723665475845337, |
| "logps/chosen": -229.318115234375, |
| "logps/rejected": -314.98675537109375, |
| "loss": 0.2303, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -1.1190117597579956, |
| "rewards/margins": 4.453101634979248, |
| "rewards/rejected": -5.572113037109375, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.8746666666666667, |
| "grad_norm": 3.7597198486328125, |
| "learning_rate": 4.1631045476449865e-06, |
| "logits/chosen": -1.9217109680175781, |
| "logits/rejected": -1.765979528427124, |
| "logps/chosen": -237.1017303466797, |
| "logps/rejected": -344.9607849121094, |
| "loss": 0.1314, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8680995106697083, |
| "rewards/margins": 5.295489311218262, |
| "rewards/rejected": -6.163588523864746, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8853333333333333, |
| "grad_norm": 1.5503333806991577, |
| "learning_rate": 4.141499133247391e-06, |
| "logits/chosen": -1.674448013305664, |
| "logits/rejected": -1.6276872158050537, |
| "logps/chosen": -268.17816162109375, |
| "logps/rejected": -357.49810791015625, |
| "loss": 0.1353, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.7698339223861694, |
| "rewards/margins": 5.1189494132995605, |
| "rewards/rejected": -5.8887834548950195, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.896, |
| "grad_norm": 3.54842472076416, |
| "learning_rate": 4.119676024643217e-06, |
| "logits/chosen": -1.6974074840545654, |
| "logits/rejected": -1.6472240686416626, |
| "logps/chosen": -257.77655029296875, |
| "logps/rejected": -349.54205322265625, |
| "loss": 0.1515, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.0973252058029175, |
| "rewards/margins": 5.101239204406738, |
| "rewards/rejected": -6.198564529418945, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.9066666666666666, |
| "grad_norm": 5.144434452056885, |
| "learning_rate": 4.0976381159944055e-06, |
| "logits/chosen": -1.803943395614624, |
| "logits/rejected": -1.6611639261245728, |
| "logps/chosen": -271.3909912109375, |
| "logps/rejected": -372.34234619140625, |
| "loss": 0.1513, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.0220460891723633, |
| "rewards/margins": 5.32880163192749, |
| "rewards/rejected": -6.350848197937012, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.9173333333333333, |
| "grad_norm": 2.2385873794555664, |
| "learning_rate": 4.07538832994949e-06, |
| "logits/chosen": -1.7902886867523193, |
| "logits/rejected": -1.5918269157409668, |
| "logps/chosen": -251.48599243164062, |
| "logps/rejected": -345.81292724609375, |
| "loss": 0.1848, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -1.0862038135528564, |
| "rewards/margins": 5.2021284103393555, |
| "rewards/rejected": -6.288332462310791, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.928, |
| "grad_norm": 4.186896324157715, |
| "learning_rate": 4.0529296172560055e-06, |
| "logits/chosen": -1.8904718160629272, |
| "logits/rejected": -1.6817710399627686, |
| "logps/chosen": -238.9318389892578, |
| "logps/rejected": -353.1915588378906, |
| "loss": 0.1555, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.286082148551941, |
| "rewards/margins": 5.383737087249756, |
| "rewards/rejected": -6.6698198318481445, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9386666666666666, |
| "grad_norm": 2.397501230239868, |
| "learning_rate": 4.030264956369158e-06, |
| "logits/chosen": -1.8631141185760498, |
| "logits/rejected": -1.6818256378173828, |
| "logps/chosen": -233.68563842773438, |
| "logps/rejected": -355.0934143066406, |
| "loss": 0.1211, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.9764868021011353, |
| "rewards/margins": 5.496334075927734, |
| "rewards/rejected": -6.472821235656738, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9493333333333334, |
| "grad_norm": 6.384281635284424, |
| "learning_rate": 4.0073973530568274e-06, |
| "logits/chosen": -1.823751449584961, |
| "logits/rejected": -1.6468112468719482, |
| "logps/chosen": -261.18084716796875, |
| "logps/rejected": -354.4629211425781, |
| "loss": 0.2524, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -1.3924033641815186, |
| "rewards/margins": 5.070466995239258, |
| "rewards/rejected": -6.462870121002197, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 3.978393077850342, |
| "learning_rate": 3.9843298400009435e-06, |
| "logits/chosen": -1.8724792003631592, |
| "logits/rejected": -1.7450891733169556, |
| "logps/chosen": -235.0048065185547, |
| "logps/rejected": -355.290771484375, |
| "loss": 0.1176, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.1844699382781982, |
| "rewards/margins": 5.504419326782227, |
| "rewards/rejected": -6.688889503479004, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9706666666666667, |
| "grad_norm": 2.8582115173339844, |
| "learning_rate": 3.961065476395296e-06, |
| "logits/chosen": -1.8186849355697632, |
| "logits/rejected": -1.6841886043548584, |
| "logps/chosen": -266.6614990234375, |
| "logps/rejected": -351.87646484375, |
| "loss": 0.1637, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.1914622783660889, |
| "rewards/margins": 5.31570291519165, |
| "rewards/rejected": -6.507164001464844, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9813333333333333, |
| "grad_norm": 2.4496090412139893, |
| "learning_rate": 3.937607347539823e-06, |
| "logits/chosen": -1.772597074508667, |
| "logits/rejected": -1.6899995803833008, |
| "logps/chosen": -258.0954895019531, |
| "logps/rejected": -382.6227722167969, |
| "loss": 0.1429, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.5825755596160889, |
| "rewards/margins": 5.6097493171691895, |
| "rewards/rejected": -7.192324638366699, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.992, |
| "grad_norm": 1.6265991926193237, |
| "learning_rate": 3.913958564431447e-06, |
| "logits/chosen": -1.9242236614227295, |
| "logits/rejected": -1.760114312171936, |
| "logps/chosen": -239.49014282226562, |
| "logps/rejected": -336.96502685546875, |
| "loss": 0.1317, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.038116693496704, |
| "rewards/margins": 5.815273761749268, |
| "rewards/rejected": -6.853390693664551, |
| "step": 465 |
| }, |
| { |
| "epoch": 1.0021333333333333, |
| "grad_norm": 2.376556396484375, |
| "learning_rate": 3.89012226335149e-06, |
| "logits/chosen": -1.7828489542007446, |
| "logits/rejected": -1.676961064338684, |
| "logps/chosen": -239.7880859375, |
| "logps/rejected": -330.6512145996094, |
| "loss": 0.2273, |
| "rewards/accuracies": 0.9013158082962036, |
| "rewards/chosen": -1.1525920629501343, |
| "rewards/margins": 4.860694408416748, |
| "rewards/rejected": -6.013286113739014, |
| "step": 470 |
| }, |
| { |
| "epoch": 1.0128, |
| "grad_norm": 3.2269089221954346, |
| "learning_rate": 3.86610160544975e-06, |
| "logits/chosen": -1.8129554986953735, |
| "logits/rejected": -1.6979223489761353, |
| "logps/chosen": -257.1238098144531, |
| "logps/rejected": -371.0331726074219, |
| "loss": 0.1038, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.2004741430282593, |
| "rewards/margins": 5.8469557762146, |
| "rewards/rejected": -7.047430515289307, |
| "step": 475 |
| }, |
| { |
| "epoch": 1.0234666666666667, |
| "grad_norm": 3.285102605819702, |
| "learning_rate": 3.841899776325267e-06, |
| "logits/chosen": -1.6993519067764282, |
| "logits/rejected": -1.5625213384628296, |
| "logps/chosen": -264.1551818847656, |
| "logps/rejected": -342.95697021484375, |
| "loss": 0.143, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.0834065675735474, |
| "rewards/margins": 5.366703987121582, |
| "rewards/rejected": -6.450111389160156, |
| "step": 480 |
| }, |
| { |
| "epoch": 1.0341333333333333, |
| "grad_norm": 2.0090343952178955, |
| "learning_rate": 3.817519985603855e-06, |
| "logits/chosen": -1.8892101049423218, |
| "logits/rejected": -1.7172794342041016, |
| "logps/chosen": -235.93984985351562, |
| "logps/rejected": -370.1104736328125, |
| "loss": 0.0987, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.0467970371246338, |
| "rewards/margins": 5.996671199798584, |
| "rewards/rejected": -7.0434675216674805, |
| "step": 485 |
| }, |
| { |
| "epoch": 1.0448, |
| "grad_norm": 1.3430184125900269, |
| "learning_rate": 3.792965466512437e-06, |
| "logits/chosen": -1.7806406021118164, |
| "logits/rejected": -1.664324402809143, |
| "logps/chosen": -261.94598388671875, |
| "logps/rejected": -347.97021484375, |
| "loss": 0.1651, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.9251692891120911, |
| "rewards/margins": 5.451348304748535, |
| "rewards/rejected": -6.376518249511719, |
| "step": 490 |
| }, |
| { |
| "epoch": 1.0554666666666668, |
| "grad_norm": 3.2143454551696777, |
| "learning_rate": 3.7682394754502687e-06, |
| "logits/chosen": -1.9018503427505493, |
| "logits/rejected": -1.776320457458496, |
| "logps/chosen": -250.692138671875, |
| "logps/rejected": -362.97637939453125, |
| "loss": 0.1558, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.189073920249939, |
| "rewards/margins": 5.650210380554199, |
| "rewards/rejected": -6.839284420013428, |
| "step": 495 |
| }, |
| { |
| "epoch": 1.0661333333333334, |
| "grad_norm": 2.513195276260376, |
| "learning_rate": 3.7433452915570685e-06, |
| "logits/chosen": -1.8409998416900635, |
| "logits/rejected": -1.7353975772857666, |
| "logps/chosen": -232.98306274414062, |
| "logps/rejected": -346.6318664550781, |
| "loss": 0.1149, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.9560970067977905, |
| "rewards/margins": 5.840074062347412, |
| "rewards/rejected": -6.796170711517334, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.0768, |
| "grad_norm": 2.5323407649993896, |
| "learning_rate": 3.718286216278143e-06, |
| "logits/chosen": -1.850425124168396, |
| "logits/rejected": -1.6678615808486938, |
| "logps/chosen": -254.27713012695312, |
| "logps/rejected": -357.68731689453125, |
| "loss": 0.1227, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.3727056980133057, |
| "rewards/margins": 5.668168067932129, |
| "rewards/rejected": -7.0408735275268555, |
| "step": 505 |
| }, |
| { |
| "epoch": 1.0874666666666666, |
| "grad_norm": 4.070751667022705, |
| "learning_rate": 3.6930655729265557e-06, |
| "logits/chosen": -1.8335998058319092, |
| "logits/rejected": -1.613699197769165, |
| "logps/chosen": -284.2073974609375, |
| "logps/rejected": -358.90069580078125, |
| "loss": 0.122, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.4791011810302734, |
| "rewards/margins": 5.614500522613525, |
| "rewards/rejected": -7.093601226806641, |
| "step": 510 |
| }, |
| { |
| "epoch": 1.0981333333333334, |
| "grad_norm": 2.953993558883667, |
| "learning_rate": 3.667686706242386e-06, |
| "logits/chosen": -1.7201465368270874, |
| "logits/rejected": -1.6716951131820679, |
| "logps/chosen": -245.04855346679688, |
| "logps/rejected": -360.8467102050781, |
| "loss": 0.1177, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.494508981704712, |
| "rewards/margins": 5.6371870040893555, |
| "rewards/rejected": -7.131695747375488, |
| "step": 515 |
| }, |
| { |
| "epoch": 1.1088, |
| "grad_norm": 4.227024078369141, |
| "learning_rate": 3.6421529819491576e-06, |
| "logits/chosen": -1.8493587970733643, |
| "logits/rejected": -1.7108128070831299, |
| "logps/chosen": -256.4563903808594, |
| "logps/rejected": -361.6827087402344, |
| "loss": 0.1113, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.465549111366272, |
| "rewards/margins": 6.157153129577637, |
| "rewards/rejected": -7.622702121734619, |
| "step": 520 |
| }, |
| { |
| "epoch": 1.1194666666666666, |
| "grad_norm": 4.163751125335693, |
| "learning_rate": 3.6164677863074783e-06, |
| "logits/chosen": -1.834673523902893, |
| "logits/rejected": -1.723134994506836, |
| "logps/chosen": -245.7028350830078, |
| "logps/rejected": -347.9207763671875, |
| "loss": 0.1446, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.819270372390747, |
| "rewards/margins": 5.923323154449463, |
| "rewards/rejected": -7.742593288421631, |
| "step": 525 |
| }, |
| { |
| "epoch": 1.1301333333333332, |
| "grad_norm": 2.243851661682129, |
| "learning_rate": 3.590634525665955e-06, |
| "logits/chosen": -1.9202182292938232, |
| "logits/rejected": -1.7526201009750366, |
| "logps/chosen": -237.74996948242188, |
| "logps/rejected": -365.3797302246094, |
| "loss": 0.1187, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.5946757793426514, |
| "rewards/margins": 6.295979022979736, |
| "rewards/rejected": -7.890654563903809, |
| "step": 530 |
| }, |
| { |
| "epoch": 1.1408, |
| "grad_norm": 1.7002406120300293, |
| "learning_rate": 3.564656626009446e-06, |
| "logits/chosen": -1.9501392841339111, |
| "logits/rejected": -1.775882363319397, |
| "logps/chosen": -249.8088836669922, |
| "logps/rejected": -361.65216064453125, |
| "loss": 0.1579, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.8825603723526, |
| "rewards/margins": 6.139547824859619, |
| "rewards/rejected": -8.02210807800293, |
| "step": 535 |
| }, |
| { |
| "epoch": 1.1514666666666666, |
| "grad_norm": 2.868656873703003, |
| "learning_rate": 3.5385375325047167e-06, |
| "logits/chosen": -1.900363564491272, |
| "logits/rejected": -1.7623260021209717, |
| "logps/chosen": -258.2088317871094, |
| "logps/rejected": -369.24456787109375, |
| "loss": 0.1002, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8428981304168701, |
| "rewards/margins": 6.456591606140137, |
| "rewards/rejected": -8.299489974975586, |
| "step": 540 |
| }, |
| { |
| "epoch": 1.1621333333333332, |
| "grad_norm": 4.660250186920166, |
| "learning_rate": 3.512280709043534e-06, |
| "logits/chosen": -1.8281036615371704, |
| "logits/rejected": -1.7140634059906006, |
| "logps/chosen": -260.32257080078125, |
| "logps/rejected": -350.28045654296875, |
| "loss": 0.161, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.2407617568969727, |
| "rewards/margins": 5.985723972320557, |
| "rewards/rejected": -7.226485252380371, |
| "step": 545 |
| }, |
| { |
| "epoch": 1.1728, |
| "grad_norm": 2.281893014907837, |
| "learning_rate": 3.4858896377832966e-06, |
| "logits/chosen": -1.729958176612854, |
| "logits/rejected": -1.7020686864852905, |
| "logps/chosen": -252.7555389404297, |
| "logps/rejected": -387.8126525878906, |
| "loss": 0.098, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.817659616470337, |
| "rewards/margins": 6.727684020996094, |
| "rewards/rejected": -8.545343399047852, |
| "step": 550 |
| }, |
| { |
| "epoch": 1.1834666666666667, |
| "grad_norm": 2.4751644134521484, |
| "learning_rate": 3.459367818685231e-06, |
| "logits/chosen": -1.7661396265029907, |
| "logits/rejected": -1.7389978170394897, |
| "logps/chosen": -279.11309814453125, |
| "logps/rejected": -386.31610107421875, |
| "loss": 0.0959, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7105395793914795, |
| "rewards/margins": 6.346243381500244, |
| "rewards/rejected": -8.056783676147461, |
| "step": 555 |
| }, |
| { |
| "epoch": 1.1941333333333333, |
| "grad_norm": 1.502102255821228, |
| "learning_rate": 3.432718769050228e-06, |
| "logits/chosen": -1.709663987159729, |
| "logits/rejected": -1.6350157260894775, |
| "logps/chosen": -248.32553100585938, |
| "logps/rejected": -371.6622009277344, |
| "loss": 0.0819, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.5119163990020752, |
| "rewards/margins": 6.394644737243652, |
| "rewards/rejected": -7.906561374664307, |
| "step": 560 |
| }, |
| { |
| "epoch": 1.2048, |
| "grad_norm": 2.423750162124634, |
| "learning_rate": 3.405946023052386e-06, |
| "logits/chosen": -1.772951364517212, |
| "logits/rejected": -1.7209113836288452, |
| "logps/chosen": -244.55960083007812, |
| "logps/rejected": -361.45416259765625, |
| "loss": 0.1398, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.6022535562515259, |
| "rewards/margins": 6.2257208824157715, |
| "rewards/rejected": -7.82797384262085, |
| "step": 565 |
| }, |
| { |
| "epoch": 1.2154666666666667, |
| "grad_norm": 5.750537395477295, |
| "learning_rate": 3.3790531312703055e-06, |
| "logits/chosen": -1.8179404735565186, |
| "logits/rejected": -1.689639687538147, |
| "logps/chosen": -268.6795959472656, |
| "logps/rejected": -380.632080078125, |
| "loss": 0.1195, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.895990014076233, |
| "rewards/margins": 6.5666608810424805, |
| "rewards/rejected": -8.462652206420898, |
| "step": 570 |
| }, |
| { |
| "epoch": 1.2261333333333333, |
| "grad_norm": 2.973874092102051, |
| "learning_rate": 3.35204366021622e-06, |
| "logits/chosen": -1.7112888097763062, |
| "logits/rejected": -1.6027559041976929, |
| "logps/chosen": -248.86685180664062, |
| "logps/rejected": -377.16497802734375, |
| "loss": 0.1024, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.2404453754425049, |
| "rewards/margins": 6.802675724029541, |
| "rewards/rejected": -8.043119430541992, |
| "step": 575 |
| }, |
| { |
| "epoch": 1.2368000000000001, |
| "grad_norm": 3.057502508163452, |
| "learning_rate": 3.3249211918630055e-06, |
| "logits/chosen": -1.7631298303604126, |
| "logits/rejected": -1.6719868183135986, |
| "logps/chosen": -265.61187744140625, |
| "logps/rejected": -348.72979736328125, |
| "loss": 0.1675, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.6381809711456299, |
| "rewards/margins": 5.736741065979004, |
| "rewards/rejected": -7.374920845031738, |
| "step": 580 |
| }, |
| { |
| "epoch": 1.2474666666666667, |
| "grad_norm": 3.961693525314331, |
| "learning_rate": 3.2976893231691424e-06, |
| "logits/chosen": -1.7621526718139648, |
| "logits/rejected": -1.6848115921020508, |
| "logps/chosen": -258.19683837890625, |
| "logps/rejected": -355.4867858886719, |
| "loss": 0.1452, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.344029188156128, |
| "rewards/margins": 5.742987632751465, |
| "rewards/rejected": -7.087018013000488, |
| "step": 585 |
| }, |
| { |
| "epoch": 1.2581333333333333, |
| "grad_norm": 4.668637752532959, |
| "learning_rate": 3.270351665601691e-06, |
| "logits/chosen": -1.7807848453521729, |
| "logits/rejected": -1.6941245794296265, |
| "logps/chosen": -263.23193359375, |
| "logps/rejected": -364.79876708984375, |
| "loss": 0.1789, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.6055269241333008, |
| "rewards/margins": 5.881946086883545, |
| "rewards/rejected": -7.4874725341796875, |
| "step": 590 |
| }, |
| { |
| "epoch": 1.2688, |
| "grad_norm": 1.626665472984314, |
| "learning_rate": 3.2429118446573417e-06, |
| "logits/chosen": -1.7861591577529907, |
| "logits/rejected": -1.6627706289291382, |
| "logps/chosen": -242.6871795654297, |
| "logps/rejected": -369.5167541503906, |
| "loss": 0.124, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.4150798320770264, |
| "rewards/margins": 6.047471523284912, |
| "rewards/rejected": -7.462552070617676, |
| "step": 595 |
| }, |
| { |
| "epoch": 1.2794666666666665, |
| "grad_norm": 1.1785833835601807, |
| "learning_rate": 3.215373499381602e-06, |
| "logits/chosen": -1.7399671077728271, |
| "logits/rejected": -1.6407349109649658, |
| "logps/chosen": -253.98306274414062, |
| "logps/rejected": -362.9200439453125, |
| "loss": 0.1039, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.304102897644043, |
| "rewards/margins": 6.121998310089111, |
| "rewards/rejected": -7.426100730895996, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.2901333333333334, |
| "grad_norm": 2.841127395629883, |
| "learning_rate": 3.1877402818861954e-06, |
| "logits/chosen": -1.7944116592407227, |
| "logits/rejected": -1.6870648860931396, |
| "logps/chosen": -275.91058349609375, |
| "logps/rejected": -376.3899230957031, |
| "loss": 0.0709, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.7364799976348877, |
| "rewards/margins": 6.606210231781006, |
| "rewards/rejected": -8.342691421508789, |
| "step": 605 |
| }, |
| { |
| "epoch": 1.3008, |
| "grad_norm": 4.171568393707275, |
| "learning_rate": 3.1600158568647145e-06, |
| "logits/chosen": -1.7378740310668945, |
| "logits/rejected": -1.6512172222137451, |
| "logps/chosen": -256.87786865234375, |
| "logps/rejected": -380.0071105957031, |
| "loss": 0.0842, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.5280927419662476, |
| "rewards/margins": 6.7154693603515625, |
| "rewards/rejected": -8.243562698364258, |
| "step": 610 |
| }, |
| { |
| "epoch": 1.3114666666666666, |
| "grad_norm": 1.7711259126663208, |
| "learning_rate": 3.1322039011066172e-06, |
| "logits/chosen": -1.7013614177703857, |
| "logits/rejected": -1.6217601299285889, |
| "logps/chosen": -290.1616516113281, |
| "logps/rejected": -374.9538269042969, |
| "loss": 0.1086, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6339313983917236, |
| "rewards/margins": 7.052574157714844, |
| "rewards/rejected": -8.686506271362305, |
| "step": 615 |
| }, |
| { |
| "epoch": 1.3221333333333334, |
| "grad_norm": 1.6588112115859985, |
| "learning_rate": 3.1043081030096125e-06, |
| "logits/chosen": -1.6777381896972656, |
| "logits/rejected": -1.5866796970367432, |
| "logps/chosen": -251.49923706054688, |
| "logps/rejected": -379.4061584472656, |
| "loss": 0.0912, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.4971685409545898, |
| "rewards/margins": 7.028067111968994, |
| "rewards/rejected": -8.525236129760742, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.3328, |
| "grad_norm": 1.202467441558838, |
| "learning_rate": 3.0763321620905072e-06, |
| "logits/chosen": -1.757503867149353, |
| "logits/rejected": -1.6371698379516602, |
| "logps/chosen": -273.4452209472656, |
| "logps/rejected": -375.41668701171875, |
| "loss": 0.108, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.6967731714248657, |
| "rewards/margins": 6.615039825439453, |
| "rewards/rejected": -8.311813354492188, |
| "step": 625 |
| }, |
| { |
| "epoch": 1.3434666666666666, |
| "grad_norm": 2.2688076496124268, |
| "learning_rate": 3.048279788494583e-06, |
| "logits/chosen": -1.785783052444458, |
| "logits/rejected": -1.7002527713775635, |
| "logps/chosen": -257.37322998046875, |
| "logps/rejected": -377.8409423828125, |
| "loss": 0.1454, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9974949359893799, |
| "rewards/margins": 6.213410377502441, |
| "rewards/rejected": -8.210905075073242, |
| "step": 630 |
| }, |
| { |
| "epoch": 1.3541333333333334, |
| "grad_norm": 2.4637012481689453, |
| "learning_rate": 3.0201547025035553e-06, |
| "logits/chosen": -1.751521110534668, |
| "logits/rejected": -1.6897504329681396, |
| "logps/chosen": -254.302978515625, |
| "logps/rejected": -378.50323486328125, |
| "loss": 0.1022, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6483898162841797, |
| "rewards/margins": 6.3102569580078125, |
| "rewards/rejected": -7.958646297454834, |
| "step": 635 |
| }, |
| { |
| "epoch": 1.3648, |
| "grad_norm": 3.8461568355560303, |
| "learning_rate": 2.9919606340421963e-06, |
| "logits/chosen": -1.7511999607086182, |
| "logits/rejected": -1.624850869178772, |
| "logps/chosen": -235.1289825439453, |
| "logps/rejected": -380.01055908203125, |
| "loss": 0.0942, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.9077924489974976, |
| "rewards/margins": 6.974759101867676, |
| "rewards/rejected": -8.882550239562988, |
| "step": 640 |
| }, |
| { |
| "epoch": 1.3754666666666666, |
| "grad_norm": 3.0227408409118652, |
| "learning_rate": 2.9637013221836723e-06, |
| "logits/chosen": -1.8264532089233398, |
| "logits/rejected": -1.678780198097229, |
| "logps/chosen": -258.0480651855469, |
| "logps/rejected": -377.66595458984375, |
| "loss": 0.129, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6189113855361938, |
| "rewards/margins": 6.719698429107666, |
| "rewards/rejected": -8.33860969543457, |
| "step": 645 |
| }, |
| { |
| "epoch": 1.3861333333333334, |
| "grad_norm": 2.1811294555664062, |
| "learning_rate": 2.9353805146536762e-06, |
| "logits/chosen": -1.7129695415496826, |
| "logits/rejected": -1.649914026260376, |
| "logps/chosen": -264.8773498535156, |
| "logps/rejected": -379.33892822265625, |
| "loss": 0.0935, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.8796179294586182, |
| "rewards/margins": 7.090023994445801, |
| "rewards/rejected": -8.96964168548584, |
| "step": 650 |
| }, |
| { |
| "epoch": 1.3968, |
| "grad_norm": 4.21340799331665, |
| "learning_rate": 2.907001967333401e-06, |
| "logits/chosen": -1.7497447729110718, |
| "logits/rejected": -1.6952203512191772, |
| "logps/chosen": -251.13497924804688, |
| "logps/rejected": -375.689697265625, |
| "loss": 0.1294, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.122582197189331, |
| "rewards/margins": 6.757372856140137, |
| "rewards/rejected": -8.879955291748047, |
| "step": 655 |
| }, |
| { |
| "epoch": 1.4074666666666666, |
| "grad_norm": 1.3289350271224976, |
| "learning_rate": 2.878569443761442e-06, |
| "logits/chosen": -1.6650772094726562, |
| "logits/rejected": -1.5900347232818604, |
| "logps/chosen": -247.1836395263672, |
| "logps/rejected": -369.2092590332031, |
| "loss": 0.108, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7309074401855469, |
| "rewards/margins": 6.929853916168213, |
| "rewards/rejected": -8.660760879516602, |
| "step": 660 |
| }, |
| { |
| "epoch": 1.4181333333333335, |
| "grad_norm": 3.958869218826294, |
| "learning_rate": 2.85008671463468e-06, |
| "logits/chosen": -1.7087314128875732, |
| "logits/rejected": -1.6473360061645508, |
| "logps/chosen": -262.6588134765625, |
| "logps/rejected": -387.8702087402344, |
| "loss": 0.1213, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.9649025201797485, |
| "rewards/margins": 6.696756839752197, |
| "rewards/rejected": -8.66165828704834, |
| "step": 665 |
| }, |
| { |
| "epoch": 1.4288, |
| "grad_norm": 3.1254711151123047, |
| "learning_rate": 2.8215575573082127e-06, |
| "logits/chosen": -1.730808973312378, |
| "logits/rejected": -1.6865466833114624, |
| "logps/chosen": -242.3497314453125, |
| "logps/rejected": -411.3504943847656, |
| "loss": 0.1223, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7539913654327393, |
| "rewards/margins": 7.253933906555176, |
| "rewards/rejected": -9.007925033569336, |
| "step": 670 |
| }, |
| { |
| "epoch": 1.4394666666666667, |
| "grad_norm": 0.7078645825386047, |
| "learning_rate": 2.792985755294404e-06, |
| "logits/chosen": -1.8119432926177979, |
| "logits/rejected": -1.7146638631820679, |
| "logps/chosen": -256.441162109375, |
| "logps/rejected": -366.22210693359375, |
| "loss": 0.1173, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.8375685214996338, |
| "rewards/margins": 6.874407768249512, |
| "rewards/rejected": -8.711977005004883, |
| "step": 675 |
| }, |
| { |
| "epoch": 1.4501333333333333, |
| "grad_norm": 3.9787957668304443, |
| "learning_rate": 2.764375097761122e-06, |
| "logits/chosen": -1.6281518936157227, |
| "logits/rejected": -1.5047622919082642, |
| "logps/chosen": -261.5027770996094, |
| "logps/rejected": -338.6407775878906, |
| "loss": 0.1576, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.5838897228240967, |
| "rewards/margins": 6.113183498382568, |
| "rewards/rejected": -7.697073459625244, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.4607999999999999, |
| "grad_norm": 3.9224820137023926, |
| "learning_rate": 2.73572937902922e-06, |
| "logits/chosen": -1.756115198135376, |
| "logits/rejected": -1.6281507015228271, |
| "logps/chosen": -256.65557861328125, |
| "logps/rejected": -374.08941650390625, |
| "loss": 0.1303, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.3831443786621094, |
| "rewards/margins": 6.403082847595215, |
| "rewards/rejected": -7.786227226257324, |
| "step": 685 |
| }, |
| { |
| "epoch": 1.4714666666666667, |
| "grad_norm": 1.6743059158325195, |
| "learning_rate": 2.7070523980693362e-06, |
| "logits/chosen": -1.5727207660675049, |
| "logits/rejected": -1.5360314846038818, |
| "logps/chosen": -290.92431640625, |
| "logps/rejected": -380.76177978515625, |
| "loss": 0.1178, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7124334573745728, |
| "rewards/margins": 6.1151628494262695, |
| "rewards/rejected": -7.827596187591553, |
| "step": 690 |
| }, |
| { |
| "epoch": 1.4821333333333333, |
| "grad_norm": 5.704246520996094, |
| "learning_rate": 2.678347957998081e-06, |
| "logits/chosen": -1.8342517614364624, |
| "logits/rejected": -1.7212531566619873, |
| "logps/chosen": -252.77523803710938, |
| "logps/rejected": -354.2868957519531, |
| "loss": 0.1209, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.7579208612442017, |
| "rewards/margins": 6.413499355316162, |
| "rewards/rejected": -8.171420097351074, |
| "step": 695 |
| }, |
| { |
| "epoch": 1.4928, |
| "grad_norm": 1.1863569021224976, |
| "learning_rate": 2.649619865573666e-06, |
| "logits/chosen": -1.764653205871582, |
| "logits/rejected": -1.6264612674713135, |
| "logps/chosen": -278.6958312988281, |
| "logps/rejected": -401.44366455078125, |
| "loss": 0.0682, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.005868434906006, |
| "rewards/margins": 7.329827785491943, |
| "rewards/rejected": -9.33569622039795, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.5034666666666667, |
| "grad_norm": 5.131344795227051, |
| "learning_rate": 2.620871930691058e-06, |
| "logits/chosen": -1.8136281967163086, |
| "logits/rejected": -1.7170677185058594, |
| "logps/chosen": -265.4339294433594, |
| "logps/rejected": -364.80029296875, |
| "loss": 0.1557, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.9572274684906006, |
| "rewards/margins": 6.483831882476807, |
| "rewards/rejected": -8.441060066223145, |
| "step": 705 |
| }, |
| { |
| "epoch": 1.5141333333333333, |
| "grad_norm": 4.181100845336914, |
| "learning_rate": 2.5921079658767152e-06, |
| "logits/chosen": -1.7611749172210693, |
| "logits/rejected": -1.6329116821289062, |
| "logps/chosen": -267.61041259765625, |
| "logps/rejected": -355.01861572265625, |
| "loss": 0.1373, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.8553298711776733, |
| "rewards/margins": 6.1329426765441895, |
| "rewards/rejected": -7.988273620605469, |
| "step": 710 |
| }, |
| { |
| "epoch": 1.5248, |
| "grad_norm": 2.085317373275757, |
| "learning_rate": 2.56333178578297e-06, |
| "logits/chosen": -1.8043543100357056, |
| "logits/rejected": -1.6410627365112305, |
| "logps/chosen": -242.35958862304688, |
| "logps/rejected": -369.40277099609375, |
| "loss": 0.0793, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.4961168766021729, |
| "rewards/margins": 7.451325416564941, |
| "rewards/rejected": -8.947443008422852, |
| "step": 715 |
| }, |
| { |
| "epoch": 1.5354666666666668, |
| "grad_norm": 1.7874598503112793, |
| "learning_rate": 2.534547206682136e-06, |
| "logits/chosen": -1.819313645362854, |
| "logits/rejected": -1.7426176071166992, |
| "logps/chosen": -257.6264343261719, |
| "logps/rejected": -392.6166076660156, |
| "loss": 0.0841, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.016232967376709, |
| "rewards/margins": 7.167724609375, |
| "rewards/rejected": -9.183959007263184, |
| "step": 720 |
| }, |
| { |
| "epoch": 1.5461333333333334, |
| "grad_norm": 2.4844038486480713, |
| "learning_rate": 2.5057580459603947e-06, |
| "logits/chosen": -1.6962602138519287, |
| "logits/rejected": -1.6825191974639893, |
| "logps/chosen": -258.6492004394531, |
| "logps/rejected": -401.8869934082031, |
| "loss": 0.093, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.1991543769836426, |
| "rewards/margins": 7.357363700866699, |
| "rewards/rejected": -9.556517601013184, |
| "step": 725 |
| }, |
| { |
| "epoch": 1.5568, |
| "grad_norm": 4.203859806060791, |
| "learning_rate": 2.4769681216115384e-06, |
| "logits/chosen": -1.8400808572769165, |
| "logits/rejected": -1.7029167413711548, |
| "logps/chosen": -242.50137329101562, |
| "logps/rejected": -361.71673583984375, |
| "loss": 0.1276, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6351560354232788, |
| "rewards/margins": 6.772261619567871, |
| "rewards/rejected": -8.407417297363281, |
| "step": 730 |
| }, |
| { |
| "epoch": 1.5674666666666668, |
| "grad_norm": 6.503302097320557, |
| "learning_rate": 2.448181251730632e-06, |
| "logits/chosen": -1.7422183752059937, |
| "logits/rejected": -1.6272938251495361, |
| "logps/chosen": -256.3648986816406, |
| "logps/rejected": -386.4632873535156, |
| "loss": 0.1335, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.1021742820739746, |
| "rewards/margins": 6.539918422698975, |
| "rewards/rejected": -8.64209270477295, |
| "step": 735 |
| }, |
| { |
| "epoch": 1.5781333333333334, |
| "grad_norm": 1.11476469039917, |
| "learning_rate": 2.419401254007658e-06, |
| "logits/chosen": -1.715961217880249, |
| "logits/rejected": -1.55876624584198, |
| "logps/chosen": -257.96417236328125, |
| "logps/rejected": -365.32354736328125, |
| "loss": 0.0886, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7964681386947632, |
| "rewards/margins": 7.368740081787109, |
| "rewards/rejected": -9.16520881652832, |
| "step": 740 |
| }, |
| { |
| "epoch": 1.5888, |
| "grad_norm": 3.136431932449341, |
| "learning_rate": 2.390631945221221e-06, |
| "logits/chosen": -1.7085437774658203, |
| "logits/rejected": -1.5970802307128906, |
| "logps/chosen": -270.44158935546875, |
| "logps/rejected": -397.62506103515625, |
| "loss": 0.0853, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.0968310832977295, |
| "rewards/margins": 7.725828647613525, |
| "rewards/rejected": -9.82265853881836, |
| "step": 745 |
| }, |
| { |
| "epoch": 1.5994666666666668, |
| "grad_norm": 5.395340442657471, |
| "learning_rate": 2.361877140732366e-06, |
| "logits/chosen": -1.708765983581543, |
| "logits/rejected": -1.634315848350525, |
| "logps/chosen": -262.7024841308594, |
| "logps/rejected": -376.64984130859375, |
| "loss": 0.0944, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.1605114936828613, |
| "rewards/margins": 7.159433841705322, |
| "rewards/rejected": -9.319944381713867, |
| "step": 750 |
| }, |
| { |
| "epoch": 1.6101333333333332, |
| "grad_norm": 2.4793126583099365, |
| "learning_rate": 2.333140653978592e-06, |
| "logits/chosen": -1.7588707208633423, |
| "logits/rejected": -1.6335443258285522, |
| "logps/chosen": -247.37265014648438, |
| "logps/rejected": -362.1492004394531, |
| "loss": 0.1044, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.7570842504501343, |
| "rewards/margins": 7.583054542541504, |
| "rewards/rejected": -9.340139389038086, |
| "step": 755 |
| }, |
| { |
| "epoch": 1.6208, |
| "grad_norm": 2.858823537826538, |
| "learning_rate": 2.3044262959681156e-06, |
| "logits/chosen": -1.648014783859253, |
| "logits/rejected": -1.6031944751739502, |
| "logps/chosen": -252.2265625, |
| "logps/rejected": -396.5431823730469, |
| "loss": 0.0798, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.9052528142929077, |
| "rewards/margins": 7.492622375488281, |
| "rewards/rejected": -9.397875785827637, |
| "step": 760 |
| }, |
| { |
| "epoch": 1.6314666666666666, |
| "grad_norm": 0.9314205646514893, |
| "learning_rate": 2.2757378747744575e-06, |
| "logits/chosen": -1.7783981561660767, |
| "logits/rejected": -1.6433979272842407, |
| "logps/chosen": -264.9280700683594, |
| "logps/rejected": -397.2356872558594, |
| "loss": 0.0802, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.034813404083252, |
| "rewards/margins": 7.162725925445557, |
| "rewards/rejected": -9.197538375854492, |
| "step": 765 |
| }, |
| { |
| "epoch": 1.6421333333333332, |
| "grad_norm": 1.140587329864502, |
| "learning_rate": 2.24707919503142e-06, |
| "logits/chosen": -1.7422771453857422, |
| "logits/rejected": -1.6701154708862305, |
| "logps/chosen": -259.47601318359375, |
| "logps/rejected": -380.9583435058594, |
| "loss": 0.0962, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1725351810455322, |
| "rewards/margins": 7.451235771179199, |
| "rewards/rejected": -9.623769760131836, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.6528, |
| "grad_norm": 4.398436069488525, |
| "learning_rate": 2.2184540574285213e-06, |
| "logits/chosen": -1.7078425884246826, |
| "logits/rejected": -1.6657629013061523, |
| "logps/chosen": -278.24066162109375, |
| "logps/rejected": -395.03900146484375, |
| "loss": 0.1356, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.2757375240325928, |
| "rewards/margins": 7.001380920410156, |
| "rewards/rejected": -9.277118682861328, |
| "step": 775 |
| }, |
| { |
| "epoch": 1.6634666666666666, |
| "grad_norm": 0.4673113226890564, |
| "learning_rate": 2.1898662582069486e-06, |
| "logits/chosen": -1.8044401407241821, |
| "logits/rejected": -1.6835638284683228, |
| "logps/chosen": -300.3841857910156, |
| "logps/rejected": -386.4563903808594, |
| "loss": 0.0949, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.046159505844116, |
| "rewards/margins": 7.99750280380249, |
| "rewards/rejected": -10.04366397857666, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.6741333333333333, |
| "grad_norm": 6.07424783706665, |
| "learning_rate": 2.1613195886561044e-06, |
| "logits/chosen": -1.7845760583877563, |
| "logits/rejected": -1.6212952136993408, |
| "logps/chosen": -241.0775146484375, |
| "logps/rejected": -365.91839599609375, |
| "loss": 0.1304, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.1666126251220703, |
| "rewards/margins": 7.4346208572387695, |
| "rewards/rejected": -9.601232528686523, |
| "step": 785 |
| }, |
| { |
| "epoch": 1.6848, |
| "grad_norm": 3.622908353805542, |
| "learning_rate": 2.132817834610811e-06, |
| "logits/chosen": -1.7454599142074585, |
| "logits/rejected": -1.5491307973861694, |
| "logps/chosen": -267.7861328125, |
| "logps/rejected": -377.89483642578125, |
| "loss": 0.0918, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.376371145248413, |
| "rewards/margins": 7.164327144622803, |
| "rewards/rejected": -9.540698051452637, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.6954666666666667, |
| "grad_norm": 2.879439115524292, |
| "learning_rate": 2.1043647759492338e-06, |
| "logits/chosen": -1.6238399744033813, |
| "logits/rejected": -1.6405404806137085, |
| "logps/chosen": -264.1553649902344, |
| "logps/rejected": -392.42059326171875, |
| "loss": 0.1461, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.0837295055389404, |
| "rewards/margins": 7.293753147125244, |
| "rewards/rejected": -9.377483367919922, |
| "step": 795 |
| }, |
| { |
| "epoch": 1.7061333333333333, |
| "grad_norm": 5.847136497497559, |
| "learning_rate": 2.075964186091599e-06, |
| "logits/chosen": -1.7471511363983154, |
| "logits/rejected": -1.696508765220642, |
| "logps/chosen": -247.0438690185547, |
| "logps/rejected": -388.80731201171875, |
| "loss": 0.0945, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.3469150066375732, |
| "rewards/margins": 7.962810516357422, |
| "rewards/rejected": -10.309725761413574, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.7168, |
| "grad_norm": 3.7853331565856934, |
| "learning_rate": 2.047619831499765e-06, |
| "logits/chosen": -1.6483488082885742, |
| "logits/rejected": -1.5347471237182617, |
| "logps/chosen": -260.1374816894531, |
| "logps/rejected": -388.5803527832031, |
| "loss": 0.1162, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.084120035171509, |
| "rewards/margins": 7.762931823730469, |
| "rewards/rejected": -9.847051620483398, |
| "step": 805 |
| }, |
| { |
| "epoch": 1.7274666666666667, |
| "grad_norm": 0.991887629032135, |
| "learning_rate": 2.0193354711777155e-06, |
| "logits/chosen": -1.8399814367294312, |
| "logits/rejected": -1.6649506092071533, |
| "logps/chosen": -283.5755615234375, |
| "logps/rejected": -396.29107666015625, |
| "loss": 0.0995, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.306251049041748, |
| "rewards/margins": 7.420642852783203, |
| "rewards/rejected": -9.726892471313477, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.7381333333333333, |
| "grad_norm": 2.601606845855713, |
| "learning_rate": 1.991114856173048e-06, |
| "logits/chosen": -1.760546326637268, |
| "logits/rejected": -1.6287925243377686, |
| "logps/chosen": -257.34234619140625, |
| "logps/rejected": -399.37896728515625, |
| "loss": 0.0657, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.36808705329895, |
| "rewards/margins": 8.1522855758667, |
| "rewards/rejected": -10.520373344421387, |
| "step": 815 |
| }, |
| { |
| "epoch": 1.7488000000000001, |
| "grad_norm": 3.6920084953308105, |
| "learning_rate": 1.9629617290795115e-06, |
| "logits/chosen": -1.757427453994751, |
| "logits/rejected": -1.6638133525848389, |
| "logps/chosen": -256.29473876953125, |
| "logps/rejected": -385.53509521484375, |
| "loss": 0.1288, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.197131395339966, |
| "rewards/margins": 7.40597677230835, |
| "rewards/rejected": -9.603108406066895, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.7594666666666665, |
| "grad_norm": 1.5328242778778076, |
| "learning_rate": 1.934879823540663e-06, |
| "logits/chosen": -1.787667989730835, |
| "logits/rejected": -1.6369273662567139, |
| "logps/chosen": -242.244873046875, |
| "logps/rejected": -397.090087890625, |
| "loss": 0.0861, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.16058087348938, |
| "rewards/margins": 7.83211612701416, |
| "rewards/rejected": -9.992696762084961, |
| "step": 825 |
| }, |
| { |
| "epoch": 1.7701333333333333, |
| "grad_norm": 2.638007164001465, |
| "learning_rate": 1.9068728637547195e-06, |
| "logits/chosen": -1.7666289806365967, |
| "logits/rejected": -1.7008821964263916, |
| "logps/chosen": -272.5953674316406, |
| "logps/rejected": -394.2113342285156, |
| "loss": 0.0777, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.033980369567871, |
| "rewards/margins": 7.657546043395996, |
| "rewards/rejected": -9.691526412963867, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.7808000000000002, |
| "grad_norm": 4.389973163604736, |
| "learning_rate": 1.878944563980658e-06, |
| "logits/chosen": -1.7376337051391602, |
| "logits/rejected": -1.6045913696289062, |
| "logps/chosen": -255.71963500976562, |
| "logps/rejected": -367.043212890625, |
| "loss": 0.1418, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.31443452835083, |
| "rewards/margins": 7.155198574066162, |
| "rewards/rejected": -9.469633102416992, |
| "step": 835 |
| }, |
| { |
| "epoch": 1.7914666666666665, |
| "grad_norm": 2.3055145740509033, |
| "learning_rate": 1.85109862804563e-06, |
| "logits/chosen": -1.7284475564956665, |
| "logits/rejected": -1.675169587135315, |
| "logps/chosen": -266.1928405761719, |
| "logps/rejected": -362.9505920410156, |
| "loss": 0.1054, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.2873363494873047, |
| "rewards/margins": 6.60150671005249, |
| "rewards/rejected": -8.888843536376953, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.8021333333333334, |
| "grad_norm": 3.743189811706543, |
| "learning_rate": 1.823338748853766e-06, |
| "logits/chosen": -1.7340991497039795, |
| "logits/rejected": -1.6459306478500366, |
| "logps/chosen": -271.4342346191406, |
| "logps/rejected": -397.19488525390625, |
| "loss": 0.1065, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.5071041584014893, |
| "rewards/margins": 7.8493547439575195, |
| "rewards/rejected": -10.35645866394043, |
| "step": 845 |
| }, |
| { |
| "epoch": 1.8128, |
| "grad_norm": 5.754213333129883, |
| "learning_rate": 1.7956686078964257e-06, |
| "logits/chosen": -1.7911723852157593, |
| "logits/rejected": -1.6511707305908203, |
| "logps/chosen": -252.0098114013672, |
| "logps/rejected": -386.8243408203125, |
| "loss": 0.1497, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.3370299339294434, |
| "rewards/margins": 7.750609397888184, |
| "rewards/rejected": -10.087640762329102, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.8234666666666666, |
| "grad_norm": 8.280444145202637, |
| "learning_rate": 1.7680918747639608e-06, |
| "logits/chosen": -1.743274450302124, |
| "logits/rejected": -1.6286823749542236, |
| "logps/chosen": -269.9051513671875, |
| "logps/rejected": -400.05938720703125, |
| "loss": 0.1064, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.5208077430725098, |
| "rewards/margins": 7.579964637756348, |
| "rewards/rejected": -10.1007719039917, |
| "step": 855 |
| }, |
| { |
| "epoch": 1.8341333333333334, |
| "grad_norm": 1.875370740890503, |
| "learning_rate": 1.740612206659057e-06, |
| "logits/chosen": -1.7147190570831299, |
| "logits/rejected": -1.689009428024292, |
| "logps/chosen": -254.6274871826172, |
| "logps/rejected": -388.3784484863281, |
| "loss": 0.1093, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9739549160003662, |
| "rewards/margins": 7.456284523010254, |
| "rewards/rejected": -9.4302396774292, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.8448, |
| "grad_norm": 5.320019721984863, |
| "learning_rate": 1.7132332479117204e-06, |
| "logits/chosen": -1.728094458580017, |
| "logits/rejected": -1.5693343877792358, |
| "logps/chosen": -263.1184387207031, |
| "logps/rejected": -371.22613525390625, |
| "loss": 0.087, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.0235514640808105, |
| "rewards/margins": 7.408260345458984, |
| "rewards/rejected": -9.43181037902832, |
| "step": 865 |
| }, |
| { |
| "epoch": 1.8554666666666666, |
| "grad_norm": 4.183524131774902, |
| "learning_rate": 1.6859586294959666e-06, |
| "logits/chosen": -1.8199039697647095, |
| "logits/rejected": -1.6934419870376587, |
| "logps/chosen": -253.3587646484375, |
| "logps/rejected": -392.0380859375, |
| "loss": 0.078, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.7781072854995728, |
| "rewards/margins": 7.864907741546631, |
| "rewards/rejected": -9.64301586151123, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.8661333333333334, |
| "grad_norm": 1.000540852546692, |
| "learning_rate": 1.6587919685482884e-06, |
| "logits/chosen": -1.7975037097930908, |
| "logits/rejected": -1.6539417505264282, |
| "logps/chosen": -246.7928466796875, |
| "logps/rejected": -366.54071044921875, |
| "loss": 0.1199, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.086042642593384, |
| "rewards/margins": 7.348335266113281, |
| "rewards/rejected": -9.434378623962402, |
| "step": 875 |
| }, |
| { |
| "epoch": 1.8768, |
| "grad_norm": 2.6079916954040527, |
| "learning_rate": 1.6317368678879497e-06, |
| "logits/chosen": -1.6367311477661133, |
| "logits/rejected": -1.5827094316482544, |
| "logps/chosen": -256.6966247558594, |
| "logps/rejected": -376.14373779296875, |
| "loss": 0.0897, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9041290283203125, |
| "rewards/margins": 7.1437859535217285, |
| "rewards/rejected": -9.047914505004883, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.8874666666666666, |
| "grad_norm": 3.0209317207336426, |
| "learning_rate": 1.6047969155391856e-06, |
| "logits/chosen": -1.7882978916168213, |
| "logits/rejected": -1.6401067972183228, |
| "logps/chosen": -260.91717529296875, |
| "logps/rejected": -385.9642028808594, |
| "loss": 0.0945, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.86648428440094, |
| "rewards/margins": 8.293737411499023, |
| "rewards/rejected": -10.160222053527832, |
| "step": 885 |
| }, |
| { |
| "epoch": 1.8981333333333335, |
| "grad_norm": 0.41572609543800354, |
| "learning_rate": 1.5779756842553617e-06, |
| "logits/chosen": -1.7787930965423584, |
| "logits/rejected": -1.6832020282745361, |
| "logps/chosen": -259.8187561035156, |
| "logps/rejected": -407.3184509277344, |
| "loss": 0.0504, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.7974926233291626, |
| "rewards/margins": 8.537004470825195, |
| "rewards/rejected": -10.334497451782227, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.9088, |
| "grad_norm": 4.283809661865234, |
| "learning_rate": 1.551276731045157e-06, |
| "logits/chosen": -1.7304534912109375, |
| "logits/rejected": -1.6511510610580444, |
| "logps/chosen": -288.17987060546875, |
| "logps/rejected": -412.12396240234375, |
| "loss": 0.1044, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9651966094970703, |
| "rewards/margins": 7.937309265136719, |
| "rewards/rejected": -9.902505874633789, |
| "step": 895 |
| }, |
| { |
| "epoch": 1.9194666666666667, |
| "grad_norm": 1.008398413658142, |
| "learning_rate": 1.5247035967008369e-06, |
| "logits/chosen": -1.5831853151321411, |
| "logits/rejected": -1.5248817205429077, |
| "logps/chosen": -259.24981689453125, |
| "logps/rejected": -378.78558349609375, |
| "loss": 0.0808, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.084197521209717, |
| "rewards/margins": 7.348579406738281, |
| "rewards/rejected": -9.432775497436523, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.9301333333333335, |
| "grad_norm": 3.965376377105713, |
| "learning_rate": 1.498259805328678e-06, |
| "logits/chosen": -1.72213613986969, |
| "logits/rejected": -1.5788049697875977, |
| "logps/chosen": -264.9279479980469, |
| "logps/rejected": -370.3101806640625, |
| "loss": 0.0861, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.3263232707977295, |
| "rewards/margins": 7.561483860015869, |
| "rewards/rejected": -9.887807846069336, |
| "step": 905 |
| }, |
| { |
| "epoch": 1.9407999999999999, |
| "grad_norm": 4.654145240783691, |
| "learning_rate": 1.471948863881601e-06, |
| "logits/chosen": -1.7574799060821533, |
| "logits/rejected": -1.6463959217071533, |
| "logps/chosen": -275.3285217285156, |
| "logps/rejected": -394.51324462890625, |
| "loss": 0.0984, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.8904142379760742, |
| "rewards/margins": 7.419065952301025, |
| "rewards/rejected": -9.309479713439941, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.9514666666666667, |
| "grad_norm": 2.2931063175201416, |
| "learning_rate": 1.4457742616940864e-06, |
| "logits/chosen": -1.6442797183990479, |
| "logits/rejected": -1.578762412071228, |
| "logps/chosen": -277.2344665527344, |
| "logps/rejected": -399.81964111328125, |
| "loss": 0.1103, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1587257385253906, |
| "rewards/margins": 7.75518798828125, |
| "rewards/rejected": -9.913914680480957, |
| "step": 915 |
| }, |
| { |
| "epoch": 1.9621333333333333, |
| "grad_norm": 3.6048293113708496, |
| "learning_rate": 1.4197394700194165e-06, |
| "logits/chosen": -1.7335107326507568, |
| "logits/rejected": -1.616270661354065, |
| "logps/chosen": -262.0077819824219, |
| "logps/rejected": -375.99005126953125, |
| "loss": 0.106, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.032534599304199, |
| "rewards/margins": 7.5883941650390625, |
| "rewards/rejected": -9.620927810668945, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.9727999999999999, |
| "grad_norm": 2.6519317626953125, |
| "learning_rate": 1.3938479415693244e-06, |
| "logits/chosen": -1.741246223449707, |
| "logits/rejected": -1.6494983434677124, |
| "logps/chosen": -271.10101318359375, |
| "logps/rejected": -381.05450439453125, |
| "loss": 0.0948, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.9012314081192017, |
| "rewards/margins": 7.431715488433838, |
| "rewards/rejected": -9.33294677734375, |
| "step": 925 |
| }, |
| { |
| "epoch": 1.9834666666666667, |
| "grad_norm": 3.518383026123047, |
| "learning_rate": 1.368103110056096e-06, |
| "logits/chosen": -1.7782844305038452, |
| "logits/rejected": -1.6442190408706665, |
| "logps/chosen": -266.40045166015625, |
| "logps/rejected": -369.31756591796875, |
| "loss": 0.1255, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.061063289642334, |
| "rewards/margins": 7.457045078277588, |
| "rewards/rejected": -9.518108367919922, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.9941333333333333, |
| "grad_norm": 5.0803046226501465, |
| "learning_rate": 1.3425083897371983e-06, |
| "logits/chosen": -1.5946252346038818, |
| "logits/rejected": -1.5125935077667236, |
| "logps/chosen": -254.85342407226562, |
| "logps/rejected": -365.8056945800781, |
| "loss": 0.0898, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6178001165390015, |
| "rewards/margins": 7.518826961517334, |
| "rewards/rejected": -9.136625289916992, |
| "step": 935 |
| }, |
| { |
| "epoch": 2.0042666666666666, |
| "grad_norm": 2.6192548274993896, |
| "learning_rate": 1.317067174962478e-06, |
| "logits/chosen": -1.7343151569366455, |
| "logits/rejected": -1.6101001501083374, |
| "logps/chosen": -261.07666015625, |
| "logps/rejected": -379.0164794921875, |
| "loss": 0.0825, |
| "rewards/accuracies": 0.9868420958518982, |
| "rewards/chosen": -1.6997953653335571, |
| "rewards/margins": 7.317230224609375, |
| "rewards/rejected": -9.0170259475708, |
| "step": 940 |
| }, |
| { |
| "epoch": 2.0149333333333335, |
| "grad_norm": 1.5539934635162354, |
| "learning_rate": 1.291782839724009e-06, |
| "logits/chosen": -1.718090295791626, |
| "logits/rejected": -1.6289358139038086, |
| "logps/chosen": -268.7707824707031, |
| "logps/rejected": -378.489990234375, |
| "loss": 0.0841, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6988178491592407, |
| "rewards/margins": 7.449611663818359, |
| "rewards/rejected": -9.148429870605469, |
| "step": 945 |
| }, |
| { |
| "epoch": 2.0256, |
| "grad_norm": 2.7771873474121094, |
| "learning_rate": 1.2666587372086352e-06, |
| "logits/chosen": -1.6812318563461304, |
| "logits/rejected": -1.5723031759262085, |
| "logps/chosen": -254.5369415283203, |
| "logps/rejected": -378.9630126953125, |
| "loss": 0.0768, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.7308629751205444, |
| "rewards/margins": 7.691521644592285, |
| "rewards/rejected": -9.422384262084961, |
| "step": 950 |
| }, |
| { |
| "epoch": 2.0362666666666667, |
| "grad_norm": 1.9423669576644897, |
| "learning_rate": 1.241698199353275e-06, |
| "logits/chosen": -1.7462921142578125, |
| "logits/rejected": -1.6803960800170898, |
| "logps/chosen": -266.7231140136719, |
| "logps/rejected": -418.3180236816406, |
| "loss": 0.1006, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.0733628273010254, |
| "rewards/margins": 7.935437202453613, |
| "rewards/rejected": -10.008800506591797, |
| "step": 955 |
| }, |
| { |
| "epoch": 2.0469333333333335, |
| "grad_norm": 1.0960689783096313, |
| "learning_rate": 1.2169045364030404e-06, |
| "logits/chosen": -1.7646312713623047, |
| "logits/rejected": -1.6422895193099976, |
| "logps/chosen": -249.3695526123047, |
| "logps/rejected": -383.7786560058594, |
| "loss": 0.0714, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.136165142059326, |
| "rewards/margins": 7.901527404785156, |
| "rewards/rejected": -10.037691116333008, |
| "step": 960 |
| }, |
| { |
| "epoch": 2.0576, |
| "grad_norm": 1.9646838903427124, |
| "learning_rate": 1.1922810364722402e-06, |
| "logits/chosen": -1.744693398475647, |
| "logits/rejected": -1.5728899240493774, |
| "logps/chosen": -273.8901062011719, |
| "logps/rejected": -388.103515625, |
| "loss": 0.0702, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.5939252376556396, |
| "rewards/margins": 7.756191253662109, |
| "rewards/rejected": -9.350118637084961, |
| "step": 965 |
| }, |
| { |
| "epoch": 2.0682666666666667, |
| "grad_norm": 2.2951996326446533, |
| "learning_rate": 1.1678309651083055e-06, |
| "logits/chosen": -1.7867664098739624, |
| "logits/rejected": -1.6740038394927979, |
| "logps/chosen": -274.0558166503906, |
| "logps/rejected": -405.5430908203125, |
| "loss": 0.0448, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.1959640979766846, |
| "rewards/margins": 8.143606185913086, |
| "rewards/rejected": -10.339570045471191, |
| "step": 970 |
| }, |
| { |
| "epoch": 2.0789333333333335, |
| "grad_norm": 1.492812156677246, |
| "learning_rate": 1.1435575648587224e-06, |
| "logits/chosen": -1.6263166666030884, |
| "logits/rejected": -1.5808578729629517, |
| "logps/chosen": -273.41229248046875, |
| "logps/rejected": -402.88348388671875, |
| "loss": 0.0522, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.041072368621826, |
| "rewards/margins": 8.062564849853516, |
| "rewards/rejected": -10.1036376953125, |
| "step": 975 |
| }, |
| { |
| "epoch": 2.0896, |
| "grad_norm": 2.0464603900909424, |
| "learning_rate": 1.1194640548410039e-06, |
| "logits/chosen": -1.6150739192962646, |
| "logits/rejected": -1.6028960943222046, |
| "logps/chosen": -255.7452850341797, |
| "logps/rejected": -357.42822265625, |
| "loss": 0.0904, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.7968528270721436, |
| "rewards/margins": 7.02915096282959, |
| "rewards/rejected": -8.826005935668945, |
| "step": 980 |
| }, |
| { |
| "epoch": 2.1002666666666667, |
| "grad_norm": 3.042483329772949, |
| "learning_rate": 1.095553630315774e-06, |
| "logits/chosen": -1.7164863348007202, |
| "logits/rejected": -1.6247742176055908, |
| "logps/chosen": -276.2933654785156, |
| "logps/rejected": -400.1717529296875, |
| "loss": 0.0806, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.226152181625366, |
| "rewards/margins": 7.620535850524902, |
| "rewards/rejected": -9.846689224243164, |
| "step": 985 |
| }, |
| { |
| "epoch": 2.1109333333333336, |
| "grad_norm": 2.414696216583252, |
| "learning_rate": 1.0718294622630188e-06, |
| "logits/chosen": -1.6823770999908447, |
| "logits/rejected": -1.5639375448226929, |
| "logps/chosen": -289.0705261230469, |
| "logps/rejected": -421.67425537109375, |
| "loss": 0.0784, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.4110684394836426, |
| "rewards/margins": 8.174033164978027, |
| "rewards/rejected": -10.585102081298828, |
| "step": 990 |
| }, |
| { |
| "epoch": 2.1216, |
| "grad_norm": 2.291530132293701, |
| "learning_rate": 1.0482946969615495e-06, |
| "logits/chosen": -1.7374327182769775, |
| "logits/rejected": -1.6161400079727173, |
| "logps/chosen": -256.18450927734375, |
| "logps/rejected": -393.5759582519531, |
| "loss": 0.0668, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.667482852935791, |
| "rewards/margins": 8.163459777832031, |
| "rewards/rejected": -10.830942153930664, |
| "step": 995 |
| }, |
| { |
| "epoch": 2.1322666666666668, |
| "grad_norm": 3.7659263610839844, |
| "learning_rate": 1.0249524555717461e-06, |
| "logits/chosen": -1.697607398033142, |
| "logits/rejected": -1.5797332525253296, |
| "logps/chosen": -263.2526550292969, |
| "logps/rejected": -401.4608154296875, |
| "loss": 0.0987, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.1764533519744873, |
| "rewards/margins": 7.859429836273193, |
| "rewards/rejected": -10.035882949829102, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.142933333333333, |
| "grad_norm": 2.7956254482269287, |
| "learning_rate": 1.0018058337216327e-06, |
| "logits/chosen": -1.6498619318008423, |
| "logits/rejected": -1.6099936962127686, |
| "logps/chosen": -267.5693359375, |
| "logps/rejected": -401.8230895996094, |
| "loss": 0.0769, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.4292540550231934, |
| "rewards/margins": 8.179109573364258, |
| "rewards/rejected": -10.608363151550293, |
| "step": 1005 |
| }, |
| { |
| "epoch": 2.1536, |
| "grad_norm": 1.6683810949325562, |
| "learning_rate": 9.788579010963387e-07, |
| "logits/chosen": -1.7626358270645142, |
| "logits/rejected": -1.6691913604736328, |
| "logps/chosen": -294.8406066894531, |
| "logps/rejected": -387.442138671875, |
| "loss": 0.0647, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.454096555709839, |
| "rewards/margins": 7.879624843597412, |
| "rewards/rejected": -10.333722114562988, |
| "step": 1010 |
| }, |
| { |
| "epoch": 2.164266666666667, |
| "grad_norm": 2.0527291297912598, |
| "learning_rate": 9.56111701030998e-07, |
| "logits/chosen": -1.7989389896392822, |
| "logits/rejected": -1.649007797241211, |
| "logps/chosen": -257.8086242675781, |
| "logps/rejected": -396.00946044921875, |
| "loss": 0.0892, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.2390990257263184, |
| "rewards/margins": 8.126911163330078, |
| "rewards/rejected": -10.366010665893555, |
| "step": 1015 |
| }, |
| { |
| "epoch": 2.174933333333333, |
| "grad_norm": 1.3809715509414673, |
| "learning_rate": 9.335702501071459e-07, |
| "logits/chosen": -1.727193832397461, |
| "logits/rejected": -1.6351912021636963, |
| "logps/chosen": -266.216796875, |
| "logps/rejected": -408.1944885253906, |
| "loss": 0.0571, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.289905548095703, |
| "rewards/margins": 8.413689613342285, |
| "rewards/rejected": -10.703596115112305, |
| "step": 1020 |
| }, |
| { |
| "epoch": 2.1856, |
| "grad_norm": 2.758436441421509, |
| "learning_rate": 9.112365377526636e-07, |
| "logits/chosen": -1.7538105249404907, |
| "logits/rejected": -1.6839125156402588, |
| "logps/chosen": -251.68896484375, |
| "logps/rejected": -405.6483459472656, |
| "loss": 0.0612, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4383013248443604, |
| "rewards/margins": 8.36324691772461, |
| "rewards/rejected": -10.801547050476074, |
| "step": 1025 |
| }, |
| { |
| "epoch": 2.196266666666667, |
| "grad_norm": 1.1113784313201904, |
| "learning_rate": 8.891135258453195e-07, |
| "logits/chosen": -1.679633378982544, |
| "logits/rejected": -1.6179072856903076, |
| "logps/chosen": -247.2431182861328, |
| "logps/rejected": -389.5404968261719, |
| "loss": 0.0684, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3117198944091797, |
| "rewards/margins": 7.995427131652832, |
| "rewards/rejected": -10.307147026062012, |
| "step": 1030 |
| }, |
| { |
| "epoch": 2.206933333333333, |
| "grad_norm": 3.6547203063964844, |
| "learning_rate": 8.672041483199709e-07, |
| "logits/chosen": -1.726601243019104, |
| "logits/rejected": -1.6291310787200928, |
| "logps/chosen": -255.78787231445312, |
| "logps/rejected": -379.4979553222656, |
| "loss": 0.0652, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.340198040008545, |
| "rewards/margins": 7.787562370300293, |
| "rewards/rejected": -10.12775993347168, |
| "step": 1035 |
| }, |
| { |
| "epoch": 2.2176, |
| "grad_norm": 2.5580599308013916, |
| "learning_rate": 8.455113107794652e-07, |
| "logits/chosen": -1.7268718481063843, |
| "logits/rejected": -1.6617317199707031, |
| "logps/chosen": -246.4077606201172, |
| "logps/rejected": -419.98321533203125, |
| "loss": 0.0552, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.661684036254883, |
| "rewards/margins": 9.117815017700195, |
| "rewards/rejected": -11.779499053955078, |
| "step": 1040 |
| }, |
| { |
| "epoch": 2.228266666666667, |
| "grad_norm": 2.44766902923584, |
| "learning_rate": 8.240378901093035e-07, |
| "logits/chosen": -1.7534101009368896, |
| "logits/rejected": -1.5951759815216064, |
| "logps/chosen": -284.9955139160156, |
| "logps/rejected": -385.4869384765625, |
| "loss": 0.1098, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.5828628540039062, |
| "rewards/margins": 7.782918453216553, |
| "rewards/rejected": -10.365781784057617, |
| "step": 1045 |
| }, |
| { |
| "epoch": 2.238933333333333, |
| "grad_norm": 3.560786724090576, |
| "learning_rate": 8.027867340961113e-07, |
| "logits/chosen": -1.662253975868225, |
| "logits/rejected": -1.5555870532989502, |
| "logps/chosen": -270.2218933105469, |
| "logps/rejected": -380.3742370605469, |
| "loss": 0.1164, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -2.591815710067749, |
| "rewards/margins": 7.998503684997559, |
| "rewards/rejected": -10.590320587158203, |
| "step": 1050 |
| }, |
| { |
| "epoch": 2.2496, |
| "grad_norm": 2.0515780448913574, |
| "learning_rate": 7.817606610499639e-07, |
| "logits/chosen": -1.6859337091445923, |
| "logits/rejected": -1.62407648563385, |
| "logps/chosen": -270.041015625, |
| "logps/rejected": -386.9949645996094, |
| "loss": 0.0834, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.4862797260284424, |
| "rewards/margins": 7.569838047027588, |
| "rewards/rejected": -10.056117057800293, |
| "step": 1055 |
| }, |
| { |
| "epoch": 2.2602666666666664, |
| "grad_norm": 3.088653564453125, |
| "learning_rate": 7.609624594306278e-07, |
| "logits/chosen": -1.7772037982940674, |
| "logits/rejected": -1.6899620294570923, |
| "logps/chosen": -271.0976257324219, |
| "logps/rejected": -385.71441650390625, |
| "loss": 0.0812, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.923471689224243, |
| "rewards/margins": 7.496604919433594, |
| "rewards/rejected": -10.420077323913574, |
| "step": 1060 |
| }, |
| { |
| "epoch": 2.2709333333333332, |
| "grad_norm": 3.4833757877349854, |
| "learning_rate": 7.403948874777556e-07, |
| "logits/chosen": -1.7119638919830322, |
| "logits/rejected": -1.6148226261138916, |
| "logps/chosen": -261.0135192871094, |
| "logps/rejected": -381.09100341796875, |
| "loss": 0.1112, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.67107892036438, |
| "rewards/margins": 7.778635501861572, |
| "rewards/rejected": -10.449714660644531, |
| "step": 1065 |
| }, |
| { |
| "epoch": 2.2816, |
| "grad_norm": 3.0664234161376953, |
| "learning_rate": 7.20060672845091e-07, |
| "logits/chosen": -1.6737985610961914, |
| "logits/rejected": -1.6506000757217407, |
| "logps/chosen": -269.85247802734375, |
| "logps/rejected": -413.30511474609375, |
| "loss": 0.0576, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.866260051727295, |
| "rewards/margins": 8.69055461883545, |
| "rewards/rejected": -11.556814193725586, |
| "step": 1070 |
| }, |
| { |
| "epoch": 2.2922666666666665, |
| "grad_norm": 1.538775086402893, |
| "learning_rate": 6.99962512238733e-07, |
| "logits/chosen": -1.6387313604354858, |
| "logits/rejected": -1.5764515399932861, |
| "logps/chosen": -252.82766723632812, |
| "logps/rejected": -385.333984375, |
| "loss": 0.0883, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.387939691543579, |
| "rewards/margins": 8.016633987426758, |
| "rewards/rejected": -10.404573440551758, |
| "step": 1075 |
| }, |
| { |
| "epoch": 2.3029333333333333, |
| "grad_norm": 1.7143809795379639, |
| "learning_rate": 6.801030710594949e-07, |
| "logits/chosen": -1.5965774059295654, |
| "logits/rejected": -1.536005973815918, |
| "logps/chosen": -261.13275146484375, |
| "logps/rejected": -383.4078369140625, |
| "loss": 0.0765, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.6068761348724365, |
| "rewards/margins": 7.689870357513428, |
| "rewards/rejected": -10.296747207641602, |
| "step": 1080 |
| }, |
| { |
| "epoch": 2.3136, |
| "grad_norm": 4.500543117523193, |
| "learning_rate": 6.604849830494264e-07, |
| "logits/chosen": -1.7209281921386719, |
| "logits/rejected": -1.6301357746124268, |
| "logps/chosen": -252.2537384033203, |
| "logps/rejected": -401.78277587890625, |
| "loss": 0.0489, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.3812525272369385, |
| "rewards/margins": 8.41871166229248, |
| "rewards/rejected": -10.799964904785156, |
| "step": 1085 |
| }, |
| { |
| "epoch": 2.3242666666666665, |
| "grad_norm": 1.3474994897842407, |
| "learning_rate": 6.411108499425253e-07, |
| "logits/chosen": -1.7181062698364258, |
| "logits/rejected": -1.596534252166748, |
| "logps/chosen": -276.0671081542969, |
| "logps/rejected": -398.2869567871094, |
| "loss": 0.0748, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.573090076446533, |
| "rewards/margins": 7.86136531829834, |
| "rewards/rejected": -10.434455871582031, |
| "step": 1090 |
| }, |
| { |
| "epoch": 2.3349333333333333, |
| "grad_norm": 1.8286346197128296, |
| "learning_rate": 6.219832411197013e-07, |
| "logits/chosen": -1.6104246377944946, |
| "logits/rejected": -1.5124198198318481, |
| "logps/chosen": -272.6994934082031, |
| "logps/rejected": -403.438232421875, |
| "loss": 0.0775, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.20379638671875, |
| "rewards/margins": 8.984777450561523, |
| "rewards/rejected": -11.188573837280273, |
| "step": 1095 |
| }, |
| { |
| "epoch": 2.3456, |
| "grad_norm": 0.6198655962944031, |
| "learning_rate": 6.031046932680229e-07, |
| "logits/chosen": -1.727940559387207, |
| "logits/rejected": -1.5797127485275269, |
| "logps/chosen": -262.6995849609375, |
| "logps/rejected": -391.15765380859375, |
| "loss": 0.0549, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.521888256072998, |
| "rewards/margins": 8.586507797241211, |
| "rewards/rejected": -11.108396530151367, |
| "step": 1100 |
| }, |
| { |
| "epoch": 2.3562666666666665, |
| "grad_norm": 1.2429842948913574, |
| "learning_rate": 5.844777100443091e-07, |
| "logits/chosen": -1.6657642126083374, |
| "logits/rejected": -1.6233059167861938, |
| "logps/chosen": -265.207275390625, |
| "logps/rejected": -429.60089111328125, |
| "loss": 0.0806, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.748042583465576, |
| "rewards/margins": 8.55555534362793, |
| "rewards/rejected": -11.303597450256348, |
| "step": 1105 |
| }, |
| { |
| "epoch": 2.3669333333333333, |
| "grad_norm": 2.5982189178466797, |
| "learning_rate": 5.66104761743092e-07, |
| "logits/chosen": -1.833526611328125, |
| "logits/rejected": -1.7374675273895264, |
| "logps/chosen": -254.8466339111328, |
| "logps/rejected": -398.44610595703125, |
| "loss": 0.0519, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.018444538116455, |
| "rewards/margins": 8.912156105041504, |
| "rewards/rejected": -11.930599212646484, |
| "step": 1110 |
| }, |
| { |
| "epoch": 2.3776, |
| "grad_norm": 3.6712732315063477, |
| "learning_rate": 5.479882849690102e-07, |
| "logits/chosen": -1.6810152530670166, |
| "logits/rejected": -1.617287278175354, |
| "logps/chosen": -278.04498291015625, |
| "logps/rejected": -404.9657287597656, |
| "loss": 0.0683, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.8550310134887695, |
| "rewards/margins": 7.767541408538818, |
| "rewards/rejected": -10.62257194519043, |
| "step": 1115 |
| }, |
| { |
| "epoch": 2.3882666666666665, |
| "grad_norm": 1.2086095809936523, |
| "learning_rate": 5.301306823136687e-07, |
| "logits/chosen": -1.6428139209747314, |
| "logits/rejected": -1.569113850593567, |
| "logps/chosen": -260.32574462890625, |
| "logps/rejected": -399.7398986816406, |
| "loss": 0.1033, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.7781503200531006, |
| "rewards/margins": 8.46976089477539, |
| "rewards/rejected": -11.247909545898438, |
| "step": 1120 |
| }, |
| { |
| "epoch": 2.3989333333333334, |
| "grad_norm": 1.18802809715271, |
| "learning_rate": 5.125343220370088e-07, |
| "logits/chosen": -1.6931226253509521, |
| "logits/rejected": -1.592712640762329, |
| "logps/chosen": -253.77578735351562, |
| "logps/rejected": -374.4710388183594, |
| "loss": 0.0581, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5571885108947754, |
| "rewards/margins": 8.29760456085205, |
| "rewards/rejected": -10.854792594909668, |
| "step": 1125 |
| }, |
| { |
| "epoch": 2.4096, |
| "grad_norm": 3.1134982109069824, |
| "learning_rate": 4.952015377532329e-07, |
| "logits/chosen": -1.7127673625946045, |
| "logits/rejected": -1.7162590026855469, |
| "logps/chosen": -270.6441955566406, |
| "logps/rejected": -401.02496337890625, |
| "loss": 0.0903, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.0124757289886475, |
| "rewards/margins": 8.173250198364258, |
| "rewards/rejected": -11.185726165771484, |
| "step": 1130 |
| }, |
| { |
| "epoch": 2.4202666666666666, |
| "grad_norm": 6.751755237579346, |
| "learning_rate": 4.781346281213204e-07, |
| "logits/chosen": -1.7349708080291748, |
| "logits/rejected": -1.66085684299469, |
| "logps/chosen": -260.246337890625, |
| "logps/rejected": -419.47943115234375, |
| "loss": 0.066, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.005652904510498, |
| "rewards/margins": 9.477314949035645, |
| "rewards/rejected": -12.482967376708984, |
| "step": 1135 |
| }, |
| { |
| "epoch": 2.4309333333333334, |
| "grad_norm": 2.8330957889556885, |
| "learning_rate": 4.613358565401818e-07, |
| "logits/chosen": -1.615511178970337, |
| "logits/rejected": -1.502151370048523, |
| "logps/chosen": -264.5709533691406, |
| "logps/rejected": -379.5435791015625, |
| "loss": 0.0647, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5337510108947754, |
| "rewards/margins": 7.921186923980713, |
| "rewards/rejected": -10.454936981201172, |
| "step": 1140 |
| }, |
| { |
| "epoch": 2.4416, |
| "grad_norm": 2.5002682209014893, |
| "learning_rate": 4.448074508484906e-07, |
| "logits/chosen": -1.7348816394805908, |
| "logits/rejected": -1.641600251197815, |
| "logps/chosen": -261.57965087890625, |
| "logps/rejected": -403.952880859375, |
| "loss": 0.0556, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5588488578796387, |
| "rewards/margins": 8.548656463623047, |
| "rewards/rejected": -11.107504844665527, |
| "step": 1145 |
| }, |
| { |
| "epoch": 2.4522666666666666, |
| "grad_norm": 6.115179538726807, |
| "learning_rate": 4.2855160302922777e-07, |
| "logits/chosen": -1.697669267654419, |
| "logits/rejected": -1.6198641061782837, |
| "logps/chosen": -256.3226623535156, |
| "logps/rejected": -405.70111083984375, |
| "loss": 0.0852, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.8319969177246094, |
| "rewards/margins": 8.845357894897461, |
| "rewards/rejected": -11.67735481262207, |
| "step": 1150 |
| }, |
| { |
| "epoch": 2.4629333333333334, |
| "grad_norm": 4.671123027801514, |
| "learning_rate": 4.125704689189819e-07, |
| "logits/chosen": -1.665469765663147, |
| "logits/rejected": -1.5715079307556152, |
| "logps/chosen": -243.5298309326172, |
| "logps/rejected": -388.2669372558594, |
| "loss": 0.0924, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.220625400543213, |
| "rewards/margins": 8.359115600585938, |
| "rewards/rejected": -10.579740524291992, |
| "step": 1155 |
| }, |
| { |
| "epoch": 2.4736000000000002, |
| "grad_norm": 1.6122468709945679, |
| "learning_rate": 3.9686616792204677e-07, |
| "logits/chosen": -1.7475868463516235, |
| "logits/rejected": -1.692379355430603, |
| "logps/chosen": -283.0847473144531, |
| "logps/rejected": -408.1714782714844, |
| "loss": 0.0395, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.709376573562622, |
| "rewards/margins": 8.082962036132812, |
| "rewards/rejected": -10.792339324951172, |
| "step": 1160 |
| }, |
| { |
| "epoch": 2.4842666666666666, |
| "grad_norm": 2.0977368354797363, |
| "learning_rate": 3.8144078272934676e-07, |
| "logits/chosen": -1.654688835144043, |
| "logits/rejected": -1.6329755783081055, |
| "logps/chosen": -286.69622802734375, |
| "logps/rejected": -413.87481689453125, |
| "loss": 0.088, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.884509325027466, |
| "rewards/margins": 8.137341499328613, |
| "rewards/rejected": -11.021848678588867, |
| "step": 1165 |
| }, |
| { |
| "epoch": 2.4949333333333334, |
| "grad_norm": 3.7184035778045654, |
| "learning_rate": 3.6629635904222943e-07, |
| "logits/chosen": -1.6443443298339844, |
| "logits/rejected": -1.5297214984893799, |
| "logps/chosen": -286.8924865722656, |
| "logps/rejected": -398.5106506347656, |
| "loss": 0.1043, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.0164103507995605, |
| "rewards/margins": 7.627072334289551, |
| "rewards/rejected": -10.64348316192627, |
| "step": 1170 |
| }, |
| { |
| "epoch": 2.5056000000000003, |
| "grad_norm": 4.416862487792969, |
| "learning_rate": 3.514349053011698e-07, |
| "logits/chosen": -1.6677930355072021, |
| "logits/rejected": -1.5294122695922852, |
| "logps/chosen": -269.85345458984375, |
| "logps/rejected": -371.2940979003906, |
| "loss": 0.0723, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5542593002319336, |
| "rewards/margins": 7.903220176696777, |
| "rewards/rejected": -10.457479476928711, |
| "step": 1175 |
| }, |
| { |
| "epoch": 2.5162666666666667, |
| "grad_norm": 2.2231297492980957, |
| "learning_rate": 3.3685839241941133e-07, |
| "logits/chosen": -1.6161854267120361, |
| "logits/rejected": -1.6047331094741821, |
| "logps/chosen": -283.4258117675781, |
| "logps/rejected": -387.2071228027344, |
| "loss": 0.0765, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.8923115730285645, |
| "rewards/margins": 8.108506202697754, |
| "rewards/rejected": -11.000818252563477, |
| "step": 1180 |
| }, |
| { |
| "epoch": 2.5269333333333335, |
| "grad_norm": 1.9496991634368896, |
| "learning_rate": 3.2256875352158435e-07, |
| "logits/chosen": -1.778193712234497, |
| "logits/rejected": -1.6277202367782593, |
| "logps/chosen": -268.64862060546875, |
| "logps/rejected": -414.49481201171875, |
| "loss": 0.0705, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5748419761657715, |
| "rewards/margins": 8.338712692260742, |
| "rewards/rejected": -10.913556098937988, |
| "step": 1185 |
| }, |
| { |
| "epoch": 2.5376, |
| "grad_norm": 1.1859385967254639, |
| "learning_rate": 3.085678836873399e-07, |
| "logits/chosen": -1.7131965160369873, |
| "logits/rejected": -1.6504449844360352, |
| "logps/chosen": -251.01370239257812, |
| "logps/rejected": -401.9787292480469, |
| "loss": 0.0762, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.776256799697876, |
| "rewards/margins": 8.66688346862793, |
| "rewards/rejected": -11.443140029907227, |
| "step": 1190 |
| }, |
| { |
| "epoch": 2.5482666666666667, |
| "grad_norm": 1.0037485361099243, |
| "learning_rate": 2.9485763970002106e-07, |
| "logits/chosen": -1.6434129476547241, |
| "logits/rejected": -1.5615208148956299, |
| "logps/chosen": -273.0927734375, |
| "logps/rejected": -408.0320739746094, |
| "loss": 0.0603, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.6806273460388184, |
| "rewards/margins": 8.583781242370605, |
| "rewards/rejected": -11.264409065246582, |
| "step": 1195 |
| }, |
| { |
| "epoch": 2.558933333333333, |
| "grad_norm": 2.8410496711730957, |
| "learning_rate": 2.8143983980042174e-07, |
| "logits/chosen": -1.6743905544281006, |
| "logits/rejected": -1.6090198755264282, |
| "logps/chosen": -269.591552734375, |
| "logps/rejected": -399.52471923828125, |
| "loss": 0.0858, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.9564836025238037, |
| "rewards/margins": 7.9727783203125, |
| "rewards/rejected": -10.929262161254883, |
| "step": 1200 |
| }, |
| { |
| "epoch": 2.5696, |
| "grad_norm": 2.85349178314209, |
| "learning_rate": 2.683162634456515e-07, |
| "logits/chosen": -1.679058313369751, |
| "logits/rejected": -1.529710292816162, |
| "logps/chosen": -280.95941162109375, |
| "logps/rejected": -372.44256591796875, |
| "loss": 0.0859, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.4623801708221436, |
| "rewards/margins": 7.8578782081604, |
| "rewards/rejected": -10.320258140563965, |
| "step": 1205 |
| }, |
| { |
| "epoch": 2.5802666666666667, |
| "grad_norm": 3.7565083503723145, |
| "learning_rate": 2.5548865107314606e-07, |
| "logits/chosen": -1.6767339706420898, |
| "logits/rejected": -1.5683711767196655, |
| "logps/chosen": -302.63616943359375, |
| "logps/rejected": -418.7508850097656, |
| "loss": 0.0827, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.8508572578430176, |
| "rewards/margins": 8.236178398132324, |
| "rewards/rejected": -11.0870361328125, |
| "step": 1210 |
| }, |
| { |
| "epoch": 2.590933333333333, |
| "grad_norm": 3.984247922897339, |
| "learning_rate": 2.429587038698511e-07, |
| "logits/chosen": -1.7127422094345093, |
| "logits/rejected": -1.6515560150146484, |
| "logps/chosen": -257.34368896484375, |
| "logps/rejected": -387.74627685546875, |
| "loss": 0.0789, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.859555959701538, |
| "rewards/margins": 8.351922988891602, |
| "rewards/rejected": -11.211479187011719, |
| "step": 1215 |
| }, |
| { |
| "epoch": 2.6016, |
| "grad_norm": 2.0166029930114746, |
| "learning_rate": 2.307280835466144e-07, |
| "logits/chosen": -1.6319023370742798, |
| "logits/rejected": -1.5826950073242188, |
| "logps/chosen": -260.2449951171875, |
| "logps/rejected": -415.6009216308594, |
| "loss": 0.0724, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.7726876735687256, |
| "rewards/margins": 8.852060317993164, |
| "rewards/rejected": -11.624747276306152, |
| "step": 1220 |
| }, |
| { |
| "epoch": 2.6122666666666667, |
| "grad_norm": 1.6875503063201904, |
| "learning_rate": 2.1879841211780856e-07, |
| "logits/chosen": -1.6714298725128174, |
| "logits/rejected": -1.5990326404571533, |
| "logps/chosen": -252.33450317382812, |
| "logps/rejected": -385.48095703125, |
| "loss": 0.07, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.4542059898376465, |
| "rewards/margins": 8.378938674926758, |
| "rewards/rejected": -10.833145141601562, |
| "step": 1225 |
| }, |
| { |
| "epoch": 2.622933333333333, |
| "grad_norm": 3.2986953258514404, |
| "learning_rate": 2.0717127168622202e-07, |
| "logits/chosen": -1.6889839172363281, |
| "logits/rejected": -1.562585711479187, |
| "logps/chosen": -326.72442626953125, |
| "logps/rejected": -389.68267822265625, |
| "loss": 0.0875, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.467181444168091, |
| "rewards/margins": 8.251823425292969, |
| "rewards/rejected": -10.71900463104248, |
| "step": 1230 |
| }, |
| { |
| "epoch": 2.6336, |
| "grad_norm": 2.2151730060577393, |
| "learning_rate": 1.958482042332424e-07, |
| "logits/chosen": -1.699887990951538, |
| "logits/rejected": -1.5633143186569214, |
| "logps/chosen": -285.1319580078125, |
| "logps/rejected": -402.71697998046875, |
| "loss": 0.0948, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.002988338470459, |
| "rewards/margins": 7.786578178405762, |
| "rewards/rejected": -10.789566993713379, |
| "step": 1235 |
| }, |
| { |
| "epoch": 2.6442666666666668, |
| "grad_norm": 4.6926655769348145, |
| "learning_rate": 1.8483071141435937e-07, |
| "logits/chosen": -1.6601158380508423, |
| "logits/rejected": -1.5603011846542358, |
| "logps/chosen": -263.892333984375, |
| "logps/rejected": -367.2433166503906, |
| "loss": 0.1363, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.742337942123413, |
| "rewards/margins": 7.589631080627441, |
| "rewards/rejected": -10.331968307495117, |
| "step": 1240 |
| }, |
| { |
| "epoch": 2.654933333333333, |
| "grad_norm": 3.295954942703247, |
| "learning_rate": 1.741202543600176e-07, |
| "logits/chosen": -1.7194404602050781, |
| "logits/rejected": -1.5846221446990967, |
| "logps/chosen": -241.400146484375, |
| "logps/rejected": -396.5201721191406, |
| "loss": 0.0682, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.6485438346862793, |
| "rewards/margins": 9.085619926452637, |
| "rewards/rejected": -11.734164237976074, |
| "step": 1245 |
| }, |
| { |
| "epoch": 2.6656, |
| "grad_norm": 6.180679798126221, |
| "learning_rate": 1.6371825348184155e-07, |
| "logits/chosen": -1.6426560878753662, |
| "logits/rejected": -1.5631107091903687, |
| "logps/chosen": -276.7149353027344, |
| "logps/rejected": -389.96380615234375, |
| "loss": 0.1129, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.8253414630889893, |
| "rewards/margins": 8.019664764404297, |
| "rewards/rejected": -10.845005989074707, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.676266666666667, |
| "grad_norm": 0.9536997079849243, |
| "learning_rate": 1.5362608828426246e-07, |
| "logits/chosen": -1.7700952291488647, |
| "logits/rejected": -1.7249057292938232, |
| "logps/chosen": -256.5973815917969, |
| "logps/rejected": -408.5018005371094, |
| "loss": 0.0952, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.539684295654297, |
| "rewards/margins": 8.208810806274414, |
| "rewards/rejected": -10.748494148254395, |
| "step": 1255 |
| }, |
| { |
| "epoch": 2.686933333333333, |
| "grad_norm": 2.8590283393859863, |
| "learning_rate": 1.438450971815686e-07, |
| "logits/chosen": -1.7384529113769531, |
| "logits/rejected": -1.6048595905303955, |
| "logps/chosen": -244.7389678955078, |
| "logps/rejected": -378.0864562988281, |
| "loss": 0.0781, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.570819854736328, |
| "rewards/margins": 8.148038864135742, |
| "rewards/rejected": -10.71885871887207, |
| "step": 1260 |
| }, |
| { |
| "epoch": 2.6976, |
| "grad_norm": 3.8086490631103516, |
| "learning_rate": 1.3437657732040783e-07, |
| "logits/chosen": -1.6877682209014893, |
| "logits/rejected": -1.6243743896484375, |
| "logps/chosen": -265.04913330078125, |
| "logps/rejected": -389.056640625, |
| "loss": 0.1076, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.82427716255188, |
| "rewards/margins": 7.797641754150391, |
| "rewards/rejected": -10.621920585632324, |
| "step": 1265 |
| }, |
| { |
| "epoch": 2.708266666666667, |
| "grad_norm": 1.2340490818023682, |
| "learning_rate": 1.2522178440776022e-07, |
| "logits/chosen": -1.6442654132843018, |
| "logits/rejected": -1.5744202136993408, |
| "logps/chosen": -265.56964111328125, |
| "logps/rejected": -391.43963623046875, |
| "loss": 0.0637, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.7177109718322754, |
| "rewards/margins": 8.296258926391602, |
| "rewards/rejected": -11.013969421386719, |
| "step": 1270 |
| }, |
| { |
| "epoch": 2.718933333333333, |
| "grad_norm": 1.5882465839385986, |
| "learning_rate": 1.1638193254440655e-07, |
| "logits/chosen": -1.6713117361068726, |
| "logits/rejected": -1.57658052444458, |
| "logps/chosen": -279.78704833984375, |
| "logps/rejected": -401.1554870605469, |
| "loss": 0.0471, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.8568999767303467, |
| "rewards/margins": 9.124058723449707, |
| "rewards/rejected": -11.980958938598633, |
| "step": 1275 |
| }, |
| { |
| "epoch": 2.7296, |
| "grad_norm": 5.013668060302734, |
| "learning_rate": 1.0785819406391562e-07, |
| "logits/chosen": -1.638289213180542, |
| "logits/rejected": -1.6082589626312256, |
| "logps/chosen": -245.79031372070312, |
| "logps/rejected": -423.3377380371094, |
| "loss": 0.0595, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.737645387649536, |
| "rewards/margins": 9.465682983398438, |
| "rewards/rejected": -12.203328132629395, |
| "step": 1280 |
| }, |
| { |
| "epoch": 2.740266666666667, |
| "grad_norm": 4.975745677947998, |
| "learning_rate": 9.965169937717095e-08, |
| "logits/chosen": -1.6764233112335205, |
| "logits/rejected": -1.6596872806549072, |
| "logps/chosen": -258.89019775390625, |
| "logps/rejected": -408.6078186035156, |
| "loss": 0.0602, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.8898043632507324, |
| "rewards/margins": 8.435752868652344, |
| "rewards/rejected": -11.325556755065918, |
| "step": 1285 |
| }, |
| { |
| "epoch": 2.7509333333333332, |
| "grad_norm": 1.6985228061676025, |
| "learning_rate": 9.176353682245676e-08, |
| "logits/chosen": -1.7202489376068115, |
| "logits/rejected": -1.611146330833435, |
| "logps/chosen": -250.17483520507812, |
| "logps/rejected": -398.2389221191406, |
| "loss": 0.0759, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.7857604026794434, |
| "rewards/margins": 8.919150352478027, |
| "rewards/rejected": -11.704911231994629, |
| "step": 1290 |
| }, |
| { |
| "epoch": 2.7616, |
| "grad_norm": 3.58396577835083, |
| "learning_rate": 8.41947525211223e-08, |
| "logits/chosen": -1.5990982055664062, |
| "logits/rejected": -1.5592104196548462, |
| "logps/chosen": -269.1976623535156, |
| "logps/rejected": -397.72674560546875, |
| "loss": 0.0831, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.7801098823547363, |
| "rewards/margins": 8.252833366394043, |
| "rewards/rejected": -11.032944679260254, |
| "step": 1295 |
| }, |
| { |
| "epoch": 2.772266666666667, |
| "grad_norm": 3.4320976734161377, |
| "learning_rate": 7.694635023884789e-08, |
| "logits/chosen": -1.6502788066864014, |
| "logits/rejected": -1.634393334388733, |
| "logps/chosen": -268.66253662109375, |
| "logps/rejected": -408.7780456542969, |
| "loss": 0.0665, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.4980883598327637, |
| "rewards/margins": 8.380696296691895, |
| "rewards/rejected": -10.8787841796875, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.7829333333333333, |
| "grad_norm": 1.1601080894470215, |
| "learning_rate": 7.00192912525241e-08, |
| "logits/chosen": -1.707038164138794, |
| "logits/rejected": -1.5817816257476807, |
| "logps/chosen": -267.05767822265625, |
| "logps/rejected": -411.76776123046875, |
| "loss": 0.0579, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5934057235717773, |
| "rewards/margins": 9.068880081176758, |
| "rewards/rejected": -11.662285804748535, |
| "step": 1305 |
| }, |
| { |
| "epoch": 2.7936, |
| "grad_norm": 3.5436959266662598, |
| "learning_rate": 6.341449422277047e-08, |
| "logits/chosen": -1.6015123128890991, |
| "logits/rejected": -1.546696662902832, |
| "logps/chosen": -273.44683837890625, |
| "logps/rejected": -400.2070007324219, |
| "loss": 0.0551, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4368176460266113, |
| "rewards/margins": 8.266563415527344, |
| "rewards/rejected": -10.703381538391113, |
| "step": 1310 |
| }, |
| { |
| "epoch": 2.804266666666667, |
| "grad_norm": 2.056448459625244, |
| "learning_rate": 5.7132835072101486e-08, |
| "logits/chosen": -1.65818190574646, |
| "logits/rejected": -1.6212623119354248, |
| "logps/chosen": -271.24542236328125, |
| "logps/rejected": -416.03265380859375, |
| "loss": 0.0621, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.530935287475586, |
| "rewards/margins": 9.317224502563477, |
| "rewards/rejected": -11.848158836364746, |
| "step": 1315 |
| }, |
| { |
| "epoch": 2.8149333333333333, |
| "grad_norm": 1.1611863374710083, |
| "learning_rate": 5.117514686876379e-08, |
| "logits/chosen": -1.6843284368515015, |
| "logits/rejected": -1.570382833480835, |
| "logps/chosen": -247.26882934570312, |
| "logps/rejected": -382.57171630859375, |
| "loss": 0.0613, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.619004487991333, |
| "rewards/margins": 8.069233894348145, |
| "rewards/rejected": -10.688237190246582, |
| "step": 1320 |
| }, |
| { |
| "epoch": 2.8256, |
| "grad_norm": 3.386897087097168, |
| "learning_rate": 4.5542219716255274e-08, |
| "logits/chosen": -1.6479991674423218, |
| "logits/rejected": -1.647812843322754, |
| "logps/chosen": -264.9526062011719, |
| "logps/rejected": -393.6275939941406, |
| "loss": 0.0578, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.8374381065368652, |
| "rewards/margins": 8.005861282348633, |
| "rewards/rejected": -10.84329891204834, |
| "step": 1325 |
| }, |
| { |
| "epoch": 2.836266666666667, |
| "grad_norm": 2.9551398754119873, |
| "learning_rate": 4.023480064854174e-08, |
| "logits/chosen": -1.709986925125122, |
| "logits/rejected": -1.658769965171814, |
| "logps/chosen": -261.8425598144531, |
| "logps/rejected": -389.1148681640625, |
| "loss": 0.0814, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.9073290824890137, |
| "rewards/margins": 7.660365104675293, |
| "rewards/rejected": -10.567693710327148, |
| "step": 1330 |
| }, |
| { |
| "epoch": 2.8469333333333333, |
| "grad_norm": 5.993083953857422, |
| "learning_rate": 3.525359353098584e-08, |
| "logits/chosen": -1.7044435739517212, |
| "logits/rejected": -1.613206148147583, |
| "logps/chosen": -260.56781005859375, |
| "logps/rejected": -400.2635192871094, |
| "loss": 0.0947, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.7162492275238037, |
| "rewards/margins": 8.33190631866455, |
| "rewards/rejected": -11.048154830932617, |
| "step": 1335 |
| }, |
| { |
| "epoch": 2.8576, |
| "grad_norm": 0.7436990141868591, |
| "learning_rate": 3.059925896700094e-08, |
| "logits/chosen": -1.5562223196029663, |
| "logits/rejected": -1.5099818706512451, |
| "logps/chosen": -282.28704833984375, |
| "logps/rejected": -367.90142822265625, |
| "loss": 0.066, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5948193073272705, |
| "rewards/margins": 7.88361120223999, |
| "rewards/rejected": -10.478429794311523, |
| "step": 1340 |
| }, |
| { |
| "epoch": 2.8682666666666665, |
| "grad_norm": 1.7763005495071411, |
| "learning_rate": 2.6272414210443964e-08, |
| "logits/chosen": -1.6868610382080078, |
| "logits/rejected": -1.6632941961288452, |
| "logps/chosen": -266.9272766113281, |
| "logps/rejected": -396.6302795410156, |
| "loss": 0.1068, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.7057154178619385, |
| "rewards/margins": 8.647436141967773, |
| "rewards/rejected": -11.353150367736816, |
| "step": 1345 |
| }, |
| { |
| "epoch": 2.8789333333333333, |
| "grad_norm": 3.7743892669677734, |
| "learning_rate": 2.2273633083753632e-08, |
| "logits/chosen": -1.6585218906402588, |
| "logits/rejected": -1.613133430480957, |
| "logps/chosen": -289.80340576171875, |
| "logps/rejected": -405.9570007324219, |
| "loss": 0.0767, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.0055086612701416, |
| "rewards/margins": 8.358158111572266, |
| "rewards/rejected": -11.363667488098145, |
| "step": 1350 |
| }, |
| { |
| "epoch": 2.8895999999999997, |
| "grad_norm": 0.7690500020980835, |
| "learning_rate": 1.8603445901851647e-08, |
| "logits/chosen": -1.6909067630767822, |
| "logits/rejected": -1.5630600452423096, |
| "logps/chosen": -267.53607177734375, |
| "logps/rejected": -386.2433166503906, |
| "loss": 0.0566, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.2426962852478027, |
| "rewards/margins": 8.468097686767578, |
| "rewards/rejected": -10.710794448852539, |
| "step": 1355 |
| }, |
| { |
| "epoch": 2.9002666666666665, |
| "grad_norm": 4.579198837280273, |
| "learning_rate": 1.5262339401813375e-08, |
| "logits/chosen": -1.647733449935913, |
| "logits/rejected": -1.6072362661361694, |
| "logps/chosen": -268.2415466308594, |
| "logps/rejected": -409.1459655761719, |
| "loss": 0.0983, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.9298815727233887, |
| "rewards/margins": 8.94694709777832, |
| "rewards/rejected": -11.876829147338867, |
| "step": 1360 |
| }, |
| { |
| "epoch": 2.9109333333333334, |
| "grad_norm": 2.3603615760803223, |
| "learning_rate": 1.2250756678315601e-08, |
| "logits/chosen": -1.6706626415252686, |
| "logits/rejected": -1.598555326461792, |
| "logps/chosen": -275.09356689453125, |
| "logps/rejected": -386.44921875, |
| "loss": 0.0754, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.6181488037109375, |
| "rewards/margins": 7.823951721191406, |
| "rewards/rejected": -10.442100524902344, |
| "step": 1365 |
| }, |
| { |
| "epoch": 2.9215999999999998, |
| "grad_norm": 3.3083324432373047, |
| "learning_rate": 9.569097124875193e-09, |
| "logits/chosen": -1.6924657821655273, |
| "logits/rejected": -1.5973902940750122, |
| "logps/chosen": -266.7765197753906, |
| "logps/rejected": -410.7627868652344, |
| "loss": 0.0799, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.732797145843506, |
| "rewards/margins": 8.773340225219727, |
| "rewards/rejected": -11.506136894226074, |
| "step": 1370 |
| }, |
| { |
| "epoch": 2.9322666666666666, |
| "grad_norm": 4.005508899688721, |
| "learning_rate": 7.217716380881479e-09, |
| "logits/chosen": -1.6512973308563232, |
| "logits/rejected": -1.5719560384750366, |
| "logps/chosen": -287.5769958496094, |
| "logps/rejected": -392.5569763183594, |
| "loss": 0.0754, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.802158832550049, |
| "rewards/margins": 8.369941711425781, |
| "rewards/rejected": -11.172099113464355, |
| "step": 1375 |
| }, |
| { |
| "epoch": 2.9429333333333334, |
| "grad_norm": 2.5721302032470703, |
| "learning_rate": 5.196926284430359e-09, |
| "logits/chosen": -1.6722768545150757, |
| "logits/rejected": -1.5862112045288086, |
| "logps/chosen": -276.39044189453125, |
| "logps/rejected": -428.83062744140625, |
| "loss": 0.0756, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.7317731380462646, |
| "rewards/margins": 7.940549373626709, |
| "rewards/rejected": -10.672322273254395, |
| "step": 1380 |
| }, |
| { |
| "epoch": 2.9536, |
| "grad_norm": 1.1050063371658325, |
| "learning_rate": 3.5069948309712775e-09, |
| "logits/chosen": -1.563157320022583, |
| "logits/rejected": -1.5820673704147339, |
| "logps/chosen": -266.9634094238281, |
| "logps/rejected": -400.5401611328125, |
| "loss": 0.0924, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.8875529766082764, |
| "rewards/margins": 7.935434818267822, |
| "rewards/rejected": -10.822988510131836, |
| "step": 1385 |
| }, |
| { |
| "epoch": 2.9642666666666666, |
| "grad_norm": 2.0641963481903076, |
| "learning_rate": 2.1481461377634294e-09, |
| "logits/chosen": -1.6637370586395264, |
| "logits/rejected": -1.621468186378479, |
| "logps/chosen": -285.5246887207031, |
| "logps/rejected": -401.16314697265625, |
| "loss": 0.0639, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.8416502475738525, |
| "rewards/margins": 8.040994644165039, |
| "rewards/rejected": -10.882644653320312, |
| "step": 1390 |
| }, |
| { |
| "epoch": 2.9749333333333334, |
| "grad_norm": 1.0203123092651367, |
| "learning_rate": 1.120560414154259e-09, |
| "logits/chosen": -1.6394908428192139, |
| "logits/rejected": -1.5650370121002197, |
| "logps/chosen": -250.5342254638672, |
| "logps/rejected": -398.98797607421875, |
| "loss": 0.0769, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.790039539337158, |
| "rewards/margins": 8.689876556396484, |
| "rewards/rejected": -11.4799165725708, |
| "step": 1395 |
| }, |
| { |
| "epoch": 2.9856, |
| "grad_norm": 3.153134346008301, |
| "learning_rate": 4.2437393768079983e-10, |
| "logits/chosen": -1.6397968530654907, |
| "logits/rejected": -1.585146427154541, |
| "logps/chosen": -279.3368225097656, |
| "logps/rejected": -408.64739990234375, |
| "loss": 0.0813, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.9198312759399414, |
| "rewards/margins": 8.351078987121582, |
| "rewards/rejected": -11.270910263061523, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.9962666666666666, |
| "grad_norm": 4.959558010101318, |
| "learning_rate": 5.967903599635171e-11, |
| "logits/chosen": -1.687861680984497, |
| "logits/rejected": -1.4993536472320557, |
| "logps/chosen": -250.8742218017578, |
| "logps/rejected": -405.6380920410156, |
| "loss": 0.0809, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.5417988300323486, |
| "rewards/margins": 8.413880348205566, |
| "rewards/rejected": -10.955678939819336, |
| "step": 1405 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 1407, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
