| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 363, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.04155844155844156, |
| "grad_norm": 3.118415117263794, |
| "learning_rate": 1.8181818181818183e-06, |
| "logits/chosen": -0.8127259016036987, |
| "logits/rejected": -0.7634922862052917, |
| "logps/chosen": -228.6646728515625, |
| "logps/rejected": -271.3758239746094, |
| "loss": 0.6923, |
| "rewards/accuracies": 0.36250001192092896, |
| "rewards/chosen": -0.0006901646265760064, |
| "rewards/margins": 0.001700992463156581, |
| "rewards/rejected": -0.002391157206147909, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.08311688311688312, |
| "grad_norm": 3.1249163150787354, |
| "learning_rate": 4.0909090909090915e-06, |
| "logits/chosen": -0.8295533061027527, |
| "logits/rejected": -0.7937226891517639, |
| "logps/chosen": -219.41513061523438, |
| "logps/rejected": -279.0150146484375, |
| "loss": 0.6819, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.0035048150457441807, |
| "rewards/margins": 0.022944308817386627, |
| "rewards/rejected": -0.02644912339746952, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.12467532467532468, |
| "grad_norm": 3.300832748413086, |
| "learning_rate": 4.999103931944114e-06, |
| "logits/chosen": -0.9027387499809265, |
| "logits/rejected": -0.7889591455459595, |
| "logps/chosen": -222.32571411132812, |
| "logps/rejected": -293.6187744140625, |
| "loss": 0.652, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.0009602688951417804, |
| "rewards/margins": 0.0870375782251358, |
| "rewards/rejected": -0.08799784630537033, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.16623376623376623, |
| "grad_norm": 2.7533063888549805, |
| "learning_rate": 4.993630286525634e-06, |
| "logits/chosen": -0.8960437774658203, |
| "logits/rejected": -0.824598491191864, |
| "logps/chosen": -232.5225830078125, |
| "logps/rejected": -296.52178955078125, |
| "loss": 0.6123, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.017542826011776924, |
| "rewards/margins": 0.18044860661029816, |
| "rewards/rejected": -0.19799141585826874, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.2077922077922078, |
| "grad_norm": 2.7143568992614746, |
| "learning_rate": 4.983191696363525e-06, |
| "logits/chosen": -0.923931896686554, |
| "logits/rejected": -0.8416165113449097, |
| "logps/chosen": -220.6460418701172, |
| "logps/rejected": -273.8211364746094, |
| "loss": 0.5717, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -0.03508078306913376, |
| "rewards/margins": 0.2878609597682953, |
| "rewards/rejected": -0.32294172048568726, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.24935064935064935, |
| "grad_norm": 2.480844259262085, |
| "learning_rate": 4.9678089452217785e-06, |
| "logits/chosen": -0.9296607971191406, |
| "logits/rejected": -0.7941066026687622, |
| "logps/chosen": -239.7672882080078, |
| "logps/rejected": -307.1377258300781, |
| "loss": 0.5489, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.08770466595888138, |
| "rewards/margins": 0.3718782067298889, |
| "rewards/rejected": -0.4595828652381897, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.2909090909090909, |
| "grad_norm": 2.883957862854004, |
| "learning_rate": 4.947512660940388e-06, |
| "logits/chosen": -0.9467341303825378, |
| "logits/rejected": -0.8958338499069214, |
| "logps/chosen": -236.58203125, |
| "logps/rejected": -299.7516174316406, |
| "loss": 0.5295, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.06386536359786987, |
| "rewards/margins": 0.455936998128891, |
| "rewards/rejected": -0.519802451133728, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.33246753246753247, |
| "grad_norm": 2.9537580013275146, |
| "learning_rate": 4.922343254453769e-06, |
| "logits/chosen": -0.9613215327262878, |
| "logits/rejected": -0.8938771486282349, |
| "logps/chosen": -209.87484741210938, |
| "logps/rejected": -261.8564453125, |
| "loss": 0.5066, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -0.04000057652592659, |
| "rewards/margins": 0.5617119669914246, |
| "rewards/rejected": -0.6017125844955444, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.37402597402597404, |
| "grad_norm": 3.0379507541656494, |
| "learning_rate": 4.8923508393305224e-06, |
| "logits/chosen": -0.9870550036430359, |
| "logits/rejected": -0.8364689946174622, |
| "logps/chosen": -224.9286651611328, |
| "logps/rejected": -275.58392333984375, |
| "loss": 0.4765, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": -0.03620234876871109, |
| "rewards/margins": 0.6848794221878052, |
| "rewards/rejected": -0.7210818529129028, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.4155844155844156, |
| "grad_norm": 2.38826847076416, |
| "learning_rate": 4.857595131994781e-06, |
| "logits/chosen": -1.0010199546813965, |
| "logits/rejected": -0.9080400466918945, |
| "logps/chosen": -250.68527221679688, |
| "logps/rejected": -344.46478271484375, |
| "loss": 0.4251, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -0.09372053295373917, |
| "rewards/margins": 0.9252897500991821, |
| "rewards/rejected": -1.019010305404663, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.45714285714285713, |
| "grad_norm": 2.718043804168701, |
| "learning_rate": 4.818145332827762e-06, |
| "logits/chosen": -1.1310384273529053, |
| "logits/rejected": -0.9854855537414551, |
| "logps/chosen": -231.0263214111328, |
| "logps/rejected": -298.1824645996094, |
| "loss": 0.4418, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.12084142863750458, |
| "rewards/margins": 0.9410508275032043, |
| "rewards/rejected": -1.0618922710418701, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.4987012987012987, |
| "grad_norm": 2.1554527282714844, |
| "learning_rate": 4.7740799883862966e-06, |
| "logits/chosen": -0.9824150800704956, |
| "logits/rejected": -0.9336239099502563, |
| "logps/chosen": -211.98684692382812, |
| "logps/rejected": -292.4842224121094, |
| "loss": 0.4413, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.06550006568431854, |
| "rewards/margins": 0.9869768023490906, |
| "rewards/rejected": -1.0524768829345703, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.5402597402597402, |
| "grad_norm": 2.788320302963257, |
| "learning_rate": 4.725486835012632e-06, |
| "logits/chosen": -1.1025915145874023, |
| "logits/rejected": -1.0172704458236694, |
| "logps/chosen": -228.7750244140625, |
| "logps/rejected": -302.1152648925781, |
| "loss": 0.4106, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.13185444474220276, |
| "rewards/margins": 1.0933724641799927, |
| "rewards/rejected": -1.225226879119873, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.5818181818181818, |
| "grad_norm": 2.9373393058776855, |
| "learning_rate": 4.672462624146918e-06, |
| "logits/chosen": -1.133795976638794, |
| "logits/rejected": -0.9968055486679077, |
| "logps/chosen": -224.1367950439453, |
| "logps/rejected": -320.6668701171875, |
| "loss": 0.3655, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.16425569355487823, |
| "rewards/margins": 1.38435959815979, |
| "rewards/rejected": -1.5486153364181519, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.6233766233766234, |
| "grad_norm": 2.555952548980713, |
| "learning_rate": 4.615112929690163e-06, |
| "logits/chosen": -1.009514570236206, |
| "logits/rejected": -0.9566370248794556, |
| "logps/chosen": -252.9624481201172, |
| "logps/rejected": -294.30816650390625, |
| "loss": 0.403, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.15342703461647034, |
| "rewards/margins": 1.1806553602218628, |
| "rewards/rejected": -1.3340823650360107, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.6649350649350649, |
| "grad_norm": 2.2290637493133545, |
| "learning_rate": 4.553551937801229e-06, |
| "logits/chosen": -1.0907081365585327, |
| "logits/rejected": -1.0329780578613281, |
| "logps/chosen": -233.9783172607422, |
| "logps/rejected": -298.1121826171875, |
| "loss": 0.3585, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.1846320629119873, |
| "rewards/margins": 1.4989664554595947, |
| "rewards/rejected": -1.6835987567901611, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.7064935064935065, |
| "grad_norm": 2.9799797534942627, |
| "learning_rate": 4.487902219546383e-06, |
| "logits/chosen": -1.2056300640106201, |
| "logits/rejected": -1.0564621686935425, |
| "logps/chosen": -225.80825805664062, |
| "logps/rejected": -286.21380615234375, |
| "loss": 0.3455, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.1425703465938568, |
| "rewards/margins": 1.559043526649475, |
| "rewards/rejected": -1.7016140222549438, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.7480519480519481, |
| "grad_norm": 4.232228755950928, |
| "learning_rate": 4.418294486854064e-06, |
| "logits/chosen": -1.1301639080047607, |
| "logits/rejected": -0.9990237951278687, |
| "logps/chosen": -235.9337158203125, |
| "logps/rejected": -288.6039733886719, |
| "loss": 0.3599, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.130110502243042, |
| "rewards/margins": 1.5865774154663086, |
| "rewards/rejected": -1.7166879177093506, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.7896103896103897, |
| "grad_norm": 3.81459641456604, |
| "learning_rate": 4.3448673322607915e-06, |
| "logits/chosen": -1.1437747478485107, |
| "logits/rejected": -1.0269885063171387, |
| "logps/chosen": -230.104248046875, |
| "logps/rejected": -305.1618347167969, |
| "loss": 0.3399, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.08539257198572159, |
| "rewards/margins": 1.8511079549789429, |
| "rewards/rejected": -1.9365005493164062, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.8311688311688312, |
| "grad_norm": 2.949822425842285, |
| "learning_rate": 4.267766952966369e-06, |
| "logits/chosen": -1.1605863571166992, |
| "logits/rejected": -1.0450435876846313, |
| "logps/chosen": -222.7265625, |
| "logps/rejected": -324.3167419433594, |
| "loss": 0.3465, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.04091508686542511, |
| "rewards/margins": 1.9012877941131592, |
| "rewards/rejected": -1.9422029256820679, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.8727272727272727, |
| "grad_norm": 2.7308614253997803, |
| "learning_rate": 4.187146859747826e-06, |
| "logits/chosen": -1.104167103767395, |
| "logits/rejected": -1.0075145959854126, |
| "logps/chosen": -228.25424194335938, |
| "logps/rejected": -315.0980529785156, |
| "loss": 0.3543, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.004563205409795046, |
| "rewards/margins": 1.9307657480239868, |
| "rewards/rejected": -1.9353290796279907, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.9142857142857143, |
| "grad_norm": 2.497864007949829, |
| "learning_rate": 4.103167571311652e-06, |
| "logits/chosen": -1.2123258113861084, |
| "logits/rejected": -1.0642549991607666, |
| "logps/chosen": -220.7777862548828, |
| "logps/rejected": -307.74310302734375, |
| "loss": 0.31, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.08179270476102829, |
| "rewards/margins": 1.9646203517913818, |
| "rewards/rejected": -2.046412944793701, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.9558441558441558, |
| "grad_norm": 2.9862191677093506, |
| "learning_rate": 4.015996294692889e-06, |
| "logits/chosen": -1.1412394046783447, |
| "logits/rejected": -1.1184909343719482, |
| "logps/chosen": -234.40109252929688, |
| "logps/rejected": -285.2551574707031, |
| "loss": 0.3687, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.0041265422478318214, |
| "rewards/margins": 1.6963088512420654, |
| "rewards/rejected": -1.6921823024749756, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.9974025974025974, |
| "grad_norm": 3.2241032123565674, |
| "learning_rate": 3.9258065923374104e-06, |
| "logits/chosen": -1.2083184719085693, |
| "logits/rejected": -1.087278962135315, |
| "logps/chosen": -208.6204376220703, |
| "logps/rejected": -345.3356018066406, |
| "loss": 0.3047, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.12136807292699814, |
| "rewards/margins": 2.2191343307495117, |
| "rewards/rejected": -2.097766160964966, |
| "step": 120 |
| }, |
| { |
| "epoch": 1.0332467532467533, |
| "grad_norm": 2.3973424434661865, |
| "learning_rate": 3.832778036530276e-06, |
| "logits/chosen": -1.0659514665603638, |
| "logits/rejected": -0.989536464214325, |
| "logps/chosen": -229.2854766845703, |
| "logps/rejected": -303.9345703125, |
| "loss": 0.2628, |
| "rewards/accuracies": 0.8913043737411499, |
| "rewards/chosen": 0.2346079796552658, |
| "rewards/margins": 2.2249648571014404, |
| "rewards/rejected": -1.9903569221496582, |
| "step": 125 |
| }, |
| { |
| "epoch": 1.0748051948051949, |
| "grad_norm": 2.8788726329803467, |
| "learning_rate": 3.7370958518581694e-06, |
| "logits/chosen": -1.17106294631958, |
| "logits/rejected": -1.0628446340560913, |
| "logps/chosen": -228.1200714111328, |
| "logps/rejected": -287.6686096191406, |
| "loss": 0.3305, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.18449993431568146, |
| "rewards/margins": 1.797311782836914, |
| "rewards/rejected": -1.612811803817749, |
| "step": 130 |
| }, |
| { |
| "epoch": 1.1163636363636364, |
| "grad_norm": 1.94839346408844, |
| "learning_rate": 3.638950546417824e-06, |
| "logits/chosen": -1.198904275894165, |
| "logits/rejected": -1.0711965560913086, |
| "logps/chosen": -229.5878448486328, |
| "logps/rejected": -309.1488952636719, |
| "loss": 0.2589, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.2590901851654053, |
| "rewards/margins": 2.1628074645996094, |
| "rewards/rejected": -1.9037176370620728, |
| "step": 135 |
| }, |
| { |
| "epoch": 1.157922077922078, |
| "grad_norm": 2.6632540225982666, |
| "learning_rate": 3.5385375325047167e-06, |
| "logits/chosen": -1.2561719417572021, |
| "logits/rejected": -1.0732624530792236, |
| "logps/chosen": -213.3624267578125, |
| "logps/rejected": -296.18035888671875, |
| "loss": 0.2866, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.18848955631256104, |
| "rewards/margins": 2.0759098529815674, |
| "rewards/rejected": -1.887420415878296, |
| "step": 140 |
| }, |
| { |
| "epoch": 1.1994805194805194, |
| "grad_norm": 2.6034018993377686, |
| "learning_rate": 3.4360567375372426e-06, |
| "logits/chosen": -1.2088276147842407, |
| "logits/rejected": -1.0747075080871582, |
| "logps/chosen": -224.8641815185547, |
| "logps/rejected": -305.76824951171875, |
| "loss": 0.2851, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.09721609950065613, |
| "rewards/margins": 2.325167179107666, |
| "rewards/rejected": -2.2279508113861084, |
| "step": 145 |
| }, |
| { |
| "epoch": 1.241038961038961, |
| "grad_norm": 2.1052863597869873, |
| "learning_rate": 3.3317122059910607e-06, |
| "logits/chosen": -1.2572283744812012, |
| "logits/rejected": -1.1288259029388428, |
| "logps/chosen": -223.6044158935547, |
| "logps/rejected": -327.5605163574219, |
| "loss": 0.2647, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.09010759741067886, |
| "rewards/margins": 2.6917688846588135, |
| "rewards/rejected": -2.601661205291748, |
| "step": 150 |
| }, |
| { |
| "epoch": 1.2825974025974025, |
| "grad_norm": 2.5198380947113037, |
| "learning_rate": 3.225711693136156e-06, |
| "logits/chosen": -1.2977545261383057, |
| "logits/rejected": -1.166739821434021, |
| "logps/chosen": -236.06753540039062, |
| "logps/rejected": -299.90655517578125, |
| "loss": 0.2904, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.016246700659394264, |
| "rewards/margins": 2.2283177375793457, |
| "rewards/rejected": -2.212070941925049, |
| "step": 155 |
| }, |
| { |
| "epoch": 1.324155844155844, |
| "grad_norm": 2.1668765544891357, |
| "learning_rate": 3.118266251385539e-06, |
| "logits/chosen": -1.1280694007873535, |
| "logits/rejected": -1.0987536907196045, |
| "logps/chosen": -250.9461669921875, |
| "logps/rejected": -305.9465026855469, |
| "loss": 0.29, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.01789233088493347, |
| "rewards/margins": 2.355379104614258, |
| "rewards/rejected": -2.337486743927002, |
| "step": 160 |
| }, |
| { |
| "epoch": 1.3657142857142857, |
| "grad_norm": 2.4839677810668945, |
| "learning_rate": 3.0095898100791472e-06, |
| "logits/chosen": -1.2380702495574951, |
| "logits/rejected": -1.0927748680114746, |
| "logps/chosen": -226.11709594726562, |
| "logps/rejected": -324.3223876953125, |
| "loss": 0.2639, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.16840746998786926, |
| "rewards/margins": 2.606884479522705, |
| "rewards/rejected": -2.775291919708252, |
| "step": 165 |
| }, |
| { |
| "epoch": 1.4072727272727272, |
| "grad_norm": 2.544842481613159, |
| "learning_rate": 2.8998987495396506e-06, |
| "logits/chosen": -1.3116706609725952, |
| "logits/rejected": -1.1664602756500244, |
| "logps/chosen": -223.4804229736328, |
| "logps/rejected": -307.60455322265625, |
| "loss": 0.2416, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.16324956715106964, |
| "rewards/margins": 2.7645745277404785, |
| "rewards/rejected": -2.9278244972229004, |
| "step": 170 |
| }, |
| { |
| "epoch": 1.4488311688311688, |
| "grad_norm": 2.2766852378845215, |
| "learning_rate": 2.7894114702482254e-06, |
| "logits/chosen": -1.2931270599365234, |
| "logits/rejected": -1.1453607082366943, |
| "logps/chosen": -225.3311004638672, |
| "logps/rejected": -329.5317687988281, |
| "loss": 0.275, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.1200091615319252, |
| "rewards/margins": 2.7051124572753906, |
| "rewards/rejected": -2.8251214027404785, |
| "step": 175 |
| }, |
| { |
| "epoch": 1.4903896103896104, |
| "grad_norm": 2.1356124877929688, |
| "learning_rate": 2.678347957998081e-06, |
| "logits/chosen": -1.301461935043335, |
| "logits/rejected": -1.1489789485931396, |
| "logps/chosen": -223.24655151367188, |
| "logps/rejected": -314.0998229980469, |
| "loss": 0.2134, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.17566713690757751, |
| "rewards/margins": 2.733579635620117, |
| "rewards/rejected": -2.9092462062835693, |
| "step": 180 |
| }, |
| { |
| "epoch": 1.531948051948052, |
| "grad_norm": 2.0670270919799805, |
| "learning_rate": 2.566929345891562e-06, |
| "logits/chosen": -1.1717848777770996, |
| "logits/rejected": -1.0892704725265503, |
| "logps/chosen": -230.27536010742188, |
| "logps/rejected": -311.3392333984375, |
| "loss": 0.2352, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.049989253282547, |
| "rewards/margins": 2.9402599334716797, |
| "rewards/rejected": -2.890270709991455, |
| "step": 185 |
| }, |
| { |
| "epoch": 1.5735064935064935, |
| "grad_norm": 2.3355753421783447, |
| "learning_rate": 2.455377474052885e-06, |
| "logits/chosen": -1.1918342113494873, |
| "logits/rejected": -1.0741770267486572, |
| "logps/chosen": -239.7108154296875, |
| "logps/rejected": -299.80035400390625, |
| "loss": 0.2795, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.24867768585681915, |
| "rewards/margins": 2.535564661026001, |
| "rewards/rejected": -2.2868869304656982, |
| "step": 190 |
| }, |
| { |
| "epoch": 1.615064935064935, |
| "grad_norm": 2.0783815383911133, |
| "learning_rate": 2.3439144479331776e-06, |
| "logits/chosen": -1.3459298610687256, |
| "logits/rejected": -1.2816716432571411, |
| "logps/chosen": -219.70138549804688, |
| "logps/rejected": -328.4333801269531, |
| "loss": 0.2118, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.07029136270284653, |
| "rewards/margins": 3.2027976512908936, |
| "rewards/rejected": -3.1325061321258545, |
| "step": 195 |
| }, |
| { |
| "epoch": 1.6566233766233767, |
| "grad_norm": 1.9809454679489136, |
| "learning_rate": 2.2327621960872187e-06, |
| "logits/chosen": -1.2197625637054443, |
| "logits/rejected": -1.0895978212356567, |
| "logps/chosen": -229.3469696044922, |
| "logps/rejected": -329.76580810546875, |
| "loss": 0.253, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.10655846446752548, |
| "rewards/margins": 2.7795863151550293, |
| "rewards/rejected": -2.673027515411377, |
| "step": 200 |
| }, |
| { |
| "epoch": 1.6981818181818182, |
| "grad_norm": 2.6429646015167236, |
| "learning_rate": 2.1221420283023984e-06, |
| "logits/chosen": -1.27398681640625, |
| "logits/rejected": -1.0956755876541138, |
| "logps/chosen": -229.72119140625, |
| "logps/rejected": -311.6751403808594, |
| "loss": 0.2748, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.03542603179812431, |
| "rewards/margins": 2.605504035949707, |
| "rewards/rejected": -2.570078134536743, |
| "step": 205 |
| }, |
| { |
| "epoch": 1.7397402597402598, |
| "grad_norm": 2.050103187561035, |
| "learning_rate": 2.01227419495968e-06, |
| "logits/chosen": -1.2580993175506592, |
| "logits/rejected": -1.2016228437423706, |
| "logps/chosen": -225.03189086914062, |
| "logps/rejected": -315.2294616699219, |
| "loss": 0.2545, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.12700282037258148, |
| "rewards/margins": 2.712761402130127, |
| "rewards/rejected": -2.585758686065674, |
| "step": 210 |
| }, |
| { |
| "epoch": 1.7812987012987014, |
| "grad_norm": 2.6419899463653564, |
| "learning_rate": 1.9033774485038794e-06, |
| "logits/chosen": -1.2604008913040161, |
| "logits/rejected": -1.1434435844421387, |
| "logps/chosen": -227.2825164794922, |
| "logps/rejected": -290.4469299316406, |
| "loss": 0.2702, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.12941348552703857, |
| "rewards/margins": 2.5677075386047363, |
| "rewards/rejected": -2.438293933868408, |
| "step": 215 |
| }, |
| { |
| "epoch": 1.822857142857143, |
| "grad_norm": 2.732487440109253, |
| "learning_rate": 1.7956686078964257e-06, |
| "logits/chosen": -1.289029836654663, |
| "logits/rejected": -1.2009754180908203, |
| "logps/chosen": -225.47366333007812, |
| "logps/rejected": -300.55572509765625, |
| "loss": 0.2567, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.09105754643678665, |
| "rewards/margins": 2.691190719604492, |
| "rewards/rejected": -2.600132942199707, |
| "step": 220 |
| }, |
| { |
| "epoch": 1.8644155844155845, |
| "grad_norm": 2.813246726989746, |
| "learning_rate": 1.6893621269177821e-06, |
| "logits/chosen": -1.2733620405197144, |
| "logits/rejected": -1.145566463470459, |
| "logps/chosen": -238.2940673828125, |
| "logps/rejected": -295.91827392578125, |
| "loss": 0.2587, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.13102269172668457, |
| "rewards/margins": 2.568708896636963, |
| "rewards/rejected": -2.4376862049102783, |
| "step": 225 |
| }, |
| { |
| "epoch": 1.905974025974026, |
| "grad_norm": 2.8184409141540527, |
| "learning_rate": 1.5846696671790524e-06, |
| "logits/chosen": -1.233093023300171, |
| "logits/rejected": -1.1369177103042603, |
| "logps/chosen": -209.8083953857422, |
| "logps/rejected": -361.1885986328125, |
| "loss": 0.2688, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.050070393830537796, |
| "rewards/margins": 2.9840142726898193, |
| "rewards/rejected": -2.9339439868927, |
| "step": 230 |
| }, |
| { |
| "epoch": 1.9475324675324677, |
| "grad_norm": 2.254124641418457, |
| "learning_rate": 1.4817996766929532e-06, |
| "logits/chosen": -1.1859557628631592, |
| "logits/rejected": -1.1998345851898193, |
| "logps/chosen": -212.0352783203125, |
| "logps/rejected": -320.67767333984375, |
| "loss": 0.2405, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.10705818235874176, |
| "rewards/margins": 3.07060170173645, |
| "rewards/rejected": -2.96354341506958, |
| "step": 235 |
| }, |
| { |
| "epoch": 1.9890909090909092, |
| "grad_norm": 2.4340288639068604, |
| "learning_rate": 1.380956974843219e-06, |
| "logits/chosen": -1.3080205917358398, |
| "logits/rejected": -1.1855772733688354, |
| "logps/chosen": -234.13052368164062, |
| "logps/rejected": -322.0392150878906, |
| "loss": 0.2689, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.0906623974442482, |
| "rewards/margins": 2.882385730743408, |
| "rewards/rejected": -2.7917237281799316, |
| "step": 240 |
| }, |
| { |
| "epoch": 2.024935064935065, |
| "grad_norm": 2.381237030029297, |
| "learning_rate": 1.282342344578782e-06, |
| "logits/chosen": -1.2758800983428955, |
| "logits/rejected": -1.1762343645095825, |
| "logps/chosen": -210.26197814941406, |
| "logps/rejected": -304.9644775390625, |
| "loss": 0.2013, |
| "rewards/accuracies": 0.9347826242446899, |
| "rewards/chosen": 0.18103013932704926, |
| "rewards/margins": 3.0971362590789795, |
| "rewards/rejected": -2.9161057472229004, |
| "step": 245 |
| }, |
| { |
| "epoch": 2.0664935064935066, |
| "grad_norm": 1.9230319261550903, |
| "learning_rate": 1.186152132644718e-06, |
| "logits/chosen": -1.2861894369125366, |
| "logits/rejected": -1.1287521123886108, |
| "logps/chosen": -218.40707397460938, |
| "logps/rejected": -319.4270935058594, |
| "loss": 0.224, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.07273466885089874, |
| "rewards/margins": 3.143413543701172, |
| "rewards/rejected": -3.070678949356079, |
| "step": 250 |
| }, |
| { |
| "epoch": 2.108051948051948, |
| "grad_norm": 2.3148598670959473, |
| "learning_rate": 1.0925778586458813e-06, |
| "logits/chosen": -1.3198808431625366, |
| "logits/rejected": -1.221094012260437, |
| "logps/chosen": -214.1598663330078, |
| "logps/rejected": -299.68084716796875, |
| "loss": 0.2052, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.11255063116550446, |
| "rewards/margins": 3.0849621295928955, |
| "rewards/rejected": -2.972411632537842, |
| "step": 255 |
| }, |
| { |
| "epoch": 2.1496103896103897, |
| "grad_norm": 2.569136142730713, |
| "learning_rate": 1.0018058337216327e-06, |
| "logits/chosen": -1.2940465211868286, |
| "logits/rejected": -1.2342337369918823, |
| "logps/chosen": -226.5349578857422, |
| "logps/rejected": -316.8858337402344, |
| "loss": 0.2304, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.11650697886943817, |
| "rewards/margins": 2.7404751777648926, |
| "rewards/rejected": -2.6239686012268066, |
| "step": 260 |
| }, |
| { |
| "epoch": 2.1911688311688313, |
| "grad_norm": 1.9575142860412598, |
| "learning_rate": 9.140167895908867e-07, |
| "logits/chosen": -1.186423659324646, |
| "logits/rejected": -1.1101840734481812, |
| "logps/chosen": -224.22244262695312, |
| "logps/rejected": -320.3080139160156, |
| "loss": 0.2132, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.07010473310947418, |
| "rewards/margins": 2.980109691619873, |
| "rewards/rejected": -2.9100048542022705, |
| "step": 265 |
| }, |
| { |
| "epoch": 2.232727272727273, |
| "grad_norm": 1.8908374309539795, |
| "learning_rate": 8.293855187060601e-07, |
| "logits/chosen": -1.2614704370498657, |
| "logits/rejected": -1.1322941780090332, |
| "logps/chosen": -245.6070098876953, |
| "logps/rejected": -335.3045654296875, |
| "loss": 0.2018, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.18193891644477844, |
| "rewards/margins": 3.2319583892822266, |
| "rewards/rejected": -3.0500195026397705, |
| "step": 270 |
| }, |
| { |
| "epoch": 2.2742857142857145, |
| "grad_norm": 1.9720425605773926, |
| "learning_rate": 7.480805262324112e-07, |
| "logits/chosen": -1.2367236614227295, |
| "logits/rejected": -1.1424002647399902, |
| "logps/chosen": -225.2272491455078, |
| "logps/rejected": -308.2543640136719, |
| "loss": 0.2476, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.06707721948623657, |
| "rewards/margins": 2.86738920211792, |
| "rewards/rejected": -2.8003122806549072, |
| "step": 275 |
| }, |
| { |
| "epoch": 2.315844155844156, |
| "grad_norm": 2.199530601501465, |
| "learning_rate": 6.70263694545687e-07, |
| "logits/chosen": -1.2416431903839111, |
| "logits/rejected": -1.1705714464187622, |
| "logps/chosen": -216.54519653320312, |
| "logps/rejected": -313.75299072265625, |
| "loss": 0.2173, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.14036111533641815, |
| "rewards/margins": 3.161411762237549, |
| "rewards/rejected": -3.0210506916046143, |
| "step": 280 |
| }, |
| { |
| "epoch": 2.357402597402597, |
| "grad_norm": 2.594045639038086, |
| "learning_rate": 5.960899609160725e-07, |
| "logits/chosen": -1.3122602701187134, |
| "logits/rejected": -1.1919419765472412, |
| "logps/chosen": -229.4156494140625, |
| "logps/rejected": -325.54656982421875, |
| "loss": 0.2024, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.0318605899810791, |
| "rewards/margins": 3.3778507709503174, |
| "rewards/rejected": -3.3459904193878174, |
| "step": 285 |
| }, |
| { |
| "epoch": 2.3989610389610387, |
| "grad_norm": 2.318593740463257, |
| "learning_rate": 5.257070090202101e-07, |
| "logits/chosen": -1.296639084815979, |
| "logits/rejected": -1.1731235980987549, |
| "logps/chosen": -223.69839477539062, |
| "logps/rejected": -317.9754333496094, |
| "loss": 0.2178, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.005478203296661377, |
| "rewards/margins": 3.3782591819763184, |
| "rewards/rejected": -3.383737564086914, |
| "step": 290 |
| }, |
| { |
| "epoch": 2.4405194805194803, |
| "grad_norm": 2.6067514419555664, |
| "learning_rate": 4.592549748954858e-07, |
| "logits/chosen": -1.2447936534881592, |
| "logits/rejected": -1.1643249988555908, |
| "logps/chosen": -237.61068725585938, |
| "logps/rejected": -288.54351806640625, |
| "loss": 0.2758, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.03469625487923622, |
| "rewards/margins": 2.545546054840088, |
| "rewards/rejected": -2.510849952697754, |
| "step": 295 |
| }, |
| { |
| "epoch": 2.482077922077922, |
| "grad_norm": 1.8700066804885864, |
| "learning_rate": 3.9686616792204677e-07, |
| "logits/chosen": -1.2930142879486084, |
| "logits/rejected": -1.1466633081436157, |
| "logps/chosen": -240.37625122070312, |
| "logps/rejected": -333.6015625, |
| "loss": 0.1731, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.03592072054743767, |
| "rewards/margins": 3.346038341522217, |
| "rewards/rejected": -3.3819591999053955, |
| "step": 300 |
| }, |
| { |
| "epoch": 2.5236363636363635, |
| "grad_norm": 2.5204155445098877, |
| "learning_rate": 3.386648073881019e-07, |
| "logits/chosen": -1.2621270418167114, |
| "logits/rejected": -1.1422834396362305, |
| "logps/chosen": -232.3268585205078, |
| "logps/rejected": -293.70245361328125, |
| "loss": 0.2533, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.11021828651428223, |
| "rewards/margins": 2.717705249786377, |
| "rewards/rejected": -2.607487201690674, |
| "step": 305 |
| }, |
| { |
| "epoch": 2.565194805194805, |
| "grad_norm": 3.1326005458831787, |
| "learning_rate": 2.84766775162994e-07, |
| "logits/chosen": -1.2745239734649658, |
| "logits/rejected": -1.189447045326233, |
| "logps/chosen": -248.35812377929688, |
| "logps/rejected": -322.8234558105469, |
| "loss": 0.2217, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.07117398083209991, |
| "rewards/margins": 3.0546634197235107, |
| "rewards/rejected": -2.9834892749786377, |
| "step": 310 |
| }, |
| { |
| "epoch": 2.6067532467532466, |
| "grad_norm": 1.8207591772079468, |
| "learning_rate": 2.3527938497050678e-07, |
| "logits/chosen": -1.3199981451034546, |
| "logits/rejected": -1.2175192832946777, |
| "logps/chosen": -229.90371704101562, |
| "logps/rejected": -376.37445068359375, |
| "loss": 0.1869, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.0209323950111866, |
| "rewards/margins": 3.4481894969940186, |
| "rewards/rejected": -3.4272570610046387, |
| "step": 315 |
| }, |
| { |
| "epoch": 2.648311688311688, |
| "grad_norm": 2.0498719215393066, |
| "learning_rate": 1.9030116872178317e-07, |
| "logits/chosen": -1.2227413654327393, |
| "logits/rejected": -1.1524617671966553, |
| "logps/chosen": -226.3324737548828, |
| "logps/rejected": -299.68780517578125, |
| "loss": 0.258, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.059439968317747116, |
| "rewards/margins": 2.7543835639953613, |
| "rewards/rejected": -2.8138232231140137, |
| "step": 320 |
| }, |
| { |
| "epoch": 2.6898701298701297, |
| "grad_norm": 2.4288649559020996, |
| "learning_rate": 1.4992168033327482e-07, |
| "logits/chosen": -1.3343864679336548, |
| "logits/rejected": -1.2201334238052368, |
| "logps/chosen": -213.62020874023438, |
| "logps/rejected": -305.35467529296875, |
| "loss": 0.197, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.12054959684610367, |
| "rewards/margins": 2.9617972373962402, |
| "rewards/rejected": -2.841248035430908, |
| "step": 325 |
| }, |
| { |
| "epoch": 2.7314285714285713, |
| "grad_norm": 1.6564867496490479, |
| "learning_rate": 1.1422131742034048e-07, |
| "logits/chosen": -1.328583002090454, |
| "logits/rejected": -1.1812621355056763, |
| "logps/chosen": -216.4242706298828, |
| "logps/rejected": -310.3609313964844, |
| "loss": 0.2156, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.05895359441637993, |
| "rewards/margins": 3.110816478729248, |
| "rewards/rejected": -3.1697702407836914, |
| "step": 330 |
| }, |
| { |
| "epoch": 2.772987012987013, |
| "grad_norm": 2.33535099029541, |
| "learning_rate": 8.327116122150647e-08, |
| "logits/chosen": -1.2217092514038086, |
| "logits/rejected": -1.1084274053573608, |
| "logps/chosen": -221.68539428710938, |
| "logps/rejected": -336.6221618652344, |
| "loss": 0.1963, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.07560758292675018, |
| "rewards/margins": 3.6819679737091064, |
| "rewards/rejected": -3.606360673904419, |
| "step": 335 |
| }, |
| { |
| "epoch": 2.8145454545454545, |
| "grad_norm": 1.6087279319763184, |
| "learning_rate": 5.7132835072101486e-08, |
| "logits/chosen": -1.3235546350479126, |
| "logits/rejected": -1.2055761814117432, |
| "logps/chosen": -224.8578338623047, |
| "logps/rejected": -323.5769958496094, |
| "loss": 0.2213, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.050518132746219635, |
| "rewards/margins": 3.143819808959961, |
| "rewards/rejected": -3.093301773071289, |
| "step": 340 |
| }, |
| { |
| "epoch": 2.856103896103896, |
| "grad_norm": 2.025810718536377, |
| "learning_rate": 3.585838170906353e-08, |
| "logits/chosen": -1.3240934610366821, |
| "logits/rejected": -1.1658360958099365, |
| "logps/chosen": -233.7198028564453, |
| "logps/rejected": -306.67138671875, |
| "loss": 0.2142, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.00679791858419776, |
| "rewards/margins": 3.107149839401245, |
| "rewards/rejected": -3.1003518104553223, |
| "step": 345 |
| }, |
| { |
| "epoch": 2.8976623376623376, |
| "grad_norm": 2.809882640838623, |
| "learning_rate": 1.9490159651203988e-08, |
| "logits/chosen": -1.3499139547348022, |
| "logits/rejected": -1.290265679359436, |
| "logps/chosen": -239.82080078125, |
| "logps/rejected": -331.41510009765625, |
| "loss": 0.212, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.10003846883773804, |
| "rewards/margins": 3.5371921062469482, |
| "rewards/rejected": -3.4371535778045654, |
| "step": 350 |
| }, |
| { |
| "epoch": 2.939220779220779, |
| "grad_norm": 2.6004693508148193, |
| "learning_rate": 8.060758861239414e-09, |
| "logits/chosen": -1.3200178146362305, |
| "logits/rejected": -1.2445639371871948, |
| "logps/chosen": -216.0928192138672, |
| "logps/rejected": -329.24566650390625, |
| "loss": 0.2011, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.07652398198843002, |
| "rewards/margins": 3.232393264770508, |
| "rewards/rejected": -3.155869483947754, |
| "step": 355 |
| }, |
| { |
| "epoch": 2.9807792207792208, |
| "grad_norm": 2.5794732570648193, |
| "learning_rate": 1.5929358575206277e-09, |
| "logits/chosen": -1.3764307498931885, |
| "logits/rejected": -1.218400478363037, |
| "logps/chosen": -225.61376953125, |
| "logps/rejected": -306.9019470214844, |
| "loss": 0.2348, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.024400614202022552, |
| "rewards/margins": 3.0199968814849854, |
| "rewards/rejected": -3.0443975925445557, |
| "step": 360 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 363, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
