| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 1407, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.010666666666666666, |
| "grad_norm": 3.225456953048706, |
| "learning_rate": 4.651162790697675e-07, |
| "logits/chosen": -1.452316164970398, |
| "logits/rejected": -1.3630012273788452, |
| "logps/chosen": -216.8773651123047, |
| "logps/rejected": -261.6772155761719, |
| "loss": 0.6934, |
| "rewards/accuracies": 0.29374998807907104, |
| "rewards/chosen": -0.00024420738918706775, |
| "rewards/margins": -0.0005015610950067639, |
| "rewards/rejected": 0.00025735379313118756, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.021333333333333333, |
| "grad_norm": 3.985062599182129, |
| "learning_rate": 1.0465116279069768e-06, |
| "logits/chosen": -1.3076436519622803, |
| "logits/rejected": -1.2246109247207642, |
| "logps/chosen": -230.8943328857422, |
| "logps/rejected": -299.6863708496094, |
| "loss": 0.6899, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.001177475554868579, |
| "rewards/margins": 0.006514736916869879, |
| "rewards/rejected": -0.005337262060493231, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.032, |
| "grad_norm": 3.3894476890563965, |
| "learning_rate": 1.6279069767441862e-06, |
| "logits/chosen": -1.3111854791641235, |
| "logits/rejected": -1.3214471340179443, |
| "logps/chosen": -250.80496215820312, |
| "logps/rejected": -299.97125244140625, |
| "loss": 0.6834, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.0006973742274567485, |
| "rewards/margins": 0.019777875393629074, |
| "rewards/rejected": -0.020475249737501144, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.042666666666666665, |
| "grad_norm": 2.994020938873291, |
| "learning_rate": 2.2093023255813954e-06, |
| "logits/chosen": -1.3028817176818848, |
| "logits/rejected": -1.1915266513824463, |
| "logps/chosen": -255.93081665039062, |
| "logps/rejected": -287.74224853515625, |
| "loss": 0.6737, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.002559587825089693, |
| "rewards/margins": 0.03989601880311966, |
| "rewards/rejected": -0.042455606162548065, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.05333333333333334, |
| "grad_norm": 3.090261220932007, |
| "learning_rate": 2.790697674418605e-06, |
| "logits/chosen": -1.388347864151001, |
| "logits/rejected": -1.328805923461914, |
| "logps/chosen": -250.98336791992188, |
| "logps/rejected": -322.8138122558594, |
| "loss": 0.6594, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -0.002105569699779153, |
| "rewards/margins": 0.07019330561161041, |
| "rewards/rejected": -0.07229887694120407, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.064, |
| "grad_norm": 3.1172730922698975, |
| "learning_rate": 3.372093023255814e-06, |
| "logits/chosen": -1.4825924634933472, |
| "logits/rejected": -1.3452062606811523, |
| "logps/chosen": -253.34860229492188, |
| "logps/rejected": -290.7360534667969, |
| "loss": 0.6473, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.01585470139980316, |
| "rewards/margins": 0.09762205183506012, |
| "rewards/rejected": -0.11347675323486328, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07466666666666667, |
| "grad_norm": 3.13596773147583, |
| "learning_rate": 3.953488372093024e-06, |
| "logits/chosen": -1.5015944242477417, |
| "logits/rejected": -1.3286340236663818, |
| "logps/chosen": -244.2925262451172, |
| "logps/rejected": -298.30621337890625, |
| "loss": 0.6208, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.01058957539498806, |
| "rewards/margins": 0.15787732601165771, |
| "rewards/rejected": -0.16846691071987152, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.08533333333333333, |
| "grad_norm": 2.9116740226745605, |
| "learning_rate": 4.5348837209302326e-06, |
| "logits/chosen": -1.4416661262512207, |
| "logits/rejected": -1.2964617013931274, |
| "logps/chosen": -243.5440216064453, |
| "logps/rejected": -299.6397705078125, |
| "loss": 0.5937, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.019765138626098633, |
| "rewards/margins": 0.22198882699012756, |
| "rewards/rejected": -0.2417539656162262, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.096, |
| "grad_norm": 3.0077366828918457, |
| "learning_rate": 4.99999336897255e-06, |
| "logits/chosen": -1.459515929222107, |
| "logits/rejected": -1.3635624647140503, |
| "logps/chosen": -238.49807739257812, |
| "logps/rejected": -266.1226501464844, |
| "loss": 0.5827, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -0.038423217833042145, |
| "rewards/margins": 0.2630027234554291, |
| "rewards/rejected": -0.3014259338378906, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.10666666666666667, |
| "grad_norm": 3.088587999343872, |
| "learning_rate": 4.999761286705285e-06, |
| "logits/chosen": -1.4864553213119507, |
| "logits/rejected": -1.4152582883834839, |
| "logps/chosen": -233.3901824951172, |
| "logps/rejected": -296.8989562988281, |
| "loss": 0.5212, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.0003606337995734066, |
| "rewards/margins": 0.42451077699661255, |
| "rewards/rejected": -0.42487144470214844, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11733333333333333, |
| "grad_norm": 2.77523136138916, |
| "learning_rate": 4.999197688241076e-06, |
| "logits/chosen": -1.5394684076309204, |
| "logits/rejected": -1.3223412036895752, |
| "logps/chosen": -242.56759643554688, |
| "logps/rejected": -281.634765625, |
| "loss": 0.5129, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.07203203439712524, |
| "rewards/margins": 0.5109471678733826, |
| "rewards/rejected": -0.582979142665863, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.128, |
| "grad_norm": 2.9487898349761963, |
| "learning_rate": 4.9983026483238685e-06, |
| "logits/chosen": -1.4420719146728516, |
| "logits/rejected": -1.2731995582580566, |
| "logps/chosen": -224.287109375, |
| "logps/rejected": -312.1568908691406, |
| "loss": 0.461, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.2297721803188324, |
| "rewards/margins": 0.6778525114059448, |
| "rewards/rejected": -0.9076247215270996, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.13866666666666666, |
| "grad_norm": 2.45941162109375, |
| "learning_rate": 4.997076285653087e-06, |
| "logits/chosen": -1.5383515357971191, |
| "logits/rejected": -1.4481399059295654, |
| "logps/chosen": -274.0958557128906, |
| "logps/rejected": -329.88018798828125, |
| "loss": 0.4413, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.3417445719242096, |
| "rewards/margins": 0.7747383713722229, |
| "rewards/rejected": -1.1164829730987549, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.14933333333333335, |
| "grad_norm": 2.6682322025299072, |
| "learning_rate": 4.995518762867897e-06, |
| "logits/chosen": -1.5974746942520142, |
| "logits/rejected": -1.3631844520568848, |
| "logps/chosen": -217.626953125, |
| "logps/rejected": -307.88116455078125, |
| "loss": 0.3906, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.3570924401283264, |
| "rewards/margins": 1.0183016061782837, |
| "rewards/rejected": -1.3753941059112549, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 2.8534865379333496, |
| "learning_rate": 4.993630286525634e-06, |
| "logits/chosen": -1.6104965209960938, |
| "logits/rejected": -1.4973475933074951, |
| "logps/chosen": -241.7733154296875, |
| "logps/rejected": -289.15728759765625, |
| "loss": 0.3991, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.44355034828186035, |
| "rewards/margins": 0.9931395649909973, |
| "rewards/rejected": -1.4366897344589233, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.17066666666666666, |
| "grad_norm": 3.2399303913116455, |
| "learning_rate": 4.99141110707441e-06, |
| "logits/chosen": -1.5593732595443726, |
| "logits/rejected": -1.3765180110931396, |
| "logps/chosen": -244.89749145507812, |
| "logps/rejected": -303.7383728027344, |
| "loss": 0.3772, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.5426279306411743, |
| "rewards/margins": 1.1284551620483398, |
| "rewards/rejected": -1.6710832118988037, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18133333333333335, |
| "grad_norm": 3.0792360305786133, |
| "learning_rate": 4.9888615188199e-06, |
| "logits/chosen": -1.6551406383514404, |
| "logits/rejected": -1.4097787141799927, |
| "logps/chosen": -245.085693359375, |
| "logps/rejected": -293.0350036621094, |
| "loss": 0.3911, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.5418190956115723, |
| "rewards/margins": 1.2352501153945923, |
| "rewards/rejected": -1.777069330215454, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.192, |
| "grad_norm": 2.767003059387207, |
| "learning_rate": 4.9859818598863104e-06, |
| "logits/chosen": -1.569866418838501, |
| "logits/rejected": -1.4290711879730225, |
| "logps/chosen": -244.1849822998047, |
| "logps/rejected": -301.0859680175781, |
| "loss": 0.3505, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.5658442974090576, |
| "rewards/margins": 1.3717987537384033, |
| "rewards/rejected": -1.9376428127288818, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.20266666666666666, |
| "grad_norm": 2.779585838317871, |
| "learning_rate": 4.982772512171538e-06, |
| "logits/chosen": -1.5476526021957397, |
| "logits/rejected": -1.4095839262008667, |
| "logps/chosen": -251.42025756835938, |
| "logps/rejected": -317.84771728515625, |
| "loss": 0.3451, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.6214931607246399, |
| "rewards/margins": 1.5358760356903076, |
| "rewards/rejected": -2.1573691368103027, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.21333333333333335, |
| "grad_norm": 2.5582046508789062, |
| "learning_rate": 4.979233901296523e-06, |
| "logits/chosen": -1.6822445392608643, |
| "logits/rejected": -1.4913674592971802, |
| "logps/chosen": -238.1501007080078, |
| "logps/rejected": -317.0533142089844, |
| "loss": 0.2784, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.4816685616970062, |
| "rewards/margins": 1.7280359268188477, |
| "rewards/rejected": -2.209704637527466, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.224, |
| "grad_norm": 2.492095470428467, |
| "learning_rate": 4.975366496548804e-06, |
| "logits/chosen": -1.58907151222229, |
| "logits/rejected": -1.4344779253005981, |
| "logps/chosen": -252.6648406982422, |
| "logps/rejected": -321.97332763671875, |
| "loss": 0.303, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.5509522557258606, |
| "rewards/margins": 1.748511552810669, |
| "rewards/rejected": -2.2994637489318848, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.23466666666666666, |
| "grad_norm": 3.3831777572631836, |
| "learning_rate": 4.971170810820279e-06, |
| "logits/chosen": -1.6393448114395142, |
| "logits/rejected": -1.5223277807235718, |
| "logps/chosen": -234.4925079345703, |
| "logps/rejected": -295.9253845214844, |
| "loss": 0.3034, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.5105643272399902, |
| "rewards/margins": 1.7626917362213135, |
| "rewards/rejected": -2.2732558250427246, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.24533333333333332, |
| "grad_norm": 2.4063918590545654, |
| "learning_rate": 4.96664740053919e-06, |
| "logits/chosen": -1.647438406944275, |
| "logits/rejected": -1.532233476638794, |
| "logps/chosen": -236.671875, |
| "logps/rejected": -318.50775146484375, |
| "loss": 0.2571, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.8266940116882324, |
| "rewards/margins": 2.115816593170166, |
| "rewards/rejected": -2.9425106048583984, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.256, |
| "grad_norm": 2.415274143218994, |
| "learning_rate": 4.961796865596328e-06, |
| "logits/chosen": -1.7164440155029297, |
| "logits/rejected": -1.5145854949951172, |
| "logps/chosen": -237.99853515625, |
| "logps/rejected": -316.65167236328125, |
| "loss": 0.2933, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.7486313581466675, |
| "rewards/margins": 2.1729655265808105, |
| "rewards/rejected": -2.9215967655181885, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.26666666666666666, |
| "grad_norm": 1.8586480617523193, |
| "learning_rate": 4.956619849265473e-06, |
| "logits/chosen": -1.7343181371688843, |
| "logits/rejected": -1.5120888948440552, |
| "logps/chosen": -241.61019897460938, |
| "logps/rejected": -336.1865234375, |
| "loss": 0.2291, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.631386399269104, |
| "rewards/margins": 2.4859204292297363, |
| "rewards/rejected": -3.117306709289551, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.2773333333333333, |
| "grad_norm": 3.7649638652801514, |
| "learning_rate": 4.951117038118091e-06, |
| "logits/chosen": -1.7571346759796143, |
| "logits/rejected": -1.582757592201233, |
| "logps/chosen": -240.4564971923828, |
| "logps/rejected": -326.0672302246094, |
| "loss": 0.2523, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.7324283719062805, |
| "rewards/margins": 2.460465908050537, |
| "rewards/rejected": -3.192894458770752, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.288, |
| "grad_norm": 1.8477269411087036, |
| "learning_rate": 4.945289161932273e-06, |
| "logits/chosen": -1.7058353424072266, |
| "logits/rejected": -1.5484061241149902, |
| "logps/chosen": -234.3208465576172, |
| "logps/rejected": -316.4405822753906, |
| "loss": 0.2389, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.644821047782898, |
| "rewards/margins": 2.5761804580688477, |
| "rewards/rejected": -3.221001386642456, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.2986666666666667, |
| "grad_norm": 2.963148832321167, |
| "learning_rate": 4.93913699359596e-06, |
| "logits/chosen": -1.7283592224121094, |
| "logits/rejected": -1.5544356107711792, |
| "logps/chosen": -235.847900390625, |
| "logps/rejected": -317.0403747558594, |
| "loss": 0.2739, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.6758811473846436, |
| "rewards/margins": 2.5437233448028564, |
| "rewards/rejected": -3.219604015350342, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.30933333333333335, |
| "grad_norm": 3.0582330226898193, |
| "learning_rate": 4.9326613490044364e-06, |
| "logits/chosen": -1.6446880102157593, |
| "logits/rejected": -1.4448646306991577, |
| "logps/chosen": -258.02813720703125, |
| "logps/rejected": -310.2739562988281, |
| "loss": 0.2851, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.7733806371688843, |
| "rewards/margins": 2.419772148132324, |
| "rewards/rejected": -3.19315242767334, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 2.466308832168579, |
| "learning_rate": 4.925863086952132e-06, |
| "logits/chosen": -1.6388626098632812, |
| "logits/rejected": -1.455078363418579, |
| "logps/chosen": -245.62631225585938, |
| "logps/rejected": -345.78521728515625, |
| "loss": 0.2196, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.46061477065086365, |
| "rewards/margins": 2.8316876888275146, |
| "rewards/rejected": -3.292302370071411, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33066666666666666, |
| "grad_norm": 2.5503790378570557, |
| "learning_rate": 4.918743109018727e-06, |
| "logits/chosen": -1.7483993768692017, |
| "logits/rejected": -1.5743939876556396, |
| "logps/chosen": -231.1587371826172, |
| "logps/rejected": -314.0552673339844, |
| "loss": 0.2273, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.6226693987846375, |
| "rewards/margins": 2.552253246307373, |
| "rewards/rejected": -3.174922466278076, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.3413333333333333, |
| "grad_norm": 2.485719680786133, |
| "learning_rate": 4.911302359449585e-06, |
| "logits/chosen": -1.7262346744537354, |
| "logits/rejected": -1.5447709560394287, |
| "logps/chosen": -254.2543487548828, |
| "logps/rejected": -323.9798278808594, |
| "loss": 0.2383, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.7263298630714417, |
| "rewards/margins": 2.8138160705566406, |
| "rewards/rejected": -3.5401458740234375, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.352, |
| "grad_norm": 3.7984070777893066, |
| "learning_rate": 4.9035418250305314e-06, |
| "logits/chosen": -1.656524896621704, |
| "logits/rejected": -1.4639983177185059, |
| "logps/chosen": -234.14254760742188, |
| "logps/rejected": -314.20318603515625, |
| "loss": 0.206, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.35529714822769165, |
| "rewards/margins": 2.8109519481658936, |
| "rewards/rejected": -3.1662492752075195, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.3626666666666667, |
| "grad_norm": 2.783888339996338, |
| "learning_rate": 4.895462534956981e-06, |
| "logits/chosen": -1.6165010929107666, |
| "logits/rejected": -1.5533854961395264, |
| "logps/chosen": -242.182861328125, |
| "logps/rejected": -333.15863037109375, |
| "loss": 0.2425, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.5480275750160217, |
| "rewards/margins": 2.9635632038116455, |
| "rewards/rejected": -3.5115904808044434, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.37333333333333335, |
| "grad_norm": 6.773368835449219, |
| "learning_rate": 4.887065560697449e-06, |
| "logits/chosen": -1.6875731945037842, |
| "logits/rejected": -1.507009744644165, |
| "logps/chosen": -254.5814971923828, |
| "logps/rejected": -332.4895935058594, |
| "loss": 0.2905, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.5142589807510376, |
| "rewards/margins": 2.550226926803589, |
| "rewards/rejected": -3.064485788345337, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.384, |
| "grad_norm": 2.7218058109283447, |
| "learning_rate": 4.878352015851459e-06, |
| "logits/chosen": -1.6414203643798828, |
| "logits/rejected": -1.4779311418533325, |
| "logps/chosen": -270.7060852050781, |
| "logps/rejected": -332.2988586425781, |
| "loss": 0.1879, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.36767151951789856, |
| "rewards/margins": 3.0560736656188965, |
| "rewards/rejected": -3.4237449169158936, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39466666666666667, |
| "grad_norm": 2.207134962081909, |
| "learning_rate": 4.869323056001852e-06, |
| "logits/chosen": -1.675317406654358, |
| "logits/rejected": -1.4597893953323364, |
| "logps/chosen": -240.0108642578125, |
| "logps/rejected": -306.22418212890625, |
| "loss": 0.197, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.2930644154548645, |
| "rewards/margins": 2.977311372756958, |
| "rewards/rejected": -3.2703757286071777, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.4053333333333333, |
| "grad_norm": 2.3709778785705566, |
| "learning_rate": 4.859979878561536e-06, |
| "logits/chosen": -1.663538932800293, |
| "logits/rejected": -1.516006588935852, |
| "logps/chosen": -222.03439331054688, |
| "logps/rejected": -327.6264343261719, |
| "loss": 0.1997, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.18494608998298645, |
| "rewards/margins": 3.1293203830718994, |
| "rewards/rejected": -3.3142666816711426, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.416, |
| "grad_norm": 1.9043569564819336, |
| "learning_rate": 4.850323722614685e-06, |
| "logits/chosen": -1.7305800914764404, |
| "logits/rejected": -1.612801194190979, |
| "logps/chosen": -245.2909698486328, |
| "logps/rejected": -351.641357421875, |
| "loss": 0.1775, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5533159971237183, |
| "rewards/margins": 3.2820746898651123, |
| "rewards/rejected": -3.835390567779541, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.4266666666666667, |
| "grad_norm": 2.5433902740478516, |
| "learning_rate": 4.840355868752416e-06, |
| "logits/chosen": -1.571982741355896, |
| "logits/rejected": -1.4637776613235474, |
| "logps/chosen": -299.1954345703125, |
| "logps/rejected": -337.9162292480469, |
| "loss": 0.1962, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3970106244087219, |
| "rewards/margins": 3.3511834144592285, |
| "rewards/rejected": -3.7481937408447266, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.43733333333333335, |
| "grad_norm": 2.6432783603668213, |
| "learning_rate": 4.830077638902955e-06, |
| "logits/chosen": -1.7969417572021484, |
| "logits/rejected": -1.6737842559814453, |
| "logps/chosen": -228.6658477783203, |
| "logps/rejected": -323.2589416503906, |
| "loss": 0.1855, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.45908913016319275, |
| "rewards/margins": 3.377959728240967, |
| "rewards/rejected": -3.8370490074157715, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.448, |
| "grad_norm": 2.609254837036133, |
| "learning_rate": 4.8194903961563255e-06, |
| "logits/chosen": -1.8882259130477905, |
| "logits/rejected": -1.6982256174087524, |
| "logps/chosen": -247.7525634765625, |
| "logps/rejected": -331.72723388671875, |
| "loss": 0.1426, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.49562868475914, |
| "rewards/margins": 3.640789747238159, |
| "rewards/rejected": -4.136418342590332, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.45866666666666667, |
| "grad_norm": 2.308016300201416, |
| "learning_rate": 4.808595544583578e-06, |
| "logits/chosen": -1.772505521774292, |
| "logits/rejected": -1.6354877948760986, |
| "logps/chosen": -234.55142211914062, |
| "logps/rejected": -352.1875305175781, |
| "loss": 0.1857, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.5159016251564026, |
| "rewards/margins": 3.877976894378662, |
| "rewards/rejected": -4.393878936767578, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.4693333333333333, |
| "grad_norm": 1.5003650188446045, |
| "learning_rate": 4.797394529050577e-06, |
| "logits/chosen": -1.6615060567855835, |
| "logits/rejected": -1.5364525318145752, |
| "logps/chosen": -272.47418212890625, |
| "logps/rejected": -360.6487731933594, |
| "loss": 0.1639, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.691543698310852, |
| "rewards/margins": 3.9234442710876465, |
| "rewards/rejected": -4.614988327026367, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 2.611100435256958, |
| "learning_rate": 4.785888835026393e-06, |
| "logits/chosen": -1.745161771774292, |
| "logits/rejected": -1.5977957248687744, |
| "logps/chosen": -260.530029296875, |
| "logps/rejected": -306.3468017578125, |
| "loss": 0.2476, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.3127722144126892, |
| "rewards/margins": 3.3172645568847656, |
| "rewards/rejected": -3.6300368309020996, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.49066666666666664, |
| "grad_norm": 1.957841157913208, |
| "learning_rate": 4.7740799883862966e-06, |
| "logits/chosen": -1.8661781549453735, |
| "logits/rejected": -1.6978685855865479, |
| "logps/chosen": -233.84231567382812, |
| "logps/rejected": -317.6667785644531, |
| "loss": 0.1736, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.3665715754032135, |
| "rewards/margins": 3.6203083992004395, |
| "rewards/rejected": -3.986879825592041, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5013333333333333, |
| "grad_norm": 1.5022276639938354, |
| "learning_rate": 4.761969555209394e-06, |
| "logits/chosen": -1.778978943824768, |
| "logits/rejected": -1.6453516483306885, |
| "logps/chosen": -249.01815795898438, |
| "logps/rejected": -362.90460205078125, |
| "loss": 0.1509, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8386176228523254, |
| "rewards/margins": 4.385830402374268, |
| "rewards/rejected": -5.224447727203369, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.512, |
| "grad_norm": 2.0542163848876953, |
| "learning_rate": 4.7495591415709435e-06, |
| "logits/chosen": -1.8296219110488892, |
| "logits/rejected": -1.6568787097930908, |
| "logps/chosen": -228.88412475585938, |
| "logps/rejected": -314.2283630371094, |
| "loss": 0.2659, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.5388306975364685, |
| "rewards/margins": 3.918764591217041, |
| "rewards/rejected": -4.457594871520996, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5226666666666666, |
| "grad_norm": 3.47914719581604, |
| "learning_rate": 4.736850393329353e-06, |
| "logits/chosen": -1.779706358909607, |
| "logits/rejected": -1.5548733472824097, |
| "logps/chosen": -252.67044067382812, |
| "logps/rejected": -338.41217041015625, |
| "loss": 0.2123, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.5982939600944519, |
| "rewards/margins": 3.9195141792297363, |
| "rewards/rejected": -4.517808437347412, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5333333333333333, |
| "grad_norm": 3.8012492656707764, |
| "learning_rate": 4.7238449959079095e-06, |
| "logits/chosen": -1.8602869510650635, |
| "logits/rejected": -1.6974502801895142, |
| "logps/chosen": -241.8807830810547, |
| "logps/rejected": -328.1159362792969, |
| "loss": 0.2123, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.4912596344947815, |
| "rewards/margins": 4.016532897949219, |
| "rewards/rejected": -4.5077924728393555, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.544, |
| "grad_norm": 1.548546552658081, |
| "learning_rate": 4.710544674071259e-06, |
| "logits/chosen": -1.8098742961883545, |
| "logits/rejected": -1.6698535680770874, |
| "logps/chosen": -236.11611938476562, |
| "logps/rejected": -343.9376220703125, |
| "loss": 0.1609, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4053850769996643, |
| "rewards/margins": 4.2683796882629395, |
| "rewards/rejected": -4.673764228820801, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5546666666666666, |
| "grad_norm": 3.3183388710021973, |
| "learning_rate": 4.696951191696673e-06, |
| "logits/chosen": -1.8579626083374023, |
| "logits/rejected": -1.7205890417099, |
| "logps/chosen": -230.24563598632812, |
| "logps/rejected": -340.3718566894531, |
| "loss": 0.1665, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.5163661241531372, |
| "rewards/margins": 4.36639928817749, |
| "rewards/rejected": -4.882765769958496, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5653333333333334, |
| "grad_norm": 2.8505144119262695, |
| "learning_rate": 4.6830663515401196e-06, |
| "logits/chosen": -1.7807422876358032, |
| "logits/rejected": -1.5978543758392334, |
| "logps/chosen": -237.2873992919922, |
| "logps/rejected": -336.2684326171875, |
| "loss": 0.2242, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.35498833656311035, |
| "rewards/margins": 3.7952804565429688, |
| "rewards/rejected": -4.150269508361816, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.576, |
| "grad_norm": 2.377626895904541, |
| "learning_rate": 4.6688919949971855e-06, |
| "logits/chosen": -1.7819585800170898, |
| "logits/rejected": -1.6391322612762451, |
| "logps/chosen": -251.92874145507812, |
| "logps/rejected": -349.6017761230469, |
| "loss": 0.2408, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.44162049889564514, |
| "rewards/margins": 4.0541276931762695, |
| "rewards/rejected": -4.495748043060303, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5866666666666667, |
| "grad_norm": 1.5478827953338623, |
| "learning_rate": 4.654430001858874e-06, |
| "logits/chosen": -1.7001129388809204, |
| "logits/rejected": -1.5338956117630005, |
| "logps/chosen": -251.0334014892578, |
| "logps/rejected": -333.11163330078125, |
| "loss": 0.1582, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.034758586436510086, |
| "rewards/margins": 4.018908500671387, |
| "rewards/rejected": -4.053666591644287, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5973333333333334, |
| "grad_norm": 3.8507468700408936, |
| "learning_rate": 4.6396822900623075e-06, |
| "logits/chosen": -1.749701738357544, |
| "logits/rejected": -1.6001421213150024, |
| "logps/chosen": -250.6236114501953, |
| "logps/rejected": -327.2859191894531, |
| "loss": 0.1665, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2926911413669586, |
| "rewards/margins": 4.104784965515137, |
| "rewards/rejected": -4.3974761962890625, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.608, |
| "grad_norm": 3.485337972640991, |
| "learning_rate": 4.624650815436368e-06, |
| "logits/chosen": -1.7875235080718994, |
| "logits/rejected": -1.663147211074829, |
| "logps/chosen": -275.81329345703125, |
| "logps/rejected": -355.75921630859375, |
| "loss": 0.1732, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.2872824966907501, |
| "rewards/margins": 4.033877372741699, |
| "rewards/rejected": -4.321159362792969, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.6186666666666667, |
| "grad_norm": 3.0503721237182617, |
| "learning_rate": 4.609337571442324e-06, |
| "logits/chosen": -1.8553688526153564, |
| "logits/rejected": -1.7102324962615967, |
| "logps/chosen": -227.72128295898438, |
| "logps/rejected": -343.08148193359375, |
| "loss": 0.1448, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3748090863227844, |
| "rewards/margins": 4.319515228271484, |
| "rewards/rejected": -4.694324970245361, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6293333333333333, |
| "grad_norm": 1.912618637084961, |
| "learning_rate": 4.5937445889094535e-06, |
| "logits/chosen": -1.7128536701202393, |
| "logits/rejected": -1.6339744329452515, |
| "logps/chosen": -271.19769287109375, |
| "logps/rejected": -340.0036926269531, |
| "loss": 0.1723, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.5582373738288879, |
| "rewards/margins": 4.0624799728393555, |
| "rewards/rejected": -4.6207170486450195, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 1.1806559562683105, |
| "learning_rate": 4.577873935765722e-06, |
| "logits/chosen": -1.789751648902893, |
| "logits/rejected": -1.6350936889648438, |
| "logps/chosen": -255.74234008789062, |
| "logps/rejected": -362.6470642089844, |
| "loss": 0.1218, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6933242082595825, |
| "rewards/margins": 4.8959221839904785, |
| "rewards/rejected": -5.5892462730407715, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6506666666666666, |
| "grad_norm": 2.678286075592041, |
| "learning_rate": 4.561727716763532e-06, |
| "logits/chosen": -1.8357679843902588, |
| "logits/rejected": -1.708337426185608, |
| "logps/chosen": -252.95004272460938, |
| "logps/rejected": -354.3417053222656, |
| "loss": 0.1762, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.6951386332511902, |
| "rewards/margins": 4.543791770935059, |
| "rewards/rejected": -5.2389302253723145, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.6613333333333333, |
| "grad_norm": 1.354732871055603, |
| "learning_rate": 4.545308073200591e-06, |
| "logits/chosen": -1.6892789602279663, |
| "logits/rejected": -1.622051477432251, |
| "logps/chosen": -254.616943359375, |
| "logps/rejected": -349.6521301269531, |
| "loss": 0.1573, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8155484199523926, |
| "rewards/margins": 4.775422096252441, |
| "rewards/rejected": -5.590970039367676, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.672, |
| "grad_norm": 2.0910332202911377, |
| "learning_rate": 4.528617182635943e-06, |
| "logits/chosen": -1.925897240638733, |
| "logits/rejected": -1.7414014339447021, |
| "logps/chosen": -253.86239624023438, |
| "logps/rejected": -356.8898620605469, |
| "loss": 0.2282, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.9393094182014465, |
| "rewards/margins": 4.897877216339111, |
| "rewards/rejected": -5.83718729019165, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6826666666666666, |
| "grad_norm": 4.3398566246032715, |
| "learning_rate": 4.511657258601171e-06, |
| "logits/chosen": -1.7808582782745361, |
| "logits/rejected": -1.584148645401001, |
| "logps/chosen": -245.6190643310547, |
| "logps/rejected": -326.17950439453125, |
| "loss": 0.1912, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.752561092376709, |
| "rewards/margins": 4.876051425933838, |
| "rewards/rejected": -5.628612518310547, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6933333333333334, |
| "grad_norm": 1.8720273971557617, |
| "learning_rate": 4.494430550306848e-06, |
| "logits/chosen": -1.7456798553466797, |
| "logits/rejected": -1.6327714920043945, |
| "logps/chosen": -251.36740112304688, |
| "logps/rejected": -327.3114013671875, |
| "loss": 0.1792, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.30995193123817444, |
| "rewards/margins": 4.125004291534424, |
| "rewards/rejected": -4.434955596923828, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.704, |
| "grad_norm": 2.9589478969573975, |
| "learning_rate": 4.476939342344246e-06, |
| "logits/chosen": -1.8625526428222656, |
| "logits/rejected": -1.7193504571914673, |
| "logps/chosen": -234.44827270507812, |
| "logps/rejected": -323.15924072265625, |
| "loss": 0.1882, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.8839484453201294, |
| "rewards/margins": 4.334848403930664, |
| "rewards/rejected": -5.218796730041504, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7146666666666667, |
| "grad_norm": 2.8924190998077393, |
| "learning_rate": 4.45918595438236e-06, |
| "logits/chosen": -1.7802038192749023, |
| "logits/rejected": -1.6963586807250977, |
| "logps/chosen": -247.6739044189453, |
| "logps/rejected": -334.2060546875, |
| "loss": 0.1904, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.8888412714004517, |
| "rewards/margins": 4.769382953643799, |
| "rewards/rejected": -5.658223628997803, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7253333333333334, |
| "grad_norm": 2.6374990940093994, |
| "learning_rate": 4.441172740860267e-06, |
| "logits/chosen": -1.8051414489746094, |
| "logits/rejected": -1.7025120258331299, |
| "logps/chosen": -236.43789672851562, |
| "logps/rejected": -326.9657287597656, |
| "loss": 0.1302, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.7156514525413513, |
| "rewards/margins": 4.83019495010376, |
| "rewards/rejected": -5.545845985412598, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.736, |
| "grad_norm": 1.9260821342468262, |
| "learning_rate": 4.422902090674886e-06, |
| "logits/chosen": -1.7234573364257812, |
| "logits/rejected": -1.61138916015625, |
| "logps/chosen": -252.62173461914062, |
| "logps/rejected": -356.4739685058594, |
| "loss": 0.1349, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8339887857437134, |
| "rewards/margins": 4.9775390625, |
| "rewards/rejected": -5.811528205871582, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.7466666666666667, |
| "grad_norm": 4.608032703399658, |
| "learning_rate": 4.4043764268641705e-06, |
| "logits/chosen": -1.8549143075942993, |
| "logits/rejected": -1.7140591144561768, |
| "logps/chosen": -234.22421264648438, |
| "logps/rejected": -318.148681640625, |
| "loss": 0.1986, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.8342403173446655, |
| "rewards/margins": 4.861636161804199, |
| "rewards/rejected": -5.6958770751953125, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7573333333333333, |
| "grad_norm": 3.3214540481567383, |
| "learning_rate": 4.385598206285758e-06, |
| "logits/chosen": -1.80758535861969, |
| "logits/rejected": -1.6905763149261475, |
| "logps/chosen": -287.7499084472656, |
| "logps/rejected": -365.01385498046875, |
| "loss": 0.1779, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.3829190731048584, |
| "rewards/margins": 4.985753536224365, |
| "rewards/rejected": -6.368672847747803, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.768, |
| "grad_norm": 1.858853816986084, |
| "learning_rate": 4.3665699192911495e-06, |
| "logits/chosen": -1.7372462749481201, |
| "logits/rejected": -1.6430860757827759, |
| "logps/chosen": -241.73046875, |
| "logps/rejected": -353.0447692871094, |
| "loss": 0.1826, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.9526206254959106, |
| "rewards/margins": 5.21450662612915, |
| "rewards/rejected": -6.16712760925293, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7786666666666666, |
| "grad_norm": 6.181103706359863, |
| "learning_rate": 4.347294089395438e-06, |
| "logits/chosen": -1.800432562828064, |
| "logits/rejected": -1.683996558189392, |
| "logps/chosen": -250.1630401611328, |
| "logps/rejected": -342.5280456542969, |
| "loss": 0.1389, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.913178563117981, |
| "rewards/margins": 5.121844291687012, |
| "rewards/rejected": -6.035023212432861, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7893333333333333, |
| "grad_norm": 4.438066005706787, |
| "learning_rate": 4.327773272942647e-06, |
| "logits/chosen": -1.8669296503067017, |
| "logits/rejected": -1.7096134424209595, |
| "logps/chosen": -239.025146484375, |
| "logps/rejected": -333.13409423828125, |
| "loss": 0.2367, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.7618646621704102, |
| "rewards/margins": 4.629305839538574, |
| "rewards/rejected": -5.391171455383301, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 3.4000484943389893, |
| "learning_rate": 4.308010058766703e-06, |
| "logits/chosen": -1.8593333959579468, |
| "logits/rejected": -1.6820989847183228, |
| "logps/chosen": -249.3157501220703, |
| "logps/rejected": -353.73687744140625, |
| "loss": 0.1273, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8536429405212402, |
| "rewards/margins": 5.058758735656738, |
| "rewards/rejected": -5.9124016761779785, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.8106666666666666, |
| "grad_norm": 2.9082837104797363, |
| "learning_rate": 4.2880070678481135e-06, |
| "logits/chosen": -1.8044195175170898, |
| "logits/rejected": -1.719753623008728, |
| "logps/chosen": -253.3800048828125, |
| "logps/rejected": -370.4222717285156, |
| "loss": 0.1795, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.109043002128601, |
| "rewards/margins": 4.930617809295654, |
| "rewards/rejected": -6.039660930633545, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8213333333333334, |
| "grad_norm": 3.509117841720581, |
| "learning_rate": 4.267766952966369e-06, |
| "logits/chosen": -1.7582893371582031, |
| "logits/rejected": -1.6735187768936157, |
| "logps/chosen": -248.4905242919922, |
| "logps/rejected": -347.24755859375, |
| "loss": 0.1581, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.7402669191360474, |
| "rewards/margins": 5.071152687072754, |
| "rewards/rejected": -5.811419486999512, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.832, |
| "grad_norm": 1.737369179725647, |
| "learning_rate": 4.24729239834814e-06, |
| "logits/chosen": -1.8210585117340088, |
| "logits/rejected": -1.6675134897232056, |
| "logps/chosen": -260.0174865722656, |
| "logps/rejected": -339.507568359375, |
| "loss": 0.191, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.807237446308136, |
| "rewards/margins": 4.788012504577637, |
| "rewards/rejected": -5.595250129699707, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.8426666666666667, |
| "grad_norm": 2.075969934463501, |
| "learning_rate": 4.226586119311291e-06, |
| "logits/chosen": -1.7811262607574463, |
| "logits/rejected": -1.7235519886016846, |
| "logps/chosen": -269.1385498046875, |
| "logps/rejected": -357.7069091796875, |
| "loss": 0.1703, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.7997838258743286, |
| "rewards/margins": 4.9125871658325195, |
| "rewards/rejected": -5.712370872497559, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8533333333333334, |
| "grad_norm": 3.3996775150299072, |
| "learning_rate": 4.205650861904782e-06, |
| "logits/chosen": -1.8044359683990479, |
| "logits/rejected": -1.686042070388794, |
| "logps/chosen": -236.1574249267578, |
| "logps/rejected": -339.8066711425781, |
| "loss": 0.1439, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5469352006912231, |
| "rewards/margins": 4.560192108154297, |
| "rewards/rejected": -5.107127666473389, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.864, |
| "grad_norm": 3.2350564002990723, |
| "learning_rate": 4.184489402544486e-06, |
| "logits/chosen": -1.7984288930892944, |
| "logits/rejected": -1.7034324407577515, |
| "logps/chosen": -244.17660522460938, |
| "logps/rejected": -326.0493469238281, |
| "loss": 0.1875, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.790968656539917, |
| "rewards/margins": 4.585756778717041, |
| "rewards/rejected": -5.376725196838379, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.8746666666666667, |
| "grad_norm": 2.342085599899292, |
| "learning_rate": 4.1631045476449865e-06, |
| "logits/chosen": -1.804836630821228, |
| "logits/rejected": -1.6348615884780884, |
| "logps/chosen": -240.3076934814453, |
| "logps/rejected": -318.75494384765625, |
| "loss": 0.1928, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.7394973039627075, |
| "rewards/margins": 4.6910576820373535, |
| "rewards/rejected": -5.430554389953613, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8853333333333333, |
| "grad_norm": 3.078958511352539, |
| "learning_rate": 4.141499133247391e-06, |
| "logits/chosen": -1.7097524404525757, |
| "logits/rejected": -1.6636264324188232, |
| "logps/chosen": -234.5639190673828, |
| "logps/rejected": -351.9319152832031, |
| "loss": 0.1463, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.5261866450309753, |
| "rewards/margins": 5.036253452301025, |
| "rewards/rejected": -5.562439441680908, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.896, |
| "grad_norm": 2.330395221710205, |
| "learning_rate": 4.119676024643217e-06, |
| "logits/chosen": -1.6960022449493408, |
| "logits/rejected": -1.5892505645751953, |
| "logps/chosen": -237.07608032226562, |
| "logps/rejected": -319.4405212402344, |
| "loss": 0.1522, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.46109694242477417, |
| "rewards/margins": 4.772611141204834, |
| "rewards/rejected": -5.233708381652832, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.9066666666666666, |
| "grad_norm": 2.2593789100646973, |
| "learning_rate": 4.0976381159944055e-06, |
| "logits/chosen": -1.7869155406951904, |
| "logits/rejected": -1.606461763381958, |
| "logps/chosen": -273.7130432128906, |
| "logps/rejected": -362.55267333984375, |
| "loss": 0.1593, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.8516896963119507, |
| "rewards/margins": 5.3220696449279785, |
| "rewards/rejected": -6.173758506774902, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.9173333333333333, |
| "grad_norm": 3.2557544708251953, |
| "learning_rate": 4.07538832994949e-06, |
| "logits/chosen": -1.758851408958435, |
| "logits/rejected": -1.6003106832504272, |
| "logps/chosen": -267.98980712890625, |
| "logps/rejected": -371.2702941894531, |
| "loss": 0.1952, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.1154661178588867, |
| "rewards/margins": 5.60113525390625, |
| "rewards/rejected": -6.716601371765137, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.928, |
| "grad_norm": 3.192758321762085, |
| "learning_rate": 4.0529296172560055e-06, |
| "logits/chosen": -1.8879740238189697, |
| "logits/rejected": -1.6979936361312866, |
| "logps/chosen": -250.65975952148438, |
| "logps/rejected": -344.8387451171875, |
| "loss": 0.1664, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.0759499073028564, |
| "rewards/margins": 5.6617536544799805, |
| "rewards/rejected": -6.737703800201416, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9386666666666666, |
| "grad_norm": 2.523839235305786, |
| "learning_rate": 4.030264956369158e-06, |
| "logits/chosen": -1.7966762781143188, |
| "logits/rejected": -1.7065531015396118, |
| "logps/chosen": -245.68002319335938, |
| "logps/rejected": -358.82720947265625, |
| "loss": 0.1692, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.0516901016235352, |
| "rewards/margins": 5.241422176361084, |
| "rewards/rejected": -6.293112277984619, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9493333333333334, |
| "grad_norm": 3.1646761894226074, |
| "learning_rate": 4.0073973530568274e-06, |
| "logits/chosen": -1.843031644821167, |
| "logits/rejected": -1.6471502780914307, |
| "logps/chosen": -257.5235900878906, |
| "logps/rejected": -372.1407165527344, |
| "loss": 0.1023, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.7290087938308716, |
| "rewards/margins": 6.054869651794434, |
| "rewards/rejected": -6.783879280090332, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 4.48504638671875, |
| "learning_rate": 3.9843298400009435e-06, |
| "logits/chosen": -1.7822949886322021, |
| "logits/rejected": -1.6918121576309204, |
| "logps/chosen": -251.33145141601562, |
| "logps/rejected": -366.915283203125, |
| "loss": 0.1788, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.2320232391357422, |
| "rewards/margins": 5.393434047698975, |
| "rewards/rejected": -6.625456809997559, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9706666666666667, |
| "grad_norm": 2.657050848007202, |
| "learning_rate": 3.961065476395296e-06, |
| "logits/chosen": -1.7469505071640015, |
| "logits/rejected": -1.6122243404388428, |
| "logps/chosen": -268.603759765625, |
| "logps/rejected": -365.6134338378906, |
| "loss": 0.1598, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.2172812223434448, |
| "rewards/margins": 5.346413612365723, |
| "rewards/rejected": -6.563694953918457, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9813333333333333, |
| "grad_norm": 2.8152854442596436, |
| "learning_rate": 3.937607347539823e-06, |
| "logits/chosen": -1.7389802932739258, |
| "logits/rejected": -1.6374263763427734, |
| "logps/chosen": -250.3136749267578, |
| "logps/rejected": -358.8951110839844, |
| "loss": 0.1151, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.1840031147003174, |
| "rewards/margins": 6.090756416320801, |
| "rewards/rejected": -7.274759769439697, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.992, |
| "grad_norm": 1.3589340448379517, |
| "learning_rate": 3.913958564431447e-06, |
| "logits/chosen": -1.9092724323272705, |
| "logits/rejected": -1.728168249130249, |
| "logps/chosen": -288.52215576171875, |
| "logps/rejected": -378.2078552246094, |
| "loss": 0.0895, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.3161489963531494, |
| "rewards/margins": 6.398454666137695, |
| "rewards/rejected": -7.714603424072266, |
| "step": 465 |
| }, |
| { |
| "epoch": 1.0021333333333333, |
| "grad_norm": 3.265164375305176, |
| "learning_rate": 3.89012226335149e-06, |
| "logits/chosen": -1.7122092247009277, |
| "logits/rejected": -1.6343815326690674, |
| "logps/chosen": -256.2262878417969, |
| "logps/rejected": -383.70245361328125, |
| "loss": 0.1672, |
| "rewards/accuracies": 0.9276315569877625, |
| "rewards/chosen": -1.3532406091690063, |
| "rewards/margins": 5.975292205810547, |
| "rewards/rejected": -7.328533172607422, |
| "step": 470 |
| }, |
| { |
| "epoch": 1.0128, |
| "grad_norm": 3.8232693672180176, |
| "learning_rate": 3.86610160544975e-06, |
| "logits/chosen": -1.8605287075042725, |
| "logits/rejected": -1.6789414882659912, |
| "logps/chosen": -258.9883728027344, |
| "logps/rejected": -362.35162353515625, |
| "loss": 0.0859, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.2631183862686157, |
| "rewards/margins": 6.069647789001465, |
| "rewards/rejected": -7.332766056060791, |
| "step": 475 |
| }, |
| { |
| "epoch": 1.0234666666666667, |
| "grad_norm": 2.178936719894409, |
| "learning_rate": 3.841899776325267e-06, |
| "logits/chosen": -1.6484073400497437, |
| "logits/rejected": -1.539050817489624, |
| "logps/chosen": -269.6177978515625, |
| "logps/rejected": -361.5688171386719, |
| "loss": 0.1179, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.2398256063461304, |
| "rewards/margins": 6.038259029388428, |
| "rewards/rejected": -7.278084754943848, |
| "step": 480 |
| }, |
| { |
| "epoch": 1.0341333333333333, |
| "grad_norm": 0.980421245098114, |
| "learning_rate": 3.817519985603855e-06, |
| "logits/chosen": -1.905623197555542, |
| "logits/rejected": -1.7547218799591064, |
| "logps/chosen": -250.7723846435547, |
| "logps/rejected": -385.75262451171875, |
| "loss": 0.0857, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6823866367340088, |
| "rewards/margins": 6.899411201477051, |
| "rewards/rejected": -8.58179759979248, |
| "step": 485 |
| }, |
| { |
| "epoch": 1.0448, |
| "grad_norm": 1.7659993171691895, |
| "learning_rate": 3.792965466512437e-06, |
| "logits/chosen": -1.7362083196640015, |
| "logits/rejected": -1.6089004278182983, |
| "logps/chosen": -272.89288330078125, |
| "logps/rejected": -345.1723327636719, |
| "loss": 0.1426, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.6883974075317383, |
| "rewards/margins": 5.6151299476623535, |
| "rewards/rejected": -7.303526878356934, |
| "step": 490 |
| }, |
| { |
| "epoch": 1.0554666666666668, |
| "grad_norm": 2.1790196895599365, |
| "learning_rate": 3.7682394754502687e-06, |
| "logits/chosen": -1.8438827991485596, |
| "logits/rejected": -1.7237058877944946, |
| "logps/chosen": -248.16622924804688, |
| "logps/rejected": -366.1410217285156, |
| "loss": 0.0907, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.643491506576538, |
| "rewards/margins": 6.8003034591674805, |
| "rewards/rejected": -8.443795204162598, |
| "step": 495 |
| }, |
| { |
| "epoch": 1.0661333333333334, |
| "grad_norm": 0.9579430222511292, |
| "learning_rate": 3.7433452915570685e-06, |
| "logits/chosen": -1.7718284130096436, |
| "logits/rejected": -1.6880018711090088, |
| "logps/chosen": -251.2505645751953, |
| "logps/rejected": -377.408203125, |
| "loss": 0.1142, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.4772014617919922, |
| "rewards/margins": 6.702824592590332, |
| "rewards/rejected": -8.180025100708008, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.0768, |
| "grad_norm": 2.1115434169769287, |
| "learning_rate": 3.718286216278143e-06, |
| "logits/chosen": -1.904197335243225, |
| "logits/rejected": -1.7636644840240479, |
| "logps/chosen": -245.1167449951172, |
| "logps/rejected": -346.50262451171875, |
| "loss": 0.1178, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9693208932876587, |
| "rewards/margins": 6.109748840332031, |
| "rewards/rejected": -8.079069137573242, |
| "step": 505 |
| }, |
| { |
| "epoch": 1.0874666666666666, |
| "grad_norm": 2.8808727264404297, |
| "learning_rate": 3.6930655729265557e-06, |
| "logits/chosen": -1.7936203479766846, |
| "logits/rejected": -1.7006099224090576, |
| "logps/chosen": -258.9440002441406, |
| "logps/rejected": -360.27838134765625, |
| "loss": 0.1172, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7141954898834229, |
| "rewards/margins": 6.435247898101807, |
| "rewards/rejected": -8.149443626403809, |
| "step": 510 |
| }, |
| { |
| "epoch": 1.0981333333333334, |
| "grad_norm": 2.3610522747039795, |
| "learning_rate": 3.667686706242386e-06, |
| "logits/chosen": -1.7922704219818115, |
| "logits/rejected": -1.697859764099121, |
| "logps/chosen": -243.87789916992188, |
| "logps/rejected": -360.91607666015625, |
| "loss": 0.1032, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.5759384632110596, |
| "rewards/margins": 6.990079402923584, |
| "rewards/rejected": -8.566018104553223, |
| "step": 515 |
| }, |
| { |
| "epoch": 1.1088, |
| "grad_norm": 1.427386999130249, |
| "learning_rate": 3.6421529819491576e-06, |
| "logits/chosen": -1.8297468423843384, |
| "logits/rejected": -1.7564407587051392, |
| "logps/chosen": -249.23031616210938, |
| "logps/rejected": -372.94390869140625, |
| "loss": 0.077, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6625449657440186, |
| "rewards/margins": 6.893601417541504, |
| "rewards/rejected": -8.556145668029785, |
| "step": 520 |
| }, |
| { |
| "epoch": 1.1194666666666666, |
| "grad_norm": 2.358030319213867, |
| "learning_rate": 3.6164677863074783e-06, |
| "logits/chosen": -1.7567555904388428, |
| "logits/rejected": -1.6881614923477173, |
| "logps/chosen": -255.1883087158203, |
| "logps/rejected": -393.2801818847656, |
| "loss": 0.098, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7937577962875366, |
| "rewards/margins": 7.25093936920166, |
| "rewards/rejected": -9.044698715209961, |
| "step": 525 |
| }, |
| { |
| "epoch": 1.1301333333333332, |
| "grad_norm": 2.982337713241577, |
| "learning_rate": 3.590634525665955e-06, |
| "logits/chosen": -1.8906761407852173, |
| "logits/rejected": -1.7957327365875244, |
| "logps/chosen": -246.13357543945312, |
| "logps/rejected": -395.7541198730469, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.8768450021743774, |
| "rewards/margins": 7.057843208312988, |
| "rewards/rejected": -8.934687614440918, |
| "step": 530 |
| }, |
| { |
| "epoch": 1.1408, |
| "grad_norm": 2.2151243686676025, |
| "learning_rate": 3.564656626009446e-06, |
| "logits/chosen": -1.837033987045288, |
| "logits/rejected": -1.7429161071777344, |
| "logps/chosen": -263.468994140625, |
| "logps/rejected": -405.4566955566406, |
| "loss": 0.1206, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.9729989767074585, |
| "rewards/margins": 6.8837761878967285, |
| "rewards/rejected": -8.85677433013916, |
| "step": 535 |
| }, |
| { |
| "epoch": 1.1514666666666666, |
| "grad_norm": 2.5153486728668213, |
| "learning_rate": 3.5385375325047167e-06, |
| "logits/chosen": -1.8511556386947632, |
| "logits/rejected": -1.6896768808364868, |
| "logps/chosen": -251.3939971923828, |
| "logps/rejected": -361.03143310546875, |
| "loss": 0.0876, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.2249603271484375, |
| "rewards/margins": 7.108116149902344, |
| "rewards/rejected": -8.333075523376465, |
| "step": 540 |
| }, |
| { |
| "epoch": 1.1621333333333332, |
| "grad_norm": 2.5955557823181152, |
| "learning_rate": 3.512280709043534e-06, |
| "logits/chosen": -1.696006417274475, |
| "logits/rejected": -1.5523803234100342, |
| "logps/chosen": -264.4658203125, |
| "logps/rejected": -375.38494873046875, |
| "loss": 0.1307, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.334220290184021, |
| "rewards/margins": 6.640855312347412, |
| "rewards/rejected": -7.975076198577881, |
| "step": 545 |
| }, |
| { |
| "epoch": 1.1728, |
| "grad_norm": 1.6547006368637085, |
| "learning_rate": 3.4858896377832966e-06, |
| "logits/chosen": -1.7349450588226318, |
| "logits/rejected": -1.7120225429534912, |
| "logps/chosen": -254.0203399658203, |
| "logps/rejected": -399.4454345703125, |
| "loss": 0.0766, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.3470942974090576, |
| "rewards/margins": 7.173849582672119, |
| "rewards/rejected": -8.520943641662598, |
| "step": 550 |
| }, |
| { |
| "epoch": 1.1834666666666667, |
| "grad_norm": 1.4159612655639648, |
| "learning_rate": 3.459367818685231e-06, |
| "logits/chosen": -1.807586431503296, |
| "logits/rejected": -1.7281315326690674, |
| "logps/chosen": -265.9685974121094, |
| "logps/rejected": -377.4097595214844, |
| "loss": 0.1151, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.1236010789871216, |
| "rewards/margins": 6.955357551574707, |
| "rewards/rejected": -8.078958511352539, |
| "step": 555 |
| }, |
| { |
| "epoch": 1.1941333333333333, |
| "grad_norm": 1.7317302227020264, |
| "learning_rate": 3.432718769050228e-06, |
| "logits/chosen": -1.661529779434204, |
| "logits/rejected": -1.6265112161636353, |
| "logps/chosen": -254.3848419189453, |
| "logps/rejected": -353.63995361328125, |
| "loss": 0.124, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.3399780988693237, |
| "rewards/margins": 6.360163688659668, |
| "rewards/rejected": -7.700140953063965, |
| "step": 560 |
| }, |
| { |
| "epoch": 1.2048, |
| "grad_norm": 4.254868507385254, |
| "learning_rate": 3.405946023052386e-06, |
| "logits/chosen": -1.791841745376587, |
| "logits/rejected": -1.6956441402435303, |
| "logps/chosen": -242.0591278076172, |
| "logps/rejected": -372.5768127441406, |
| "loss": 0.1307, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.4513027667999268, |
| "rewards/margins": 6.603933811187744, |
| "rewards/rejected": -8.055237770080566, |
| "step": 565 |
| }, |
| { |
| "epoch": 1.2154666666666667, |
| "grad_norm": 1.8187251091003418, |
| "learning_rate": 3.3790531312703055e-06, |
| "logits/chosen": -1.7333256006240845, |
| "logits/rejected": -1.6957298517227173, |
| "logps/chosen": -259.1826477050781, |
| "logps/rejected": -400.0516662597656, |
| "loss": 0.0561, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6751782894134521, |
| "rewards/margins": 7.766973972320557, |
| "rewards/rejected": -9.442152976989746, |
| "step": 570 |
| }, |
| { |
| "epoch": 1.2261333333333333, |
| "grad_norm": 2.6395459175109863, |
| "learning_rate": 3.35204366021622e-06, |
| "logits/chosen": -1.7824958562850952, |
| "logits/rejected": -1.6265745162963867, |
| "logps/chosen": -256.4351501464844, |
| "logps/rejected": -372.00408935546875, |
| "loss": 0.111, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.2362754344940186, |
| "rewards/margins": 7.035012722015381, |
| "rewards/rejected": -8.27128791809082, |
| "step": 575 |
| }, |
| { |
| "epoch": 1.2368000000000001, |
| "grad_norm": 3.0449585914611816, |
| "learning_rate": 3.3249211918630055e-06, |
| "logits/chosen": -1.7625986337661743, |
| "logits/rejected": -1.6265180110931396, |
| "logps/chosen": -248.1802978515625, |
| "logps/rejected": -383.77593994140625, |
| "loss": 0.1145, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.305022954940796, |
| "rewards/margins": 7.372431755065918, |
| "rewards/rejected": -8.677454948425293, |
| "step": 580 |
| }, |
| { |
| "epoch": 1.2474666666666667, |
| "grad_norm": 1.849378228187561, |
| "learning_rate": 3.2976893231691424e-06, |
| "logits/chosen": -1.751132607460022, |
| "logits/rejected": -1.6718765497207642, |
| "logps/chosen": -267.318359375, |
| "logps/rejected": -364.69854736328125, |
| "loss": 0.1285, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.278596043586731, |
| "rewards/margins": 6.7831621170043945, |
| "rewards/rejected": -8.06175708770752, |
| "step": 585 |
| }, |
| { |
| "epoch": 1.2581333333333333, |
| "grad_norm": 2.360128164291382, |
| "learning_rate": 3.270351665601691e-06, |
| "logits/chosen": -1.7772048711776733, |
| "logits/rejected": -1.6471363306045532, |
| "logps/chosen": -258.44476318359375, |
| "logps/rejected": -384.2423400878906, |
| "loss": 0.0679, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.230088472366333, |
| "rewards/margins": 7.426251411437988, |
| "rewards/rejected": -8.656339645385742, |
| "step": 590 |
| }, |
| { |
| "epoch": 1.2688, |
| "grad_norm": 2.0064170360565186, |
| "learning_rate": 3.2429118446573417e-06, |
| "logits/chosen": -1.7410202026367188, |
| "logits/rejected": -1.6303008794784546, |
| "logps/chosen": -251.84634399414062, |
| "logps/rejected": -384.3974609375, |
| "loss": 0.111, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.8702796697616577, |
| "rewards/margins": 6.885188102722168, |
| "rewards/rejected": -8.755468368530273, |
| "step": 595 |
| }, |
| { |
| "epoch": 1.2794666666666665, |
| "grad_norm": 5.854542255401611, |
| "learning_rate": 3.215373499381602e-06, |
| "logits/chosen": -1.774011254310608, |
| "logits/rejected": -1.6293554306030273, |
| "logps/chosen": -270.77960205078125, |
| "logps/rejected": -352.97100830078125, |
| "loss": 0.2022, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.5606591701507568, |
| "rewards/margins": 6.231796741485596, |
| "rewards/rejected": -7.792455196380615, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.2901333333333334, |
| "grad_norm": 1.544241189956665, |
| "learning_rate": 3.1877402818861954e-06, |
| "logits/chosen": -1.8023827075958252, |
| "logits/rejected": -1.7564594745635986, |
| "logps/chosen": -256.7626953125, |
| "logps/rejected": -388.5929870605469, |
| "loss": 0.101, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.5191371440887451, |
| "rewards/margins": 7.524176120758057, |
| "rewards/rejected": -9.043313980102539, |
| "step": 605 |
| }, |
| { |
| "epoch": 1.3008, |
| "grad_norm": 1.2384198904037476, |
| "learning_rate": 3.1600158568647145e-06, |
| "logits/chosen": -1.7687852382659912, |
| "logits/rejected": -1.7042837142944336, |
| "logps/chosen": -261.32220458984375, |
| "logps/rejected": -395.2125549316406, |
| "loss": 0.1093, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.778892159461975, |
| "rewards/margins": 7.3676629066467285, |
| "rewards/rejected": -9.146554946899414, |
| "step": 610 |
| }, |
| { |
| "epoch": 1.3114666666666666, |
| "grad_norm": 3.274482011795044, |
| "learning_rate": 3.1322039011066172e-06, |
| "logits/chosen": -1.742261528968811, |
| "logits/rejected": -1.6239099502563477, |
| "logps/chosen": -272.2462463378906, |
| "logps/rejected": -370.30645751953125, |
| "loss": 0.139, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.2790501117706299, |
| "rewards/margins": 6.3297247886657715, |
| "rewards/rejected": -7.6087751388549805, |
| "step": 615 |
| }, |
| { |
| "epoch": 1.3221333333333334, |
| "grad_norm": 3.0375425815582275, |
| "learning_rate": 3.1043081030096125e-06, |
| "logits/chosen": -1.7067797183990479, |
| "logits/rejected": -1.5599234104156494, |
| "logps/chosen": -254.21237182617188, |
| "logps/rejected": -340.0577697753906, |
| "loss": 0.1824, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.123809576034546, |
| "rewards/margins": 5.8070549964904785, |
| "rewards/rejected": -6.930863857269287, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.3328, |
| "grad_norm": 2.601759672164917, |
| "learning_rate": 3.0763321620905072e-06, |
| "logits/chosen": -1.6933444738388062, |
| "logits/rejected": -1.5981481075286865, |
| "logps/chosen": -262.598876953125, |
| "logps/rejected": -358.65301513671875, |
| "loss": 0.1607, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.4247936010360718, |
| "rewards/margins": 5.9775896072387695, |
| "rewards/rejected": -7.402383327484131, |
| "step": 625 |
| }, |
| { |
| "epoch": 1.3434666666666666, |
| "grad_norm": 1.8564891815185547, |
| "learning_rate": 3.048279788494583e-06, |
| "logits/chosen": -1.825899362564087, |
| "logits/rejected": -1.6926910877227783, |
| "logps/chosen": -229.56484985351562, |
| "logps/rejected": -356.9173278808594, |
| "loss": 0.1117, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.2786517143249512, |
| "rewards/margins": 6.977616310119629, |
| "rewards/rejected": -8.256268501281738, |
| "step": 630 |
| }, |
| { |
| "epoch": 1.3541333333333334, |
| "grad_norm": 2.049483060836792, |
| "learning_rate": 3.0201547025035553e-06, |
| "logits/chosen": -1.7682701349258423, |
| "logits/rejected": -1.605406403541565, |
| "logps/chosen": -273.62322998046875, |
| "logps/rejected": -379.70440673828125, |
| "loss": 0.0778, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.0921200513839722, |
| "rewards/margins": 6.923942565917969, |
| "rewards/rejected": -8.016063690185547, |
| "step": 635 |
| }, |
| { |
| "epoch": 1.3648, |
| "grad_norm": 3.9953954219818115, |
| "learning_rate": 2.9919606340421963e-06, |
| "logits/chosen": -1.7687931060791016, |
| "logits/rejected": -1.5742592811584473, |
| "logps/chosen": -267.0946044921875, |
| "logps/rejected": -376.08544921875, |
| "loss": 0.0946, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3415029048919678, |
| "rewards/margins": 7.037326812744141, |
| "rewards/rejected": -8.378829956054688, |
| "step": 640 |
| }, |
| { |
| "epoch": 1.3754666666666666, |
| "grad_norm": 1.9268256425857544, |
| "learning_rate": 2.9637013221836723e-06, |
| "logits/chosen": -1.9088714122772217, |
| "logits/rejected": -1.6881043910980225, |
| "logps/chosen": -247.6619415283203, |
| "logps/rejected": -370.9812316894531, |
| "loss": 0.1035, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.0276373624801636, |
| "rewards/margins": 6.4521050453186035, |
| "rewards/rejected": -7.479742527008057, |
| "step": 645 |
| }, |
| { |
| "epoch": 1.3861333333333334, |
| "grad_norm": 2.7961528301239014, |
| "learning_rate": 2.9353805146536762e-06, |
| "logits/chosen": -1.7485949993133545, |
| "logits/rejected": -1.6905990839004517, |
| "logps/chosen": -258.29876708984375, |
| "logps/rejected": -363.9190979003906, |
| "loss": 0.1327, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.9902496337890625, |
| "rewards/margins": 6.535128593444824, |
| "rewards/rejected": -7.525378227233887, |
| "step": 650 |
| }, |
| { |
| "epoch": 1.3968, |
| "grad_norm": 2.9360063076019287, |
| "learning_rate": 2.907001967333401e-06, |
| "logits/chosen": -1.8041868209838867, |
| "logits/rejected": -1.718971610069275, |
| "logps/chosen": -231.60531616210938, |
| "logps/rejected": -374.5210266113281, |
| "loss": 0.1069, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.2519428730010986, |
| "rewards/margins": 6.751153469085693, |
| "rewards/rejected": -8.003096580505371, |
| "step": 655 |
| }, |
| { |
| "epoch": 1.4074666666666666, |
| "grad_norm": 2.948979377746582, |
| "learning_rate": 2.878569443761442e-06, |
| "logits/chosen": -1.7529418468475342, |
| "logits/rejected": -1.5835460424423218, |
| "logps/chosen": -276.1628723144531, |
| "logps/rejected": -353.0192565917969, |
| "loss": 0.1458, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.3621124029159546, |
| "rewards/margins": 5.750298500061035, |
| "rewards/rejected": -7.112410545349121, |
| "step": 660 |
| }, |
| { |
| "epoch": 1.4181333333333335, |
| "grad_norm": 3.252516031265259, |
| "learning_rate": 2.85008671463468e-06, |
| "logits/chosen": -1.7881982326507568, |
| "logits/rejected": -1.6824554204940796, |
| "logps/chosen": -278.9662780761719, |
| "logps/rejected": -380.4771423339844, |
| "loss": 0.1478, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.5313074588775635, |
| "rewards/margins": 6.1621809005737305, |
| "rewards/rejected": -7.693489074707031, |
| "step": 665 |
| }, |
| { |
| "epoch": 1.4288, |
| "grad_norm": 1.8899283409118652, |
| "learning_rate": 2.8215575573082127e-06, |
| "logits/chosen": -1.7420358657836914, |
| "logits/rejected": -1.6729049682617188, |
| "logps/chosen": -270.51031494140625, |
| "logps/rejected": -392.34112548828125, |
| "loss": 0.0952, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.5278486013412476, |
| "rewards/margins": 6.879241943359375, |
| "rewards/rejected": -8.40709114074707, |
| "step": 670 |
| }, |
| { |
| "epoch": 1.4394666666666667, |
| "grad_norm": 1.3593741655349731, |
| "learning_rate": 2.792985755294404e-06, |
| "logits/chosen": -1.8171350955963135, |
| "logits/rejected": -1.7248483896255493, |
| "logps/chosen": -249.7633514404297, |
| "logps/rejected": -382.9278259277344, |
| "loss": 0.0659, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6088777780532837, |
| "rewards/margins": 7.664944648742676, |
| "rewards/rejected": -9.273821830749512, |
| "step": 675 |
| }, |
| { |
| "epoch": 1.4501333333333333, |
| "grad_norm": 3.274573564529419, |
| "learning_rate": 2.764375097761122e-06, |
| "logits/chosen": -1.5862953662872314, |
| "logits/rejected": -1.532132863998413, |
| "logps/chosen": -254.4930877685547, |
| "logps/rejected": -369.3152160644531, |
| "loss": 0.1444, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.3496243953704834, |
| "rewards/margins": 6.570565223693848, |
| "rewards/rejected": -7.92018985748291, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.4607999999999999, |
| "grad_norm": 2.0709228515625, |
| "learning_rate": 2.73572937902922e-06, |
| "logits/chosen": -1.7570823431015015, |
| "logits/rejected": -1.5986483097076416, |
| "logps/chosen": -265.8306579589844, |
| "logps/rejected": -376.62188720703125, |
| "loss": 0.099, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.4001950025558472, |
| "rewards/margins": 6.844396114349365, |
| "rewards/rejected": -8.244590759277344, |
| "step": 685 |
| }, |
| { |
| "epoch": 1.4714666666666667, |
| "grad_norm": 3.2238757610321045, |
| "learning_rate": 2.7070523980693362e-06, |
| "logits/chosen": -1.6330959796905518, |
| "logits/rejected": -1.511679768562317, |
| "logps/chosen": -253.7584228515625, |
| "logps/rejected": -350.31829833984375, |
| "loss": 0.1107, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.0016086101531982, |
| "rewards/margins": 6.8228440284729, |
| "rewards/rejected": -7.8244524002075195, |
| "step": 690 |
| }, |
| { |
| "epoch": 1.4821333333333333, |
| "grad_norm": 1.463384747505188, |
| "learning_rate": 2.678347957998081e-06, |
| "logits/chosen": -1.7908287048339844, |
| "logits/rejected": -1.6633214950561523, |
| "logps/chosen": -249.6637725830078, |
| "logps/rejected": -363.63128662109375, |
| "loss": 0.0945, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6576259136199951, |
| "rewards/margins": 7.192323207855225, |
| "rewards/rejected": -8.849950790405273, |
| "step": 695 |
| }, |
| { |
| "epoch": 1.4928, |
| "grad_norm": 2.0910377502441406, |
| "learning_rate": 2.649619865573666e-06, |
| "logits/chosen": -1.7448034286499023, |
| "logits/rejected": -1.6555458307266235, |
| "logps/chosen": -250.4905242919922, |
| "logps/rejected": -397.0603942871094, |
| "loss": 0.0926, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.482170820236206, |
| "rewards/margins": 7.455416202545166, |
| "rewards/rejected": -8.93758773803711, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.5034666666666667, |
| "grad_norm": 3.544623851776123, |
| "learning_rate": 2.620871930691058e-06, |
| "logits/chosen": -1.750417709350586, |
| "logits/rejected": -1.6637424230575562, |
| "logps/chosen": -259.9458312988281, |
| "logps/rejected": -364.2375183105469, |
| "loss": 0.0947, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.3259789943695068, |
| "rewards/margins": 6.424938201904297, |
| "rewards/rejected": -7.750917911529541, |
| "step": 705 |
| }, |
| { |
| "epoch": 1.5141333333333333, |
| "grad_norm": 1.3879026174545288, |
| "learning_rate": 2.5921079658767152e-06, |
| "logits/chosen": -1.7787948846817017, |
| "logits/rejected": -1.7029085159301758, |
| "logps/chosen": -246.4046630859375, |
| "logps/rejected": -381.16473388671875, |
| "loss": 0.1354, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7129465341567993, |
| "rewards/margins": 7.154146671295166, |
| "rewards/rejected": -8.867093086242676, |
| "step": 710 |
| }, |
| { |
| "epoch": 1.5248, |
| "grad_norm": 2.2066287994384766, |
| "learning_rate": 2.56333178578297e-06, |
| "logits/chosen": -1.7623145580291748, |
| "logits/rejected": -1.6360619068145752, |
| "logps/chosen": -227.5376434326172, |
| "logps/rejected": -355.4570617675781, |
| "loss": 0.1361, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.6756374835968018, |
| "rewards/margins": 7.0356597900390625, |
| "rewards/rejected": -8.711296081542969, |
| "step": 715 |
| }, |
| { |
| "epoch": 1.5354666666666668, |
| "grad_norm": 2.268791437149048, |
| "learning_rate": 2.534547206682136e-06, |
| "logits/chosen": -1.7191269397735596, |
| "logits/rejected": -1.661130666732788, |
| "logps/chosen": -258.20086669921875, |
| "logps/rejected": -383.30535888671875, |
| "loss": 0.094, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.680588960647583, |
| "rewards/margins": 7.147183418273926, |
| "rewards/rejected": -8.82777214050293, |
| "step": 720 |
| }, |
| { |
| "epoch": 1.5461333333333334, |
| "grad_norm": 1.6195091009140015, |
| "learning_rate": 2.5057580459603947e-06, |
| "logits/chosen": -1.8001463413238525, |
| "logits/rejected": -1.6632938385009766, |
| "logps/chosen": -244.26486206054688, |
| "logps/rejected": -393.6264343261719, |
| "loss": 0.0438, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.2562836408615112, |
| "rewards/margins": 7.85473108291626, |
| "rewards/rejected": -9.111015319824219, |
| "step": 725 |
| }, |
| { |
| "epoch": 1.5568, |
| "grad_norm": 3.6901800632476807, |
| "learning_rate": 2.4769681216115384e-06, |
| "logits/chosen": -1.7932723760604858, |
| "logits/rejected": -1.6791099309921265, |
| "logps/chosen": -266.78314208984375, |
| "logps/rejected": -386.56964111328125, |
| "loss": 0.1069, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.260191798210144, |
| "rewards/margins": 7.321366786956787, |
| "rewards/rejected": -8.581558227539062, |
| "step": 730 |
| }, |
| { |
| "epoch": 1.5674666666666668, |
| "grad_norm": 1.7813483476638794, |
| "learning_rate": 2.448181251730632e-06, |
| "logits/chosen": -1.673689842224121, |
| "logits/rejected": -1.5504353046417236, |
| "logps/chosen": -266.9430236816406, |
| "logps/rejected": -365.57745361328125, |
| "loss": 0.0846, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.159118890762329, |
| "rewards/margins": 7.304617404937744, |
| "rewards/rejected": -8.463736534118652, |
| "step": 735 |
| }, |
| { |
| "epoch": 1.5781333333333334, |
| "grad_norm": 3.4884979724884033, |
| "learning_rate": 2.419401254007658e-06, |
| "logits/chosen": -1.7130966186523438, |
| "logits/rejected": -1.661629319190979, |
| "logps/chosen": -244.4851837158203, |
| "logps/rejected": -335.0830383300781, |
| "loss": 0.1607, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.1306991577148438, |
| "rewards/margins": 6.248559474945068, |
| "rewards/rejected": -7.379258632659912, |
| "step": 740 |
| }, |
| { |
| "epoch": 1.5888, |
| "grad_norm": 1.881792664527893, |
| "learning_rate": 2.390631945221221e-06, |
| "logits/chosen": -1.8140137195587158, |
| "logits/rejected": -1.6874796152114868, |
| "logps/chosen": -251.9772186279297, |
| "logps/rejected": -405.4315185546875, |
| "loss": 0.114, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.2588303089141846, |
| "rewards/margins": 7.343568325042725, |
| "rewards/rejected": -8.602398872375488, |
| "step": 745 |
| }, |
| { |
| "epoch": 1.5994666666666668, |
| "grad_norm": 3.2374913692474365, |
| "learning_rate": 2.361877140732366e-06, |
| "logits/chosen": -1.7451698780059814, |
| "logits/rejected": -1.6588739156723022, |
| "logps/chosen": -244.10238647460938, |
| "logps/rejected": -370.6669616699219, |
| "loss": 0.0985, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.2883882522583008, |
| "rewards/margins": 7.572729587554932, |
| "rewards/rejected": -8.861119270324707, |
| "step": 750 |
| }, |
| { |
| "epoch": 1.6101333333333332, |
| "grad_norm": 1.0592817068099976, |
| "learning_rate": 2.333140653978592e-06, |
| "logits/chosen": -1.7900444269180298, |
| "logits/rejected": -1.6860978603363037, |
| "logps/chosen": -236.3094482421875, |
| "logps/rejected": -374.94378662109375, |
| "loss": 0.0955, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.9188971519470215, |
| "rewards/margins": 7.50311279296875, |
| "rewards/rejected": -8.42201042175293, |
| "step": 755 |
| }, |
| { |
| "epoch": 1.6208, |
| "grad_norm": 3.37741756439209, |
| "learning_rate": 2.3044262959681156e-06, |
| "logits/chosen": -1.7714325189590454, |
| "logits/rejected": -1.6465963125228882, |
| "logps/chosen": -236.4646759033203, |
| "logps/rejected": -370.81890869140625, |
| "loss": 0.1222, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.4455827474594116, |
| "rewards/margins": 7.438872337341309, |
| "rewards/rejected": -8.884454727172852, |
| "step": 760 |
| }, |
| { |
| "epoch": 1.6314666666666666, |
| "grad_norm": 1.3188512325286865, |
| "learning_rate": 2.2757378747744575e-06, |
| "logits/chosen": -1.7838866710662842, |
| "logits/rejected": -1.6729694604873657, |
| "logps/chosen": -257.4605407714844, |
| "logps/rejected": -385.75311279296875, |
| "loss": 0.082, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.1314411163330078, |
| "rewards/margins": 7.492845058441162, |
| "rewards/rejected": -8.624286651611328, |
| "step": 765 |
| }, |
| { |
| "epoch": 1.6421333333333332, |
| "grad_norm": 3.832404136657715, |
| "learning_rate": 2.24707919503142e-06, |
| "logits/chosen": -1.7909473180770874, |
| "logits/rejected": -1.700435996055603, |
| "logps/chosen": -225.17324829101562, |
| "logps/rejected": -385.271484375, |
| "loss": 0.061, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.3442027568817139, |
| "rewards/margins": 7.862277984619141, |
| "rewards/rejected": -9.206480026245117, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.6528, |
| "grad_norm": 1.5521408319473267, |
| "learning_rate": 2.2184540574285213e-06, |
| "logits/chosen": -1.6965129375457764, |
| "logits/rejected": -1.5588537454605103, |
| "logps/chosen": -269.69244384765625, |
| "logps/rejected": -360.9696960449219, |
| "loss": 0.1064, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.2951409816741943, |
| "rewards/margins": 7.005385398864746, |
| "rewards/rejected": -8.30052661895752, |
| "step": 775 |
| }, |
| { |
| "epoch": 1.6634666666666666, |
| "grad_norm": 2.485806703567505, |
| "learning_rate": 2.1898662582069486e-06, |
| "logits/chosen": -1.7989184856414795, |
| "logits/rejected": -1.713746428489685, |
| "logps/chosen": -260.45458984375, |
| "logps/rejected": -368.6556396484375, |
| "loss": 0.1179, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.8484270572662354, |
| "rewards/margins": 7.124408721923828, |
| "rewards/rejected": -8.972835540771484, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.6741333333333333, |
| "grad_norm": 4.892862319946289, |
| "learning_rate": 2.1613195886561044e-06, |
| "logits/chosen": -1.7144527435302734, |
| "logits/rejected": -1.664764404296875, |
| "logps/chosen": -230.86538696289062, |
| "logps/rejected": -350.9761657714844, |
| "loss": 0.1218, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.329289197921753, |
| "rewards/margins": 7.392025947570801, |
| "rewards/rejected": -8.721314430236816, |
| "step": 785 |
| }, |
| { |
| "epoch": 1.6848, |
| "grad_norm": 6.508184909820557, |
| "learning_rate": 2.132817834610811e-06, |
| "logits/chosen": -1.7057596445083618, |
| "logits/rejected": -1.6182563304901123, |
| "logps/chosen": -253.01986694335938, |
| "logps/rejected": -384.61895751953125, |
| "loss": 0.1046, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.4259307384490967, |
| "rewards/margins": 7.484774589538574, |
| "rewards/rejected": -8.910707473754883, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.6954666666666667, |
| "grad_norm": 2.587836265563965, |
| "learning_rate": 2.1043647759492338e-06, |
| "logits/chosen": -1.7026351690292358, |
| "logits/rejected": -1.6045818328857422, |
| "logps/chosen": -267.7701721191406, |
| "logps/rejected": -374.3067321777344, |
| "loss": 0.1003, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.1852244138717651, |
| "rewards/margins": 7.090452671051025, |
| "rewards/rejected": -8.275677680969238, |
| "step": 795 |
| }, |
| { |
| "epoch": 1.7061333333333333, |
| "grad_norm": 3.841643810272217, |
| "learning_rate": 2.075964186091599e-06, |
| "logits/chosen": -1.7560237646102905, |
| "logits/rejected": -1.7158886194229126, |
| "logps/chosen": -267.0213928222656, |
| "logps/rejected": -395.33831787109375, |
| "loss": 0.1109, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.816641092300415, |
| "rewards/margins": 7.103610992431641, |
| "rewards/rejected": -8.920252799987793, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.7168, |
| "grad_norm": 3.8351080417633057, |
| "learning_rate": 2.047619831499765e-06, |
| "logits/chosen": -1.678251028060913, |
| "logits/rejected": -1.561795949935913, |
| "logps/chosen": -245.37661743164062, |
| "logps/rejected": -384.1910400390625, |
| "loss": 0.0985, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.9903218150138855, |
| "rewards/margins": 7.894017219543457, |
| "rewards/rejected": -8.884339332580566, |
| "step": 805 |
| }, |
| { |
| "epoch": 1.7274666666666667, |
| "grad_norm": 3.4570720195770264, |
| "learning_rate": 2.0193354711777155e-06, |
| "logits/chosen": -1.7100801467895508, |
| "logits/rejected": -1.5458927154541016, |
| "logps/chosen": -284.8987731933594, |
| "logps/rejected": -392.1943359375, |
| "loss": 0.131, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.7098143100738525, |
| "rewards/margins": 7.403327941894531, |
| "rewards/rejected": -9.113142013549805, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.7381333333333333, |
| "grad_norm": 3.687009334564209, |
| "learning_rate": 1.991114856173048e-06, |
| "logits/chosen": -1.8737754821777344, |
| "logits/rejected": -1.7086925506591797, |
| "logps/chosen": -241.7539520263672, |
| "logps/rejected": -365.61285400390625, |
| "loss": 0.0872, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.461383581161499, |
| "rewards/margins": 7.248700141906738, |
| "rewards/rejected": -8.710083961486816, |
| "step": 815 |
| }, |
| { |
| "epoch": 1.7488000000000001, |
| "grad_norm": 1.9755408763885498, |
| "learning_rate": 1.9629617290795115e-06, |
| "logits/chosen": -1.752289056777954, |
| "logits/rejected": -1.6548378467559814, |
| "logps/chosen": -236.4134979248047, |
| "logps/rejected": -373.27874755859375, |
| "loss": 0.094, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.2346256971359253, |
| "rewards/margins": 7.46105432510376, |
| "rewards/rejected": -8.695680618286133, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.7594666666666665, |
| "grad_norm": 1.1113483905792236, |
| "learning_rate": 1.934879823540663e-06, |
| "logits/chosen": -1.7978004217147827, |
| "logits/rejected": -1.6400505304336548, |
| "logps/chosen": -250.33908081054688, |
| "logps/rejected": -393.55609130859375, |
| "loss": 0.0734, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.497048258781433, |
| "rewards/margins": 7.612870693206787, |
| "rewards/rejected": -9.109919548034668, |
| "step": 825 |
| }, |
| { |
| "epoch": 1.7701333333333333, |
| "grad_norm": 6.385664939880371, |
| "learning_rate": 1.9068728637547195e-06, |
| "logits/chosen": -1.766744613647461, |
| "logits/rejected": -1.5946464538574219, |
| "logps/chosen": -267.80242919921875, |
| "logps/rejected": -375.2735900878906, |
| "loss": 0.1224, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.3728525638580322, |
| "rewards/margins": 7.434065818786621, |
| "rewards/rejected": -8.806917190551758, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.7808000000000002, |
| "grad_norm": 3.626858949661255, |
| "learning_rate": 1.878944563980658e-06, |
| "logits/chosen": -1.7465204000473022, |
| "logits/rejected": -1.6045042276382446, |
| "logps/chosen": -252.1056671142578, |
| "logps/rejected": -388.88665771484375, |
| "loss": 0.0827, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.755131483078003, |
| "rewards/margins": 7.423327445983887, |
| "rewards/rejected": -9.178459167480469, |
| "step": 835 |
| }, |
| { |
| "epoch": 1.7914666666666665, |
| "grad_norm": 1.526719093322754, |
| "learning_rate": 1.85109862804563e-06, |
| "logits/chosen": -1.7173770666122437, |
| "logits/rejected": -1.6473115682601929, |
| "logps/chosen": -275.9317932128906, |
| "logps/rejected": -405.3008117675781, |
| "loss": 0.0705, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.4180126190185547, |
| "rewards/margins": 8.002815246582031, |
| "rewards/rejected": -9.420827865600586, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.8021333333333334, |
| "grad_norm": 4.655531406402588, |
| "learning_rate": 1.823338748853766e-06, |
| "logits/chosen": -1.8110342025756836, |
| "logits/rejected": -1.6926085948944092, |
| "logps/chosen": -257.8267822265625, |
| "logps/rejected": -390.58929443359375, |
| "loss": 0.1022, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7875226736068726, |
| "rewards/margins": 7.7925848960876465, |
| "rewards/rejected": -9.580107688903809, |
| "step": 845 |
| }, |
| { |
| "epoch": 1.8128, |
| "grad_norm": 4.3262434005737305, |
| "learning_rate": 1.7956686078964257e-06, |
| "logits/chosen": -1.8735952377319336, |
| "logits/rejected": -1.7246639728546143, |
| "logps/chosen": -238.3771514892578, |
| "logps/rejected": -385.5560302734375, |
| "loss": 0.0842, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.257483720779419, |
| "rewards/margins": 7.96505069732666, |
| "rewards/rejected": -9.222535133361816, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.8234666666666666, |
| "grad_norm": 2.52714467048645, |
| "learning_rate": 1.7680918747639608e-06, |
| "logits/chosen": -1.8481733798980713, |
| "logits/rejected": -1.7482507228851318, |
| "logps/chosen": -270.3813171386719, |
| "logps/rejected": -388.9419860839844, |
| "loss": 0.0817, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8209350109100342, |
| "rewards/margins": 7.654992580413818, |
| "rewards/rejected": -9.47592830657959, |
| "step": 855 |
| }, |
| { |
| "epoch": 1.8341333333333334, |
| "grad_norm": 1.7736057043075562, |
| "learning_rate": 1.740612206659057e-06, |
| "logits/chosen": -1.8067420721054077, |
| "logits/rejected": -1.7217512130737305, |
| "logps/chosen": -244.73971557617188, |
| "logps/rejected": -391.7164306640625, |
| "loss": 0.0884, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6112474203109741, |
| "rewards/margins": 8.113238334655762, |
| "rewards/rejected": -9.724488258361816, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.8448, |
| "grad_norm": 2.050666570663452, |
| "learning_rate": 1.7132332479117204e-06, |
| "logits/chosen": -1.8168611526489258, |
| "logits/rejected": -1.6906824111938477, |
| "logps/chosen": -247.4569091796875, |
| "logps/rejected": -366.1076965332031, |
| "loss": 0.1243, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.388657808303833, |
| "rewards/margins": 7.390578269958496, |
| "rewards/rejected": -8.77923583984375, |
| "step": 865 |
| }, |
| { |
| "epoch": 1.8554666666666666, |
| "grad_norm": 3.22883677482605, |
| "learning_rate": 1.6859586294959666e-06, |
| "logits/chosen": -1.6801704168319702, |
| "logits/rejected": -1.598153829574585, |
| "logps/chosen": -260.2568664550781, |
| "logps/rejected": -403.77679443359375, |
| "loss": 0.0706, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6268384456634521, |
| "rewards/margins": 9.174223899841309, |
| "rewards/rejected": -10.801061630249023, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.8661333333333334, |
| "grad_norm": 1.8140406608581543, |
| "learning_rate": 1.6587919685482884e-06, |
| "logits/chosen": -1.8333885669708252, |
| "logits/rejected": -1.7453978061676025, |
| "logps/chosen": -252.1963653564453, |
| "logps/rejected": -375.7099609375, |
| "loss": 0.1162, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9302170276641846, |
| "rewards/margins": 7.10568904876709, |
| "rewards/rejected": -9.035905838012695, |
| "step": 875 |
| }, |
| { |
| "epoch": 1.8768, |
| "grad_norm": 3.0864861011505127, |
| "learning_rate": 1.6317368678879497e-06, |
| "logits/chosen": -1.6215035915374756, |
| "logits/rejected": -1.5060207843780518, |
| "logps/chosen": -260.1852111816406, |
| "logps/rejected": -365.7777099609375, |
| "loss": 0.1116, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.2610294818878174, |
| "rewards/margins": 7.266331672668457, |
| "rewards/rejected": -8.527360916137695, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.8874666666666666, |
| "grad_norm": 6.364953994750977, |
| "learning_rate": 1.6047969155391856e-06, |
| "logits/chosen": -1.6732227802276611, |
| "logits/rejected": -1.6195056438446045, |
| "logps/chosen": -268.6829833984375, |
| "logps/rejected": -414.5812072753906, |
| "loss": 0.1082, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.7019331455230713, |
| "rewards/margins": 8.101677894592285, |
| "rewards/rejected": -9.80361270904541, |
| "step": 885 |
| }, |
| { |
| "epoch": 1.8981333333333335, |
| "grad_norm": 1.6536486148834229, |
| "learning_rate": 1.5779756842553617e-06, |
| "logits/chosen": -1.7743663787841797, |
| "logits/rejected": -1.684413194656372, |
| "logps/chosen": -267.4327087402344, |
| "logps/rejected": -410.91094970703125, |
| "loss": 0.0409, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.7548547983169556, |
| "rewards/margins": 8.745696067810059, |
| "rewards/rejected": -10.500551223754883, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.9088, |
| "grad_norm": 6.427159786224365, |
| "learning_rate": 1.551276731045157e-06, |
| "logits/chosen": -1.7461345195770264, |
| "logits/rejected": -1.6096042394638062, |
| "logps/chosen": -288.79302978515625, |
| "logps/rejected": -365.92401123046875, |
| "loss": 0.1409, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.4583120346069336, |
| "rewards/margins": 7.0046586990356445, |
| "rewards/rejected": -8.462970733642578, |
| "step": 895 |
| }, |
| { |
| "epoch": 1.9194666666666667, |
| "grad_norm": 2.0357890129089355, |
| "learning_rate": 1.5247035967008369e-06, |
| "logits/chosen": -1.6722333431243896, |
| "logits/rejected": -1.5082788467407227, |
| "logps/chosen": -270.37371826171875, |
| "logps/rejected": -395.6129150390625, |
| "loss": 0.0574, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.7694507837295532, |
| "rewards/margins": 8.128179550170898, |
| "rewards/rejected": -9.897629737854004, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.9301333333333335, |
| "grad_norm": 0.9068081378936768, |
| "learning_rate": 1.498259805328678e-06, |
| "logits/chosen": -1.7914531230926514, |
| "logits/rejected": -1.626260757446289, |
| "logps/chosen": -251.34585571289062, |
| "logps/rejected": -374.37982177734375, |
| "loss": 0.054, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6412708759307861, |
| "rewards/margins": 8.070221900939941, |
| "rewards/rejected": -9.711493492126465, |
| "step": 905 |
| }, |
| { |
| "epoch": 1.9407999999999999, |
| "grad_norm": 4.50501012802124, |
| "learning_rate": 1.471948863881601e-06, |
| "logits/chosen": -1.7430839538574219, |
| "logits/rejected": -1.6489261388778687, |
| "logps/chosen": -265.7547912597656, |
| "logps/rejected": -381.2185363769531, |
| "loss": 0.0842, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7131439447402954, |
| "rewards/margins": 8.174077033996582, |
| "rewards/rejected": -9.887219429016113, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.9514666666666667, |
| "grad_norm": 3.7587532997131348, |
| "learning_rate": 1.4457742616940864e-06, |
| "logits/chosen": -1.6547327041625977, |
| "logits/rejected": -1.6035993099212646, |
| "logps/chosen": -259.0414123535156, |
| "logps/rejected": -401.49151611328125, |
| "loss": 0.1169, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -2.150399684906006, |
| "rewards/margins": 7.753369331359863, |
| "rewards/rejected": -9.903768539428711, |
| "step": 915 |
| }, |
| { |
| "epoch": 1.9621333333333333, |
| "grad_norm": 3.2942192554473877, |
| "learning_rate": 1.4197394700194165e-06, |
| "logits/chosen": -1.7956269979476929, |
| "logits/rejected": -1.6683623790740967, |
| "logps/chosen": -274.89825439453125, |
| "logps/rejected": -404.5597839355469, |
| "loss": 0.0832, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.0448083877563477, |
| "rewards/margins": 8.11993408203125, |
| "rewards/rejected": -10.164742469787598, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.9727999999999999, |
| "grad_norm": 3.37825608253479, |
| "learning_rate": 1.3938479415693244e-06, |
| "logits/chosen": -1.789982795715332, |
| "logits/rejected": -1.6593557596206665, |
| "logps/chosen": -247.4827117919922, |
| "logps/rejected": -355.44805908203125, |
| "loss": 0.0922, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3679771423339844, |
| "rewards/margins": 7.889315605163574, |
| "rewards/rejected": -9.257293701171875, |
| "step": 925 |
| }, |
| { |
| "epoch": 1.9834666666666667, |
| "grad_norm": 1.674501895904541, |
| "learning_rate": 1.368103110056096e-06, |
| "logits/chosen": -1.7774940729141235, |
| "logits/rejected": -1.7311162948608398, |
| "logps/chosen": -244.7096710205078, |
| "logps/rejected": -395.4224548339844, |
| "loss": 0.0729, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9104505777359009, |
| "rewards/margins": 8.02760124206543, |
| "rewards/rejected": -9.9380521774292, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.9941333333333333, |
| "grad_norm": 2.0359997749328613, |
| "learning_rate": 1.3425083897371983e-06, |
| "logits/chosen": -1.6318652629852295, |
| "logits/rejected": -1.5561574697494507, |
| "logps/chosen": -274.3860778808594, |
| "logps/rejected": -406.825439453125, |
| "loss": 0.0966, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.8862063884735107, |
| "rewards/margins": 8.324484825134277, |
| "rewards/rejected": -10.210691452026367, |
| "step": 935 |
| }, |
| { |
| "epoch": 2.0042666666666666, |
| "grad_norm": 1.8420169353485107, |
| "learning_rate": 1.317067174962478e-06, |
| "logits/chosen": -1.7264703512191772, |
| "logits/rejected": -1.605676531791687, |
| "logps/chosen": -273.5916442871094, |
| "logps/rejected": -401.54718017578125, |
| "loss": 0.0943, |
| "rewards/accuracies": 0.9671052694320679, |
| "rewards/chosen": -1.6913379430770874, |
| "rewards/margins": 8.358253479003906, |
| "rewards/rejected": -10.049591064453125, |
| "step": 940 |
| }, |
| { |
| "epoch": 2.0149333333333335, |
| "grad_norm": 1.155405044555664, |
| "learning_rate": 1.291782839724009e-06, |
| "logits/chosen": -1.725711464881897, |
| "logits/rejected": -1.6092923879623413, |
| "logps/chosen": -265.33740234375, |
| "logps/rejected": -369.3727111816406, |
| "loss": 0.0769, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.9963560104370117, |
| "rewards/margins": 7.6727800369262695, |
| "rewards/rejected": -9.669134140014648, |
| "step": 945 |
| }, |
| { |
| "epoch": 2.0256, |
| "grad_norm": 2.466005802154541, |
| "learning_rate": 1.2666587372086352e-06, |
| "logits/chosen": -1.678786039352417, |
| "logits/rejected": -1.595643401145935, |
| "logps/chosen": -237.52456665039062, |
| "logps/rejected": -396.2528991699219, |
| "loss": 0.0767, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.0811383724212646, |
| "rewards/margins": 8.369074821472168, |
| "rewards/rejected": -10.450213432312012, |
| "step": 950 |
| }, |
| { |
| "epoch": 2.0362666666666667, |
| "grad_norm": 3.321977376937866, |
| "learning_rate": 1.241698199353275e-06, |
| "logits/chosen": -1.7931787967681885, |
| "logits/rejected": -1.688396692276001, |
| "logps/chosen": -245.0885467529297, |
| "logps/rejected": -375.5834655761719, |
| "loss": 0.0815, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0435004234313965, |
| "rewards/margins": 7.793776035308838, |
| "rewards/rejected": -9.837276458740234, |
| "step": 955 |
| }, |
| { |
| "epoch": 2.0469333333333335, |
| "grad_norm": 2.211179256439209, |
| "learning_rate": 1.2169045364030404e-06, |
| "logits/chosen": -1.8382015228271484, |
| "logits/rejected": -1.7111326456069946, |
| "logps/chosen": -244.8855743408203, |
| "logps/rejected": -396.12042236328125, |
| "loss": 0.0589, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.9382930994033813, |
| "rewards/margins": 7.813839912414551, |
| "rewards/rejected": -9.7521333694458, |
| "step": 960 |
| }, |
| { |
| "epoch": 2.0576, |
| "grad_norm": 0.9813823699951172, |
| "learning_rate": 1.1922810364722402e-06, |
| "logits/chosen": -1.760215163230896, |
| "logits/rejected": -1.666748046875, |
| "logps/chosen": -263.532958984375, |
| "logps/rejected": -400.35064697265625, |
| "loss": 0.0579, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.9762378931045532, |
| "rewards/margins": 8.321720123291016, |
| "rewards/rejected": -10.297958374023438, |
| "step": 965 |
| }, |
| { |
| "epoch": 2.0682666666666667, |
| "grad_norm": 3.9575207233428955, |
| "learning_rate": 1.1678309651083055e-06, |
| "logits/chosen": -1.7971630096435547, |
| "logits/rejected": -1.7192646265029907, |
| "logps/chosen": -249.60342407226562, |
| "logps/rejected": -400.5608825683594, |
| "loss": 0.0614, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.3201982975006104, |
| "rewards/margins": 8.795618057250977, |
| "rewards/rejected": -11.115816116333008, |
| "step": 970 |
| }, |
| { |
| "epoch": 2.0789333333333335, |
| "grad_norm": 1.4822748899459839, |
| "learning_rate": 1.1435575648587224e-06, |
| "logits/chosen": -1.725407361984253, |
| "logits/rejected": -1.598661184310913, |
| "logps/chosen": -253.1859893798828, |
| "logps/rejected": -377.39361572265625, |
| "loss": 0.0597, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.158353328704834, |
| "rewards/margins": 8.470383644104004, |
| "rewards/rejected": -10.62873649597168, |
| "step": 975 |
| }, |
| { |
| "epoch": 2.0896, |
| "grad_norm": 1.8873692750930786, |
| "learning_rate": 1.1194640548410039e-06, |
| "logits/chosen": -1.668302297592163, |
| "logits/rejected": -1.5627166032791138, |
| "logps/chosen": -266.97283935546875, |
| "logps/rejected": -375.2995910644531, |
| "loss": 0.0735, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.951266884803772, |
| "rewards/margins": 8.111071586608887, |
| "rewards/rejected": -10.062339782714844, |
| "step": 980 |
| }, |
| { |
| "epoch": 2.1002666666666667, |
| "grad_norm": 1.918198823928833, |
| "learning_rate": 1.095553630315774e-06, |
| "logits/chosen": -1.6616764068603516, |
| "logits/rejected": -1.6129405498504639, |
| "logps/chosen": -262.82342529296875, |
| "logps/rejected": -405.90679931640625, |
| "loss": 0.0624, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.9863789081573486, |
| "rewards/margins": 8.869156837463379, |
| "rewards/rejected": -10.855535507202148, |
| "step": 985 |
| }, |
| { |
| "epoch": 2.1109333333333336, |
| "grad_norm": 1.5651233196258545, |
| "learning_rate": 1.0718294622630188e-06, |
| "logits/chosen": -1.673699140548706, |
| "logits/rejected": -1.6072123050689697, |
| "logps/chosen": -255.4591064453125, |
| "logps/rejected": -394.9210510253906, |
| "loss": 0.065, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9069595336914062, |
| "rewards/margins": 8.828472137451172, |
| "rewards/rejected": -10.735431671142578, |
| "step": 990 |
| }, |
| { |
| "epoch": 2.1216, |
| "grad_norm": 1.483293056488037, |
| "learning_rate": 1.0482946969615495e-06, |
| "logits/chosen": -1.774780511856079, |
| "logits/rejected": -1.6786816120147705, |
| "logps/chosen": -244.82205200195312, |
| "logps/rejected": -408.83221435546875, |
| "loss": 0.065, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.166687488555908, |
| "rewards/margins": 9.450934410095215, |
| "rewards/rejected": -11.617620468139648, |
| "step": 995 |
| }, |
| { |
| "epoch": 2.1322666666666668, |
| "grad_norm": 1.5644961595535278, |
| "learning_rate": 1.0249524555717461e-06, |
| "logits/chosen": -1.7748655080795288, |
| "logits/rejected": -1.5471773147583008, |
| "logps/chosen": -278.0244140625, |
| "logps/rejected": -402.07269287109375, |
| "loss": 0.104, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.0994677543640137, |
| "rewards/margins": 7.748255729675293, |
| "rewards/rejected": -9.847723007202148, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.142933333333333, |
| "grad_norm": 4.346913814544678, |
| "learning_rate": 1.0018058337216327e-06, |
| "logits/chosen": -1.7031605243682861, |
| "logits/rejected": -1.6505285501480103, |
| "logps/chosen": -259.60772705078125, |
| "logps/rejected": -405.2123107910156, |
| "loss": 0.1133, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.2629244327545166, |
| "rewards/margins": 8.715912818908691, |
| "rewards/rejected": -10.978836059570312, |
| "step": 1005 |
| }, |
| { |
| "epoch": 2.1536, |
| "grad_norm": 0.6849018335342407, |
| "learning_rate": 9.788579010963387e-07, |
| "logits/chosen": -1.6680097579956055, |
| "logits/rejected": -1.5711143016815186, |
| "logps/chosen": -273.92779541015625, |
| "logps/rejected": -412.5943908691406, |
| "loss": 0.0395, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.9487603902816772, |
| "rewards/margins": 9.590485572814941, |
| "rewards/rejected": -11.53924560546875, |
| "step": 1010 |
| }, |
| { |
| "epoch": 2.164266666666667, |
| "grad_norm": 3.6428449153900146, |
| "learning_rate": 9.56111701030998e-07, |
| "logits/chosen": -1.7994695901870728, |
| "logits/rejected": -1.6355020999908447, |
| "logps/chosen": -275.2525329589844, |
| "logps/rejected": -391.162841796875, |
| "loss": 0.0863, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.1008682250976562, |
| "rewards/margins": 8.165130615234375, |
| "rewards/rejected": -10.265998840332031, |
| "step": 1015 |
| }, |
| { |
| "epoch": 2.174933333333333, |
| "grad_norm": 0.5808694958686829, |
| "learning_rate": 9.335702501071459e-07, |
| "logits/chosen": -1.7420467138290405, |
| "logits/rejected": -1.5945088863372803, |
| "logps/chosen": -268.7550964355469, |
| "logps/rejected": -399.09405517578125, |
| "loss": 0.0508, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.7931501865386963, |
| "rewards/margins": 8.578863143920898, |
| "rewards/rejected": -10.3720121383667, |
| "step": 1020 |
| }, |
| { |
| "epoch": 2.1856, |
| "grad_norm": 4.649181842803955, |
| "learning_rate": 9.112365377526636e-07, |
| "logits/chosen": -1.733960747718811, |
| "logits/rejected": -1.6852184534072876, |
| "logps/chosen": -247.6918487548828, |
| "logps/rejected": -402.8586730957031, |
| "loss": 0.0551, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.897204041481018, |
| "rewards/margins": 8.999551773071289, |
| "rewards/rejected": -10.896756172180176, |
| "step": 1025 |
| }, |
| { |
| "epoch": 2.196266666666667, |
| "grad_norm": 1.2479854822158813, |
| "learning_rate": 8.891135258453195e-07, |
| "logits/chosen": -1.7280356884002686, |
| "logits/rejected": -1.6294937133789062, |
| "logps/chosen": -255.19839477539062, |
| "logps/rejected": -381.9560546875, |
| "loss": 0.0714, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0049684047698975, |
| "rewards/margins": 8.376141548156738, |
| "rewards/rejected": -10.381109237670898, |
| "step": 1030 |
| }, |
| { |
| "epoch": 2.206933333333333, |
| "grad_norm": 1.556071400642395, |
| "learning_rate": 8.672041483199709e-07, |
| "logits/chosen": -1.7548717260360718, |
| "logits/rejected": -1.6648685932159424, |
| "logps/chosen": -274.25128173828125, |
| "logps/rejected": -394.82037353515625, |
| "loss": 0.0696, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.2940263748168945, |
| "rewards/margins": 8.477774620056152, |
| "rewards/rejected": -10.771800994873047, |
| "step": 1035 |
| }, |
| { |
| "epoch": 2.2176, |
| "grad_norm": 2.772015333175659, |
| "learning_rate": 8.455113107794652e-07, |
| "logits/chosen": -1.6806873083114624, |
| "logits/rejected": -1.6288923025131226, |
| "logps/chosen": -262.681640625, |
| "logps/rejected": -416.0440979003906, |
| "loss": 0.0813, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.2407727241516113, |
| "rewards/margins": 9.37430191040039, |
| "rewards/rejected": -11.61507511138916, |
| "step": 1040 |
| }, |
| { |
| "epoch": 2.228266666666667, |
| "grad_norm": 3.664442777633667, |
| "learning_rate": 8.240378901093035e-07, |
| "logits/chosen": -1.7520214319229126, |
| "logits/rejected": -1.662095308303833, |
| "logps/chosen": -269.2132568359375, |
| "logps/rejected": -416.43389892578125, |
| "loss": 0.1195, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.21559476852417, |
| "rewards/margins": 8.348793029785156, |
| "rewards/rejected": -10.564387321472168, |
| "step": 1045 |
| }, |
| { |
| "epoch": 2.238933333333333, |
| "grad_norm": 2.8179938793182373, |
| "learning_rate": 8.027867340961113e-07, |
| "logits/chosen": -1.6288158893585205, |
| "logits/rejected": -1.6131118535995483, |
| "logps/chosen": -272.9729919433594, |
| "logps/rejected": -413.86212158203125, |
| "loss": 0.0648, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.159022569656372, |
| "rewards/margins": 8.596202850341797, |
| "rewards/rejected": -10.755226135253906, |
| "step": 1050 |
| }, |
| { |
| "epoch": 2.2496, |
| "grad_norm": 1.2309969663619995, |
| "learning_rate": 7.817606610499639e-07, |
| "logits/chosen": -1.7523510456085205, |
| "logits/rejected": -1.6217739582061768, |
| "logps/chosen": -251.67251586914062, |
| "logps/rejected": -424.4778747558594, |
| "loss": 0.0427, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.8224045038223267, |
| "rewards/margins": 9.106525421142578, |
| "rewards/rejected": -10.928930282592773, |
| "step": 1055 |
| }, |
| { |
| "epoch": 2.2602666666666664, |
| "grad_norm": 2.9990038871765137, |
| "learning_rate": 7.609624594306278e-07, |
| "logits/chosen": -1.7163537740707397, |
| "logits/rejected": -1.7219412326812744, |
| "logps/chosen": -276.01055908203125, |
| "logps/rejected": -401.6703186035156, |
| "loss": 0.094, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.0416719913482666, |
| "rewards/margins": 8.327810287475586, |
| "rewards/rejected": -10.369482040405273, |
| "step": 1060 |
| }, |
| { |
| "epoch": 2.2709333333333332, |
| "grad_norm": 1.0522708892822266, |
| "learning_rate": 7.403948874777556e-07, |
| "logits/chosen": -1.6532328128814697, |
| "logits/rejected": -1.6515743732452393, |
| "logps/chosen": -241.5132293701172, |
| "logps/rejected": -397.3147888183594, |
| "loss": 0.0655, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9390627145767212, |
| "rewards/margins": 9.06468391418457, |
| "rewards/rejected": -11.00374698638916, |
| "step": 1065 |
| }, |
| { |
| "epoch": 2.2816, |
| "grad_norm": 2.331453800201416, |
| "learning_rate": 7.20060672845091e-07, |
| "logits/chosen": -1.7282425165176392, |
| "logits/rejected": -1.6557228565216064, |
| "logps/chosen": -250.88046264648438, |
| "logps/rejected": -403.5778503417969, |
| "loss": 0.054, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.550058603286743, |
| "rewards/margins": 8.61311149597168, |
| "rewards/rejected": -11.16317081451416, |
| "step": 1070 |
| }, |
| { |
| "epoch": 2.2922666666666665, |
| "grad_norm": 4.555802822113037, |
| "learning_rate": 6.99962512238733e-07, |
| "logits/chosen": -1.697750449180603, |
| "logits/rejected": -1.5816296339035034, |
| "logps/chosen": -270.8791198730469, |
| "logps/rejected": -407.1304016113281, |
| "loss": 0.0667, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.9709125757217407, |
| "rewards/margins": 8.97321891784668, |
| "rewards/rejected": -10.944131851196289, |
| "step": 1075 |
| }, |
| { |
| "epoch": 2.3029333333333333, |
| "grad_norm": 0.7534166574478149, |
| "learning_rate": 6.801030710594949e-07, |
| "logits/chosen": -1.6750532388687134, |
| "logits/rejected": -1.5743725299835205, |
| "logps/chosen": -272.9534606933594, |
| "logps/rejected": -417.29534912109375, |
| "loss": 0.0591, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.146669864654541, |
| "rewards/margins": 8.45936393737793, |
| "rewards/rejected": -10.606033325195312, |
| "step": 1080 |
| }, |
| { |
| "epoch": 2.3136, |
| "grad_norm": 3.8812808990478516, |
| "learning_rate": 6.604849830494264e-07, |
| "logits/chosen": -1.7331759929656982, |
| "logits/rejected": -1.6734988689422607, |
| "logps/chosen": -273.17071533203125, |
| "logps/rejected": -394.3504333496094, |
| "loss": 0.0694, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.4721755981445312, |
| "rewards/margins": 8.414241790771484, |
| "rewards/rejected": -10.8864164352417, |
| "step": 1085 |
| }, |
| { |
| "epoch": 2.3242666666666665, |
| "grad_norm": 2.18582820892334, |
| "learning_rate": 6.411108499425253e-07, |
| "logits/chosen": -1.8008339405059814, |
| "logits/rejected": -1.637495756149292, |
| "logps/chosen": -275.2874755859375, |
| "logps/rejected": -391.81597900390625, |
| "loss": 0.0681, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.0567822456359863, |
| "rewards/margins": 8.40910816192627, |
| "rewards/rejected": -10.465889930725098, |
| "step": 1090 |
| }, |
| { |
| "epoch": 2.3349333333333333, |
| "grad_norm": 1.457015037536621, |
| "learning_rate": 6.219832411197013e-07, |
| "logits/chosen": -1.6465427875518799, |
| "logits/rejected": -1.494945764541626, |
| "logps/chosen": -260.6093444824219, |
| "logps/rejected": -381.10870361328125, |
| "loss": 0.0754, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.6909992694854736, |
| "rewards/margins": 8.483034133911133, |
| "rewards/rejected": -10.174033164978027, |
| "step": 1095 |
| }, |
| { |
| "epoch": 2.3456, |
| "grad_norm": 3.5426363945007324, |
| "learning_rate": 6.031046932680229e-07, |
| "logits/chosen": -1.737248420715332, |
| "logits/rejected": -1.5785925388336182, |
| "logps/chosen": -246.08706665039062, |
| "logps/rejected": -378.8399963378906, |
| "loss": 0.0726, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.9671459197998047, |
| "rewards/margins": 8.330504417419434, |
| "rewards/rejected": -10.297648429870605, |
| "step": 1100 |
| }, |
| { |
| "epoch": 2.3562666666666665, |
| "grad_norm": 1.9274934530258179, |
| "learning_rate": 5.844777100443091e-07, |
| "logits/chosen": -1.668204665184021, |
| "logits/rejected": -1.669782280921936, |
| "logps/chosen": -252.9781036376953, |
| "logps/rejected": -419.4017028808594, |
| "loss": 0.0667, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.1728081703186035, |
| "rewards/margins": 9.157058715820312, |
| "rewards/rejected": -11.329866409301758, |
| "step": 1105 |
| }, |
| { |
| "epoch": 2.3669333333333333, |
| "grad_norm": 4.665882110595703, |
| "learning_rate": 5.66104761743092e-07, |
| "logits/chosen": -1.8508713245391846, |
| "logits/rejected": -1.714739203453064, |
| "logps/chosen": -245.087646484375, |
| "logps/rejected": -413.19305419921875, |
| "loss": 0.0469, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.481471538543701, |
| "rewards/margins": 9.403627395629883, |
| "rewards/rejected": -11.885098457336426, |
| "step": 1110 |
| }, |
| { |
| "epoch": 2.3776, |
| "grad_norm": 2.681351900100708, |
| "learning_rate": 5.479882849690102e-07, |
| "logits/chosen": -1.7130721807479858, |
| "logits/rejected": -1.639499306678772, |
| "logps/chosen": -258.9979553222656, |
| "logps/rejected": -405.5329284667969, |
| "loss": 0.0864, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.2818868160247803, |
| "rewards/margins": 8.402789115905762, |
| "rewards/rejected": -10.684676170349121, |
| "step": 1115 |
| }, |
| { |
| "epoch": 2.3882666666666665, |
| "grad_norm": 3.2716691493988037, |
| "learning_rate": 5.301306823136687e-07, |
| "logits/chosen": -1.7014468908309937, |
| "logits/rejected": -1.5933825969696045, |
| "logps/chosen": -249.73483276367188, |
| "logps/rejected": -382.95709228515625, |
| "loss": 0.0715, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.2783286571502686, |
| "rewards/margins": 7.969732761383057, |
| "rewards/rejected": -10.248061180114746, |
| "step": 1120 |
| }, |
| { |
| "epoch": 2.3989333333333334, |
| "grad_norm": 0.9533172249794006, |
| "learning_rate": 5.125343220370088e-07, |
| "logits/chosen": -1.689549446105957, |
| "logits/rejected": -1.6298614740371704, |
| "logps/chosen": -261.215087890625, |
| "logps/rejected": -415.8038635253906, |
| "loss": 0.0652, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.2256734371185303, |
| "rewards/margins": 8.733424186706543, |
| "rewards/rejected": -10.959096908569336, |
| "step": 1125 |
| }, |
| { |
| "epoch": 2.4096, |
| "grad_norm": 0.5545014142990112, |
| "learning_rate": 4.952015377532329e-07, |
| "logits/chosen": -1.784350037574768, |
| "logits/rejected": -1.6608524322509766, |
| "logps/chosen": -235.21969604492188, |
| "logps/rejected": -393.8280029296875, |
| "loss": 0.0702, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7608108520507812, |
| "rewards/margins": 8.938084602355957, |
| "rewards/rejected": -10.698893547058105, |
| "step": 1130 |
| }, |
| { |
| "epoch": 2.4202666666666666, |
| "grad_norm": 1.1242990493774414, |
| "learning_rate": 4.781346281213204e-07, |
| "logits/chosen": -1.7812182903289795, |
| "logits/rejected": -1.6980056762695312, |
| "logps/chosen": -263.33758544921875, |
| "logps/rejected": -392.80078125, |
| "loss": 0.0531, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3382551670074463, |
| "rewards/margins": 8.795969009399414, |
| "rewards/rejected": -11.134222984313965, |
| "step": 1135 |
| }, |
| { |
| "epoch": 2.4309333333333334, |
| "grad_norm": 1.811649203300476, |
| "learning_rate": 4.613358565401818e-07, |
| "logits/chosen": -1.714125394821167, |
| "logits/rejected": -1.5603991746902466, |
| "logps/chosen": -266.761962890625, |
| "logps/rejected": -408.21099853515625, |
| "loss": 0.0643, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.1256799697875977, |
| "rewards/margins": 9.198551177978516, |
| "rewards/rejected": -11.32423210144043, |
| "step": 1140 |
| }, |
| { |
| "epoch": 2.4416, |
| "grad_norm": 2.120972156524658, |
| "learning_rate": 4.448074508484906e-07, |
| "logits/chosen": -1.698669195175171, |
| "logits/rejected": -1.5931520462036133, |
| "logps/chosen": -266.32000732421875, |
| "logps/rejected": -399.5585021972656, |
| "loss": 0.0568, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.22153377532959, |
| "rewards/margins": 8.874009132385254, |
| "rewards/rejected": -11.09554386138916, |
| "step": 1145 |
| }, |
| { |
| "epoch": 2.4522666666666666, |
| "grad_norm": 5.987904071807861, |
| "learning_rate": 4.2855160302922777e-07, |
| "logits/chosen": -1.6893975734710693, |
| "logits/rejected": -1.5851631164550781, |
| "logps/chosen": -258.2315979003906, |
| "logps/rejected": -399.9725646972656, |
| "loss": 0.0611, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5237483978271484, |
| "rewards/margins": 8.855246543884277, |
| "rewards/rejected": -11.378995895385742, |
| "step": 1150 |
| }, |
| { |
| "epoch": 2.4629333333333334, |
| "grad_norm": 1.4711482524871826, |
| "learning_rate": 4.125704689189819e-07, |
| "logits/chosen": -1.6372854709625244, |
| "logits/rejected": -1.5050592422485352, |
| "logps/chosen": -248.77352905273438, |
| "logps/rejected": -393.07489013671875, |
| "loss": 0.0763, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.038905620574951, |
| "rewards/margins": 8.548587799072266, |
| "rewards/rejected": -10.587493896484375, |
| "step": 1155 |
| }, |
| { |
| "epoch": 2.4736000000000002, |
| "grad_norm": 3.182465076446533, |
| "learning_rate": 3.9686616792204677e-07, |
| "logits/chosen": -1.7079054117202759, |
| "logits/rejected": -1.668967604637146, |
| "logps/chosen": -274.384765625, |
| "logps/rejected": -426.66436767578125, |
| "loss": 0.0522, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5939784049987793, |
| "rewards/margins": 9.367667198181152, |
| "rewards/rejected": -11.961645126342773, |
| "step": 1160 |
| }, |
| { |
| "epoch": 2.4842666666666666, |
| "grad_norm": 1.7841264009475708, |
| "learning_rate": 3.8144078272934676e-07, |
| "logits/chosen": -1.678052306175232, |
| "logits/rejected": -1.5851737260818481, |
| "logps/chosen": -289.3586120605469, |
| "logps/rejected": -421.1358947753906, |
| "loss": 0.0504, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.6983115673065186, |
| "rewards/margins": 8.838812828063965, |
| "rewards/rejected": -11.537124633789062, |
| "step": 1165 |
| }, |
| { |
| "epoch": 2.4949333333333334, |
| "grad_norm": 4.349465370178223, |
| "learning_rate": 3.6629635904222943e-07, |
| "logits/chosen": -1.6927592754364014, |
| "logits/rejected": -1.668840765953064, |
| "logps/chosen": -268.197998046875, |
| "logps/rejected": -403.91265869140625, |
| "loss": 0.0527, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0596611499786377, |
| "rewards/margins": 8.410848617553711, |
| "rewards/rejected": -10.470510482788086, |
| "step": 1170 |
| }, |
| { |
| "epoch": 2.5056000000000003, |
| "grad_norm": 2.6785404682159424, |
| "learning_rate": 3.514349053011698e-07, |
| "logits/chosen": -1.6893892288208008, |
| "logits/rejected": -1.5429222583770752, |
| "logps/chosen": -263.0149230957031, |
| "logps/rejected": -389.00592041015625, |
| "loss": 0.0765, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.377685546875, |
| "rewards/margins": 8.378408432006836, |
| "rewards/rejected": -10.75609302520752, |
| "step": 1175 |
| }, |
| { |
| "epoch": 2.5162666666666667, |
| "grad_norm": 1.3735431432724, |
| "learning_rate": 3.3685839241941133e-07, |
| "logits/chosen": -1.6811046600341797, |
| "logits/rejected": -1.615979552268982, |
| "logps/chosen": -282.908203125, |
| "logps/rejected": -395.00885009765625, |
| "loss": 0.1003, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.4607012271881104, |
| "rewards/margins": 8.770505905151367, |
| "rewards/rejected": -11.231206893920898, |
| "step": 1180 |
| }, |
| { |
| "epoch": 2.5269333333333335, |
| "grad_norm": 4.778181076049805, |
| "learning_rate": 3.2256875352158435e-07, |
| "logits/chosen": -1.7965024709701538, |
| "logits/rejected": -1.619570016860962, |
| "logps/chosen": -269.8258056640625, |
| "logps/rejected": -424.8388671875, |
| "loss": 0.0941, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.287374973297119, |
| "rewards/margins": 9.26796817779541, |
| "rewards/rejected": -11.555343627929688, |
| "step": 1185 |
| }, |
| { |
| "epoch": 2.5376, |
| "grad_norm": 1.0934377908706665, |
| "learning_rate": 3.085678836873399e-07, |
| "logits/chosen": -1.698390007019043, |
| "logits/rejected": -1.6988176107406616, |
| "logps/chosen": -260.91217041015625, |
| "logps/rejected": -410.05828857421875, |
| "loss": 0.0825, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.011669397354126, |
| "rewards/margins": 8.771254539489746, |
| "rewards/rejected": -10.782922744750977, |
| "step": 1190 |
| }, |
| { |
| "epoch": 2.5482666666666667, |
| "grad_norm": 3.100398540496826, |
| "learning_rate": 2.9485763970002106e-07, |
| "logits/chosen": -1.7358124256134033, |
| "logits/rejected": -1.630570650100708, |
| "logps/chosen": -275.30572509765625, |
| "logps/rejected": -406.6322937011719, |
| "loss": 0.0762, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.0499629974365234, |
| "rewards/margins": 8.547712326049805, |
| "rewards/rejected": -10.597673416137695, |
| "step": 1195 |
| }, |
| { |
| "epoch": 2.558933333333333, |
| "grad_norm": 2.3817620277404785, |
| "learning_rate": 2.8143983980042174e-07, |
| "logits/chosen": -1.7879635095596313, |
| "logits/rejected": -1.6414146423339844, |
| "logps/chosen": -265.8365783691406, |
| "logps/rejected": -415.36688232421875, |
| "loss": 0.0499, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5079102516174316, |
| "rewards/margins": 9.601276397705078, |
| "rewards/rejected": -12.109187126159668, |
| "step": 1200 |
| }, |
| { |
| "epoch": 2.5696, |
| "grad_norm": 1.1379503011703491, |
| "learning_rate": 2.683162634456515e-07, |
| "logits/chosen": -1.6539013385772705, |
| "logits/rejected": -1.542270302772522, |
| "logps/chosen": -282.34429931640625, |
| "logps/rejected": -397.26898193359375, |
| "loss": 0.041, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.0202269554138184, |
| "rewards/margins": 8.675008773803711, |
| "rewards/rejected": -10.695237159729004, |
| "step": 1205 |
| }, |
| { |
| "epoch": 2.5802666666666667, |
| "grad_norm": 1.2463977336883545, |
| "learning_rate": 2.5548865107314606e-07, |
| "logits/chosen": -1.5867834091186523, |
| "logits/rejected": -1.495589256286621, |
| "logps/chosen": -287.71478271484375, |
| "logps/rejected": -385.6083068847656, |
| "loss": 0.0634, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.2334399223327637, |
| "rewards/margins": 8.365260124206543, |
| "rewards/rejected": -10.598699569702148, |
| "step": 1210 |
| }, |
| { |
| "epoch": 2.590933333333333, |
| "grad_norm": 3.3940725326538086, |
| "learning_rate": 2.429587038698511e-07, |
| "logits/chosen": -1.7194877862930298, |
| "logits/rejected": -1.652475118637085, |
| "logps/chosen": -231.0302276611328, |
| "logps/rejected": -386.63714599609375, |
| "loss": 0.0668, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5545358657836914, |
| "rewards/margins": 8.87263298034668, |
| "rewards/rejected": -11.427169799804688, |
| "step": 1215 |
| }, |
| { |
| "epoch": 2.6016, |
| "grad_norm": 1.4201222658157349, |
| "learning_rate": 2.307280835466144e-07, |
| "logits/chosen": -1.7387536764144897, |
| "logits/rejected": -1.645634412765503, |
| "logps/chosen": -272.35699462890625, |
| "logps/rejected": -395.57843017578125, |
| "loss": 0.0701, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.429680347442627, |
| "rewards/margins": 7.889183044433594, |
| "rewards/rejected": -10.318862915039062, |
| "step": 1220 |
| }, |
| { |
| "epoch": 2.6122666666666667, |
| "grad_norm": 4.102688312530518, |
| "learning_rate": 2.1879841211780856e-07, |
| "logits/chosen": -1.6071255207061768, |
| "logits/rejected": -1.6051759719848633, |
| "logps/chosen": -253.9296112060547, |
| "logps/rejected": -408.7915954589844, |
| "loss": 0.0719, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.2510740756988525, |
| "rewards/margins": 9.297788619995117, |
| "rewards/rejected": -11.548863410949707, |
| "step": 1225 |
| }, |
| { |
| "epoch": 2.622933333333333, |
| "grad_norm": 1.2044811248779297, |
| "learning_rate": 2.0717127168622202e-07, |
| "logits/chosen": -1.72200608253479, |
| "logits/rejected": -1.6253166198730469, |
| "logps/chosen": -265.43292236328125, |
| "logps/rejected": -395.1199645996094, |
| "loss": 0.0455, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.528899669647217, |
| "rewards/margins": 8.821831703186035, |
| "rewards/rejected": -11.350730895996094, |
| "step": 1230 |
| }, |
| { |
| "epoch": 2.6336, |
| "grad_norm": 2.7064669132232666, |
| "learning_rate": 1.958482042332424e-07, |
| "logits/chosen": -1.7487704753875732, |
| "logits/rejected": -1.5633618831634521, |
| "logps/chosen": -256.0242919921875, |
| "logps/rejected": -387.086181640625, |
| "loss": 0.1128, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.0679194927215576, |
| "rewards/margins": 9.494586944580078, |
| "rewards/rejected": -11.562507629394531, |
| "step": 1235 |
| }, |
| { |
| "epoch": 2.6442666666666668, |
| "grad_norm": 2.0025794506073, |
| "learning_rate": 1.8483071141435937e-07, |
| "logits/chosen": -1.5965923070907593, |
| "logits/rejected": -1.5514066219329834, |
| "logps/chosen": -288.4261474609375, |
| "logps/rejected": -398.62908935546875, |
| "loss": 0.0768, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.1800413131713867, |
| "rewards/margins": 8.478593826293945, |
| "rewards/rejected": -10.658636093139648, |
| "step": 1240 |
| }, |
| { |
| "epoch": 2.654933333333333, |
| "grad_norm": 0.43382975459098816, |
| "learning_rate": 1.741202543600176e-07, |
| "logits/chosen": -1.7435810565948486, |
| "logits/rejected": -1.6685746908187866, |
| "logps/chosen": -252.3104248046875, |
| "logps/rejected": -387.71044921875, |
| "loss": 0.0762, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.4105584621429443, |
| "rewards/margins": 8.391923904418945, |
| "rewards/rejected": -10.802482604980469, |
| "step": 1245 |
| }, |
| { |
| "epoch": 2.6656, |
| "grad_norm": 3.122593641281128, |
| "learning_rate": 1.6371825348184155e-07, |
| "logits/chosen": -1.7292636632919312, |
| "logits/rejected": -1.6958808898925781, |
| "logps/chosen": -266.6325378417969, |
| "logps/rejected": -389.1386413574219, |
| "loss": 0.1075, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.3547916412353516, |
| "rewards/margins": 8.089277267456055, |
| "rewards/rejected": -10.444067001342773, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.676266666666667, |
| "grad_norm": 2.821847915649414, |
| "learning_rate": 1.5362608828426246e-07, |
| "logits/chosen": -1.7317949533462524, |
| "logits/rejected": -1.6616737842559814, |
| "logps/chosen": -273.41741943359375, |
| "logps/rejected": -399.07659912109375, |
| "loss": 0.0669, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.323963165283203, |
| "rewards/margins": 8.672471046447754, |
| "rewards/rejected": -10.996434211730957, |
| "step": 1255 |
| }, |
| { |
| "epoch": 2.686933333333333, |
| "grad_norm": 5.389606952667236, |
| "learning_rate": 1.438450971815686e-07, |
| "logits/chosen": -1.7826637029647827, |
| "logits/rejected": -1.6701968908309937, |
| "logps/chosen": -266.99139404296875, |
| "logps/rejected": -391.0360412597656, |
| "loss": 0.0699, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.6968390941619873, |
| "rewards/margins": 8.890096664428711, |
| "rewards/rejected": -11.586936950683594, |
| "step": 1260 |
| }, |
| { |
| "epoch": 2.6976, |
| "grad_norm": 2.9304444789886475, |
| "learning_rate": 1.3437657732040783e-07, |
| "logits/chosen": -1.6977288722991943, |
| "logits/rejected": -1.575650691986084, |
| "logps/chosen": -265.2109680175781, |
| "logps/rejected": -393.3660888671875, |
| "loss": 0.0733, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.047703266143799, |
| "rewards/margins": 8.961397171020508, |
| "rewards/rejected": -11.009099960327148, |
| "step": 1265 |
| }, |
| { |
| "epoch": 2.708266666666667, |
| "grad_norm": 3.8689467906951904, |
| "learning_rate": 1.2522178440776022e-07, |
| "logits/chosen": -1.7184581756591797, |
| "logits/rejected": -1.7143863439559937, |
| "logps/chosen": -271.6422119140625, |
| "logps/rejected": -383.397705078125, |
| "loss": 0.0879, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.4679946899414062, |
| "rewards/margins": 8.029095649719238, |
| "rewards/rejected": -10.497090339660645, |
| "step": 1270 |
| }, |
| { |
| "epoch": 2.718933333333333, |
| "grad_norm": 0.6974985003471375, |
| "learning_rate": 1.1638193254440655e-07, |
| "logits/chosen": -1.7665700912475586, |
| "logits/rejected": -1.689286231994629, |
| "logps/chosen": -255.50302124023438, |
| "logps/rejected": -408.79461669921875, |
| "loss": 0.066, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.648810863494873, |
| "rewards/margins": 9.510984420776367, |
| "rewards/rejected": -12.159795761108398, |
| "step": 1275 |
| }, |
| { |
| "epoch": 2.7296, |
| "grad_norm": 1.31102454662323, |
| "learning_rate": 1.0785819406391562e-07, |
| "logits/chosen": -1.653638482093811, |
| "logits/rejected": -1.6233888864517212, |
| "logps/chosen": -266.7182922363281, |
| "logps/rejected": -418.28192138671875, |
| "loss": 0.056, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.3988518714904785, |
| "rewards/margins": 8.98476791381836, |
| "rewards/rejected": -11.383620262145996, |
| "step": 1280 |
| }, |
| { |
| "epoch": 2.740266666666667, |
| "grad_norm": 1.9283661842346191, |
| "learning_rate": 9.965169937717095e-08, |
| "logits/chosen": -1.6370359659194946, |
| "logits/rejected": -1.631044626235962, |
| "logps/chosen": -252.5908660888672, |
| "logps/rejected": -401.8351745605469, |
| "loss": 0.0834, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.1292572021484375, |
| "rewards/margins": 8.970866203308105, |
| "rewards/rejected": -11.10012435913086, |
| "step": 1285 |
| }, |
| { |
| "epoch": 2.7509333333333332, |
| "grad_norm": 2.89962100982666, |
| "learning_rate": 9.176353682245676e-08, |
| "logits/chosen": -1.75473153591156, |
| "logits/rejected": -1.631553292274475, |
| "logps/chosen": -248.2369842529297, |
| "logps/rejected": -367.32745361328125, |
| "loss": 0.0781, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.326287269592285, |
| "rewards/margins": 8.09610366821289, |
| "rewards/rejected": -10.422391891479492, |
| "step": 1290 |
| }, |
| { |
| "epoch": 2.7616, |
| "grad_norm": 3.716407299041748, |
| "learning_rate": 8.41947525211223e-08, |
| "logits/chosen": -1.6061655282974243, |
| "logits/rejected": -1.5312092304229736, |
| "logps/chosen": -253.99716186523438, |
| "logps/rejected": -383.96124267578125, |
| "loss": 0.1108, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.050841808319092, |
| "rewards/margins": 8.109245300292969, |
| "rewards/rejected": -10.160086631774902, |
| "step": 1295 |
| }, |
| { |
| "epoch": 2.772266666666667, |
| "grad_norm": 1.4873884916305542, |
| "learning_rate": 7.694635023884789e-08, |
| "logits/chosen": -1.6627674102783203, |
| "logits/rejected": -1.5674951076507568, |
| "logps/chosen": -273.51275634765625, |
| "logps/rejected": -383.80731201171875, |
| "loss": 0.0602, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.287492036819458, |
| "rewards/margins": 8.161794662475586, |
| "rewards/rejected": -10.449285507202148, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.7829333333333333, |
| "grad_norm": 3.9774677753448486, |
| "learning_rate": 7.00192912525241e-08, |
| "logits/chosen": -1.6725132465362549, |
| "logits/rejected": -1.6510508060455322, |
| "logps/chosen": -256.28558349609375, |
| "logps/rejected": -408.4872741699219, |
| "loss": 0.0857, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.293246030807495, |
| "rewards/margins": 9.179945945739746, |
| "rewards/rejected": -11.47319221496582, |
| "step": 1305 |
| }, |
| { |
| "epoch": 2.7936, |
| "grad_norm": 3.7363460063934326, |
| "learning_rate": 6.341449422277047e-08, |
| "logits/chosen": -1.5966999530792236, |
| "logits/rejected": -1.5942323207855225, |
| "logps/chosen": -289.9300842285156, |
| "logps/rejected": -430.7188415527344, |
| "loss": 0.0602, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.317781686782837, |
| "rewards/margins": 8.42338752746582, |
| "rewards/rejected": -10.741169929504395, |
| "step": 1310 |
| }, |
| { |
| "epoch": 2.804266666666667, |
| "grad_norm": 1.0523943901062012, |
| "learning_rate": 5.7132835072101486e-08, |
| "logits/chosen": -1.668763518333435, |
| "logits/rejected": -1.6661746501922607, |
| "logps/chosen": -245.1510009765625, |
| "logps/rejected": -397.2869567871094, |
| "loss": 0.0798, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.2015979290008545, |
| "rewards/margins": 8.682696342468262, |
| "rewards/rejected": -10.884294509887695, |
| "step": 1315 |
| }, |
| { |
| "epoch": 2.8149333333333333, |
| "grad_norm": 1.1371092796325684, |
| "learning_rate": 5.117514686876379e-08, |
| "logits/chosen": -1.6409333944320679, |
| "logits/rejected": -1.5510404109954834, |
| "logps/chosen": -253.13516235351562, |
| "logps/rejected": -394.7981262207031, |
| "loss": 0.0875, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.084290027618408, |
| "rewards/margins": 8.778473854064941, |
| "rewards/rejected": -10.862764358520508, |
| "step": 1320 |
| }, |
| { |
| "epoch": 2.8256, |
| "grad_norm": 4.370893955230713, |
| "learning_rate": 4.5542219716255274e-08, |
| "logits/chosen": -1.6426441669464111, |
| "logits/rejected": -1.5962680578231812, |
| "logps/chosen": -295.06854248046875, |
| "logps/rejected": -409.53399658203125, |
| "loss": 0.0983, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.649624824523926, |
| "rewards/margins": 8.616082191467285, |
| "rewards/rejected": -11.265706062316895, |
| "step": 1325 |
| }, |
| { |
| "epoch": 2.836266666666667, |
| "grad_norm": 0.8870635032653809, |
| "learning_rate": 4.023480064854174e-08, |
| "logits/chosen": -1.773553490638733, |
| "logits/rejected": -1.6232315301895142, |
| "logps/chosen": -280.57781982421875, |
| "logps/rejected": -411.35565185546875, |
| "loss": 0.0473, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.6773681640625, |
| "rewards/margins": 9.148324966430664, |
| "rewards/rejected": -11.82569408416748, |
| "step": 1330 |
| }, |
| { |
| "epoch": 2.8469333333333333, |
| "grad_norm": 2.6349360942840576, |
| "learning_rate": 3.525359353098584e-08, |
| "logits/chosen": -1.6854965686798096, |
| "logits/rejected": -1.6002693176269531, |
| "logps/chosen": -263.1238708496094, |
| "logps/rejected": -407.7985534667969, |
| "loss": 0.063, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.756936550140381, |
| "rewards/margins": 8.892518043518066, |
| "rewards/rejected": -11.649454116821289, |
| "step": 1335 |
| }, |
| { |
| "epoch": 2.8576, |
| "grad_norm": 4.308375835418701, |
| "learning_rate": 3.059925896700094e-08, |
| "logits/chosen": -1.606723427772522, |
| "logits/rejected": -1.4940506219863892, |
| "logps/chosen": -289.65118408203125, |
| "logps/rejected": -372.673583984375, |
| "loss": 0.1077, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.2103538513183594, |
| "rewards/margins": 7.562900543212891, |
| "rewards/rejected": -9.773255348205566, |
| "step": 1340 |
| }, |
| { |
| "epoch": 2.8682666666666665, |
| "grad_norm": 3.312288761138916, |
| "learning_rate": 2.6272414210443964e-08, |
| "logits/chosen": -1.6424680948257446, |
| "logits/rejected": -1.656428337097168, |
| "logps/chosen": -265.83258056640625, |
| "logps/rejected": -385.98614501953125, |
| "loss": 0.0882, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.3592121601104736, |
| "rewards/margins": 7.7441887855529785, |
| "rewards/rejected": -10.103400230407715, |
| "step": 1345 |
| }, |
| { |
| "epoch": 2.8789333333333333, |
| "grad_norm": 6.28291130065918, |
| "learning_rate": 2.2273633083753632e-08, |
| "logits/chosen": -1.7576591968536377, |
| "logits/rejected": -1.6805713176727295, |
| "logps/chosen": -248.3701171875, |
| "logps/rejected": -401.76904296875, |
| "loss": 0.0964, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.8228938579559326, |
| "rewards/margins": 9.486324310302734, |
| "rewards/rejected": -12.309218406677246, |
| "step": 1350 |
| }, |
| { |
| "epoch": 2.8895999999999997, |
| "grad_norm": 2.0318760871887207, |
| "learning_rate": 1.8603445901851647e-08, |
| "logits/chosen": -1.6315650939941406, |
| "logits/rejected": -1.558574914932251, |
| "logps/chosen": -266.645263671875, |
| "logps/rejected": -393.82257080078125, |
| "loss": 0.0685, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.1889877319335938, |
| "rewards/margins": 8.80628776550293, |
| "rewards/rejected": -10.995275497436523, |
| "step": 1355 |
| }, |
| { |
| "epoch": 2.9002666666666665, |
| "grad_norm": 4.676877021789551, |
| "learning_rate": 1.5262339401813375e-08, |
| "logits/chosen": -1.785639762878418, |
| "logits/rejected": -1.6920429468154907, |
| "logps/chosen": -268.1185302734375, |
| "logps/rejected": -410.142822265625, |
| "loss": 0.1384, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.5416979789733887, |
| "rewards/margins": 8.666342735290527, |
| "rewards/rejected": -11.208040237426758, |
| "step": 1360 |
| }, |
| { |
| "epoch": 2.9109333333333334, |
| "grad_norm": 4.1418538093566895, |
| "learning_rate": 1.2250756678315601e-08, |
| "logits/chosen": -1.7175476551055908, |
| "logits/rejected": -1.6276105642318726, |
| "logps/chosen": -279.15985107421875, |
| "logps/rejected": -424.0628967285156, |
| "loss": 0.0476, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5548946857452393, |
| "rewards/margins": 8.781354904174805, |
| "rewards/rejected": -11.336250305175781, |
| "step": 1365 |
| }, |
| { |
| "epoch": 2.9215999999999998, |
| "grad_norm": 1.394214153289795, |
| "learning_rate": 9.569097124875193e-09, |
| "logits/chosen": -1.7502952814102173, |
| "logits/rejected": -1.6516965627670288, |
| "logps/chosen": -252.2337188720703, |
| "logps/rejected": -379.22149658203125, |
| "loss": 0.1196, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.270294666290283, |
| "rewards/margins": 7.760924339294434, |
| "rewards/rejected": -10.031218528747559, |
| "step": 1370 |
| }, |
| { |
| "epoch": 2.9322666666666666, |
| "grad_norm": 3.925764799118042, |
| "learning_rate": 7.217716380881479e-09, |
| "logits/chosen": -1.6725263595581055, |
| "logits/rejected": -1.6312596797943115, |
| "logps/chosen": -278.2488098144531, |
| "logps/rejected": -412.33428955078125, |
| "loss": 0.0922, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1912899017333984, |
| "rewards/margins": 8.727452278137207, |
| "rewards/rejected": -10.918742179870605, |
| "step": 1375 |
| }, |
| { |
| "epoch": 2.9429333333333334, |
| "grad_norm": 1.50179123878479, |
| "learning_rate": 5.196926284430359e-09, |
| "logits/chosen": -1.6912105083465576, |
| "logits/rejected": -1.5623693466186523, |
| "logps/chosen": -272.2286682128906, |
| "logps/rejected": -398.8866271972656, |
| "loss": 0.0806, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.530665397644043, |
| "rewards/margins": 8.755317687988281, |
| "rewards/rejected": -11.285982131958008, |
| "step": 1380 |
| }, |
| { |
| "epoch": 2.9536, |
| "grad_norm": 2.6899476051330566, |
| "learning_rate": 3.5069948309712775e-09, |
| "logits/chosen": -1.6759449243545532, |
| "logits/rejected": -1.6454025506973267, |
| "logps/chosen": -288.8973083496094, |
| "logps/rejected": -421.2940979003906, |
| "loss": 0.0664, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.2580349445343018, |
| "rewards/margins": 8.855852127075195, |
| "rewards/rejected": -11.113885879516602, |
| "step": 1385 |
| }, |
| { |
| "epoch": 2.9642666666666666, |
| "grad_norm": 1.0512514114379883, |
| "learning_rate": 2.1481461377634294e-09, |
| "logits/chosen": -1.6516395807266235, |
| "logits/rejected": -1.6442527770996094, |
| "logps/chosen": -271.1015625, |
| "logps/rejected": -423.9894104003906, |
| "loss": 0.061, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.8409223556518555, |
| "rewards/margins": 9.194467544555664, |
| "rewards/rejected": -11.035387992858887, |
| "step": 1390 |
| }, |
| { |
| "epoch": 2.9749333333333334, |
| "grad_norm": 1.4273769855499268, |
| "learning_rate": 1.120560414154259e-09, |
| "logits/chosen": -1.6637353897094727, |
| "logits/rejected": -1.5640382766723633, |
| "logps/chosen": -255.4537811279297, |
| "logps/rejected": -424.53680419921875, |
| "loss": 0.0447, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.2763662338256836, |
| "rewards/margins": 9.897512435913086, |
| "rewards/rejected": -12.17387866973877, |
| "step": 1395 |
| }, |
| { |
| "epoch": 2.9856, |
| "grad_norm": 3.6625378131866455, |
| "learning_rate": 4.2437393768079983e-10, |
| "logits/chosen": -1.672315001487732, |
| "logits/rejected": -1.5887823104858398, |
| "logps/chosen": -275.51129150390625, |
| "logps/rejected": -381.4290466308594, |
| "loss": 0.1196, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.458815813064575, |
| "rewards/margins": 8.02712631225586, |
| "rewards/rejected": -10.485941886901855, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.9962666666666666, |
| "grad_norm": 2.525285243988037, |
| "learning_rate": 5.967903599635171e-11, |
| "logits/chosen": -1.67082941532135, |
| "logits/rejected": -1.5557708740234375, |
| "logps/chosen": -261.8987731933594, |
| "logps/rejected": -392.11273193359375, |
| "loss": 0.1019, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.351944923400879, |
| "rewards/margins": 8.210747718811035, |
| "rewards/rejected": -10.562692642211914, |
| "step": 1405 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 1407, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
