| {"epoch": 0.011428571428571429, "grad_norm": 5.672101974487305, "learning_rate": 0.0, "logits/chosen": -0.003827691078186035, "logits/rejected": 1.6622543334960938, "logps/chosen": -53.823280334472656, "logps/rejected": -83.19792175292969, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1} |
| {"epoch": 0.022857142857142857, "grad_norm": 5.676645755767822, "learning_rate": 1.0000000000000002e-06, "logits/chosen": -0.23317879438400269, "logits/rejected": 1.487736701965332, "logps/chosen": -105.3038101196289, "logps/rejected": -72.62786102294922, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 2} |
| {"epoch": 0.03428571428571429, "grad_norm": 5.494569778442383, "learning_rate": 2.0000000000000003e-06, "logits/chosen": -0.34497007727622986, "logits/rejected": 1.3572413921356201, "logps/chosen": -78.18315124511719, "logps/rejected": -92.42572021484375, "loss": 0.6925, "rewards/accuracies": 0.5, "rewards/chosen": -0.010104288347065449, "rewards/margins": 0.001475572120398283, "rewards/rejected": -0.011579860001802444, "step": 3} |
| {"epoch": 0.045714285714285714, "grad_norm": 5.875666618347168, "learning_rate": 3e-06, "logits/chosen": 0.0768582671880722, "logits/rejected": 1.793702244758606, "logps/chosen": -49.45244598388672, "logps/rejected": -70.34400939941406, "loss": 0.6824, "rewards/accuracies": 0.75, "rewards/chosen": -0.006157493684440851, "rewards/margins": 0.02201489359140396, "rewards/rejected": -0.028172386810183525, "step": 4} |
| {"epoch": 0.05714285714285714, "grad_norm": 4.4969987869262695, "learning_rate": 4.000000000000001e-06, "logits/chosen": 0.29756978154182434, "logits/rejected": 1.5599043369293213, "logps/chosen": -38.330902099609375, "logps/rejected": -58.7880859375, "loss": 0.6602, "rewards/accuracies": 1.0, "rewards/chosen": 0.0054547907784581184, "rewards/margins": 0.06765428930521011, "rewards/rejected": -0.06219949945807457, "step": 5} |
| {"epoch": 0.06857142857142857, "grad_norm": 5.193802356719971, "learning_rate": 5e-06, "logits/chosen": -0.097133569419384, "logits/rejected": 1.5903254747390747, "logps/chosen": -71.06304168701172, "logps/rejected": -69.80386352539062, "loss": 0.6118, "rewards/accuracies": 0.9375, "rewards/chosen": 0.02022118680179119, "rewards/margins": 0.1755855232477188, "rewards/rejected": -0.15536434948444366, "step": 6} |
| {"epoch": 0.08, "grad_norm": 4.568131446838379, "learning_rate": 4.998209387040829e-06, "logits/chosen": 0.22878801822662354, "logits/rejected": 1.2406954765319824, "logps/chosen": -65.77464294433594, "logps/rejected": -70.03345489501953, "loss": 0.5809, "rewards/accuracies": 1.0, "rewards/chosen": 0.008165514096617699, "rewards/margins": 0.2519247233867645, "rewards/rejected": -0.2437591850757599, "step": 7} |
| {"epoch": 0.09142857142857143, "grad_norm": 3.668226480484009, "learning_rate": 4.992840113199131e-06, "logits/chosen": -0.06199551001191139, "logits/rejected": 1.0075984001159668, "logps/chosen": -31.54962158203125, "logps/rejected": -48.84445571899414, "loss": 0.5925, "rewards/accuracies": 1.0, "rewards/chosen": 0.006166815757751465, "rewards/margins": 0.21949976682662964, "rewards/rejected": -0.21333293616771698, "step": 8} |
| {"epoch": 0.10285714285714286, "grad_norm": 4.007819652557373, "learning_rate": 4.983899869907963e-06, "logits/chosen": 0.04794704169034958, "logits/rejected": 1.0474152565002441, "logps/chosen": -74.90632629394531, "logps/rejected": -68.87115478515625, "loss": 0.5221, "rewards/accuracies": 1.0, "rewards/chosen": 0.017194844782352448, "rewards/margins": 0.40241336822509766, "rewards/rejected": -0.3852185010910034, "step": 9} |
| {"epoch": 0.11428571428571428, "grad_norm": 4.3215508460998535, "learning_rate": 4.971401463979722e-06, "logits/chosen": 0.11494935303926468, "logits/rejected": 1.1323603391647339, "logps/chosen": -57.83014678955078, "logps/rejected": -69.66509246826172, "loss": 0.4632, "rewards/accuracies": 1.0, "rewards/chosen": 0.035677470266819, "rewards/margins": 0.5738754868507385, "rewards/rejected": -0.5381979942321777, "step": 10} |
| {"epoch": 0.12571428571428572, "grad_norm": 3.5642457008361816, "learning_rate": 4.955362799260507e-06, "logits/chosen": -0.14375551044940948, "logits/rejected": 0.6537873148918152, "logps/chosen": -53.47944259643555, "logps/rejected": -81.25194549560547, "loss": 0.4828, "rewards/accuracies": 1.0, "rewards/chosen": 0.019211266189813614, "rewards/margins": 0.5275158882141113, "rewards/rejected": -0.5083046555519104, "step": 11} |
| {"epoch": 0.13714285714285715, "grad_norm": 3.5064291954040527, "learning_rate": 4.935806850983034e-06, "logits/chosen": -0.02809108793735504, "logits/rejected": 1.3847907781600952, "logps/chosen": -54.27452850341797, "logps/rejected": -84.8036117553711, "loss": 0.3669, "rewards/accuracies": 1.0, "rewards/chosen": 0.038996659219264984, "rewards/margins": 0.9113224744796753, "rewards/rejected": -0.8723257184028625, "step": 12} |
| {"epoch": 0.14857142857142858, "grad_norm": 4.0220561027526855, "learning_rate": 4.912761632854834e-06, "logits/chosen": -0.40826213359832764, "logits/rejected": 1.0781131982803345, "logps/chosen": -80.10173797607422, "logps/rejected": -64.88600158691406, "loss": 0.4242, "rewards/accuracies": 1.0, "rewards/chosen": 0.028980137780308723, "rewards/margins": 0.7560938000679016, "rewards/rejected": -0.727113664150238, "step": 13} |
| {"epoch": 0.16, "grad_norm": 3.641127347946167, "learning_rate": 4.8862601569288885e-06, "logits/chosen": -0.3284621238708496, "logits/rejected": 0.6769169569015503, "logps/chosen": -126.9131088256836, "logps/rejected": -108.0735855102539, "loss": 0.378, "rewards/accuracies": 1.0, "rewards/chosen": 0.10002796351909637, "rewards/margins": 0.9067593812942505, "rewards/rejected": -0.8067314028739929, "step": 14} |
| {"epoch": 0.17142857142857143, "grad_norm": 3.1787471771240234, "learning_rate": 4.8563403863141825e-06, "logits/chosen": -0.23578333854675293, "logits/rejected": 0.8872382044792175, "logps/chosen": -77.02982330322266, "logps/rejected": -91.4344482421875, "loss": 0.3247, "rewards/accuracies": 1.0, "rewards/chosen": -0.0036322600208222866, "rewards/margins": 1.1259247064590454, "rewards/rejected": -1.1295570135116577, "step": 15} |
| {"epoch": 0.18285714285714286, "grad_norm": 3.3664069175720215, "learning_rate": 4.823045180793914e-06, "logits/chosen": -0.33357855677604675, "logits/rejected": 0.9027751088142395, "logps/chosen": -57.791385650634766, "logps/rejected": -71.88220977783203, "loss": 0.3925, "rewards/accuracies": 1.0, "rewards/chosen": 0.03762559965252876, "rewards/margins": 0.8615567684173584, "rewards/rejected": -0.823931097984314, "step": 16} |
| {"epoch": 0.19428571428571428, "grad_norm": 2.321681261062622, "learning_rate": 4.786422235429269e-06, "logits/chosen": -0.2371266484260559, "logits/rejected": 0.4381519556045532, "logps/chosen": -45.621620178222656, "logps/rejected": -89.46638488769531, "loss": 0.3503, "rewards/accuracies": 1.0, "rewards/chosen": 0.028645562008023262, "rewards/margins": 1.0791183710098267, "rewards/rejected": -1.0504727363586426, "step": 17} |
| {"epoch": 0.2057142857142857, "grad_norm": 2.6169960498809814, "learning_rate": 4.746524012236706e-06, "logits/chosen": -0.1888342797756195, "logits/rejected": 0.44934579730033875, "logps/chosen": -69.54957580566406, "logps/rejected": -59.541568756103516, "loss": 0.4125, "rewards/accuracies": 1.0, "rewards/chosen": 0.0358089953660965, "rewards/margins": 0.8032007813453674, "rewards/rejected": -0.7673917412757874, "step": 18} |
| {"epoch": 0.21714285714285714, "grad_norm": 2.923261880874634, "learning_rate": 4.703407665036622e-06, "logits/chosen": -0.20129157602787018, "logits/rejected": 0.6290000677108765, "logps/chosen": -56.686153411865234, "logps/rejected": -87.55162048339844, "loss": 0.264, "rewards/accuracies": 1.0, "rewards/chosen": 0.054906971752643585, "rewards/margins": 1.5454007387161255, "rewards/rejected": -1.4904940128326416, "step": 19} |
| {"epoch": 0.22857142857142856, "grad_norm": 2.864265203475952, "learning_rate": 4.657134957581057e-06, "logits/chosen": -0.3821311295032501, "logits/rejected": 0.48754024505615234, "logps/chosen": -62.24723815917969, "logps/rejected": -60.93783950805664, "loss": 0.3685, "rewards/accuracies": 1.0, "rewards/chosen": 0.06970775872468948, "rewards/margins": 1.0107057094573975, "rewards/rejected": -0.9409979581832886, "step": 20} |
| {"epoch": 0.24, "grad_norm": 1.9049395322799683, "learning_rate": 4.607772175077712e-06, "logits/chosen": -0.5025634765625, "logits/rejected": 0.4401867389678955, "logps/chosen": -57.89384460449219, "logps/rejected": -79.51171112060547, "loss": 0.3142, "rewards/accuracies": 1.0, "rewards/chosen": 0.049478232860565186, "rewards/margins": 1.3050354719161987, "rewards/rejected": -1.2555571794509888, "step": 21} |
| {"epoch": 0.25142857142857145, "grad_norm": 2.7351036071777344, "learning_rate": 4.555390029237026e-06, "logits/chosen": -0.36178550124168396, "logits/rejected": 0.8178424835205078, "logps/chosen": -65.52536010742188, "logps/rejected": -100.3760986328125, "loss": 0.2818, "rewards/accuracies": 0.9375, "rewards/chosen": -0.06696957349777222, "rewards/margins": 1.5236375331878662, "rewards/rejected": -1.5906071662902832, "step": 22} |
| {"epoch": 0.25142857142857145, "eval_logits/chosen": -0.36531969904899597, "eval_logits/rejected": 0.3482947051525116, "eval_logps/chosen": -87.77478790283203, "eval_logps/rejected": -92.11418914794922, "eval_loss": 0.30896151065826416, "eval_rewards/accuracies": 0.9800000190734863, "eval_rewards/chosen": -0.20244216918945312, "eval_rewards/margins": 1.3565798997879028, "eval_rewards/rejected": -1.559022068977356, "eval_runtime": 209.0014, "eval_samples_per_second": 0.957, "eval_steps_per_second": 0.957, "step": 22} |
| {"epoch": 0.26285714285714284, "grad_norm": 2.936436653137207, "learning_rate": 4.5000635569783365e-06, "logits/chosen": -0.278434157371521, "logits/rejected": 0.1438252478837967, "logps/chosen": -70.02806091308594, "logps/rejected": -57.32536697387695, "loss": 0.3441, "rewards/accuracies": 1.0, "rewards/chosen": 0.005648649297654629, "rewards/margins": 1.069925308227539, "rewards/rejected": -1.0642765760421753, "step": 23} |
| {"epoch": 0.2742857142857143, "grad_norm": 1.480675458908081, "learning_rate": 4.4418720129402145e-06, "logits/chosen": -0.6613065004348755, "logits/rejected": 0.20875723659992218, "logps/chosen": -43.0631217956543, "logps/rejected": -113.96063995361328, "loss": 0.2264, "rewards/accuracies": 1.0, "rewards/chosen": 0.007623173296451569, "rewards/margins": 1.854811191558838, "rewards/rejected": -1.8471879959106445, "step": 24} |
| {"epoch": 0.2857142857142857, "grad_norm": 2.0406107902526855, "learning_rate": 4.3808987559489536e-06, "logits/chosen": -0.4824203848838806, "logits/rejected": 0.370841383934021, "logps/chosen": -39.89043426513672, "logps/rejected": -89.99382781982422, "loss": 0.2224, "rewards/accuracies": 1.0, "rewards/chosen": 0.0602618083357811, "rewards/margins": 1.6918809413909912, "rewards/rejected": -1.6316192150115967, "step": 25} |
| {"epoch": 0.29714285714285715, "grad_norm": 2.2385094165802, "learning_rate": 4.317231129607859e-06, "logits/chosen": -0.5961613655090332, "logits/rejected": 0.14968276023864746, "logps/chosen": -76.45050048828125, "logps/rejected": -99.48614501953125, "loss": 0.2123, "rewards/accuracies": 1.0, "rewards/chosen": 0.07128332555294037, "rewards/margins": 1.763782024383545, "rewards/rejected": -1.6924986839294434, "step": 26} |
| {"epoch": 0.30857142857142855, "grad_norm": 1.6301268339157104, "learning_rate": 4.2509603371783776e-06, "logits/chosen": -0.3383774161338806, "logits/rejected": 0.15638643503189087, "logps/chosen": -58.246971130371094, "logps/rejected": -79.45714569091797, "loss": 0.2009, "rewards/accuracies": 1.0, "rewards/chosen": -0.04440131038427353, "rewards/margins": 1.9561799764633179, "rewards/rejected": -2.0005815029144287, "step": 27} |
| {"epoch": 0.32, "grad_norm": 2.236377716064453, "learning_rate": 4.1821813109322975e-06, "logits/chosen": -0.647399365901947, "logits/rejected": 0.22453227639198303, "logps/chosen": -88.88107299804688, "logps/rejected": -96.83210754394531, "loss": 0.2263, "rewards/accuracies": 1.0, "rewards/chosen": 0.040186457335948944, "rewards/margins": 1.824659824371338, "rewards/rejected": -1.7844732999801636, "step": 28} |
| {"epoch": 0.3314285714285714, "grad_norm": 2.0542852878570557, "learning_rate": 4.110992576162193e-06, "logits/chosen": -0.5227777361869812, "logits/rejected": -0.2961636781692505, "logps/chosen": -78.49622344970703, "logps/rejected": -78.24589538574219, "loss": 0.2121, "rewards/accuracies": 1.0, "rewards/chosen": 0.10791393369436264, "rewards/margins": 1.7918672561645508, "rewards/rejected": -1.6839532852172852, "step": 29} |
| {"epoch": 0.34285714285714286, "grad_norm": 1.9906846284866333, "learning_rate": 4.037496110044885e-06, "logits/chosen": -0.44146519899368286, "logits/rejected": 0.24826864898204803, "logps/chosen": -89.6866683959961, "logps/rejected": -122.11793518066406, "loss": 0.1621, "rewards/accuracies": 1.0, "rewards/chosen": -0.04959860444068909, "rewards/margins": 2.5837507247924805, "rewards/rejected": -2.6333494186401367, "step": 30} |
| {"epoch": 0.35428571428571426, "grad_norm": 2.3201839923858643, "learning_rate": 3.961797195560118e-06, "logits/chosen": -0.37571054697036743, "logits/rejected": -0.12684349715709686, "logps/chosen": -62.36381530761719, "logps/rejected": -68.8594741821289, "loss": 0.2406, "rewards/accuracies": 1.0, "rewards/chosen": 0.013891173526644707, "rewards/margins": 1.7061822414398193, "rewards/rejected": -1.6922911405563354, "step": 31} |
| {"epoch": 0.3657142857142857, "grad_norm": 1.7459288835525513, "learning_rate": 3.884004270673711e-06, "logits/chosen": -0.4904371500015259, "logits/rejected": -0.09516231715679169, "logps/chosen": -51.67000198364258, "logps/rejected": -87.77179718017578, "loss": 0.1739, "rewards/accuracies": 1.0, "rewards/chosen": 0.05766277387738228, "rewards/margins": 2.167266845703125, "rewards/rejected": -2.1096038818359375, "step": 32} |
| {"epoch": 0.37714285714285717, "grad_norm": 1.9478814601898193, "learning_rate": 3.8042287730012117e-06, "logits/chosen": -0.7833026051521301, "logits/rejected": -0.14795458316802979, "logps/chosen": -49.5107307434082, "logps/rejected": -77.62210845947266, "loss": 0.1861, "rewards/accuracies": 1.0, "rewards/chosen": 0.02107466384768486, "rewards/margins": 1.9514578580856323, "rewards/rejected": -1.9303834438323975, "step": 33} |
| {"epoch": 0.38857142857142857, "grad_norm": 2.1133761405944824, "learning_rate": 3.7225849801745835e-06, "logits/chosen": -0.8189583420753479, "logits/rejected": -0.24318619072437286, "logps/chosen": -75.10845947265625, "logps/rejected": -111.31871032714844, "loss": 0.1489, "rewards/accuracies": 1.0, "rewards/chosen": 0.029190005734562874, "rewards/margins": 2.6350479125976562, "rewards/rejected": -2.6058578491210938, "step": 34} |
| {"epoch": 0.4, "grad_norm": 2.116342306137085, "learning_rate": 3.6391898461406045e-06, "logits/chosen": -0.4198293685913086, "logits/rejected": -0.3276508152484894, "logps/chosen": -63.513519287109375, "logps/rejected": -93.56440734863281, "loss": 0.2266, "rewards/accuracies": 1.0, "rewards/chosen": 0.09839610755443573, "rewards/margins": 1.798091173171997, "rewards/rejected": -1.6996949911117554, "step": 35} |
| {"epoch": 0.4114285714285714, "grad_norm": 1.6935662031173706, "learning_rate": 3.55416283362546e-06, "logits/chosen": -0.8055887222290039, "logits/rejected": 0.15487802028656006, "logps/chosen": -46.510643005371094, "logps/rejected": -105.87352752685547, "loss": 0.1607, "rewards/accuracies": 1.0, "rewards/chosen": 0.033490121364593506, "rewards/margins": 2.649106025695801, "rewards/rejected": -2.6156160831451416, "step": 36} |
| {"epoch": 0.4228571428571429, "grad_norm": 2.2720091342926025, "learning_rate": 3.4676257430055438e-06, "logits/chosen": -0.8104305267333984, "logits/rejected": 0.04826318100094795, "logps/chosen": -77.22013854980469, "logps/rejected": -98.19013977050781, "loss": 0.1415, "rewards/accuracies": 1.0, "rewards/chosen": 0.03917883336544037, "rewards/margins": 2.6867618560791016, "rewards/rejected": -2.6475830078125, "step": 37} |
| {"epoch": 0.4342857142857143, "grad_norm": 1.546297550201416, "learning_rate": 3.3797025378295826e-06, "logits/chosen": -0.8403704166412354, "logits/rejected": 0.052355289459228516, "logps/chosen": -56.61090087890625, "logps/rejected": -119.7930679321289, "loss": 0.1658, "rewards/accuracies": 1.0, "rewards/chosen": 0.022541070356965065, "rewards/margins": 2.8231887817382812, "rewards/rejected": -2.800647735595703, "step": 38} |
| {"epoch": 0.44571428571428573, "grad_norm": 2.0681819915771484, "learning_rate": 3.29051916724206e-06, "logits/chosen": -0.6301568746566772, "logits/rejected": -0.32246649265289307, "logps/chosen": -66.40396118164062, "logps/rejected": -81.09312438964844, "loss": 0.1799, "rewards/accuracies": 1.0, "rewards/chosen": 0.009487343952059746, "rewards/margins": 2.2801120281219482, "rewards/rejected": -2.2706246376037598, "step": 39} |
| {"epoch": 0.45714285714285713, "grad_norm": 1.4052331447601318, "learning_rate": 3.2002033855622683e-06, "logits/chosen": -0.8533369302749634, "logits/rejected": -0.4283853769302368, "logps/chosen": -47.345802307128906, "logps/rejected": -93.77117919921875, "loss": 0.1517, "rewards/accuracies": 1.0, "rewards/chosen": -0.026743369176983833, "rewards/margins": 2.6302499771118164, "rewards/rejected": -2.6569931507110596, "step": 40} |
| {"epoch": 0.4685714285714286, "grad_norm": 2.0809950828552246, "learning_rate": 3.1088845692774798e-06, "logits/chosen": -0.6270914077758789, "logits/rejected": -0.6155582070350647, "logps/chosen": -88.60835266113281, "logps/rejected": -67.6736831665039, "loss": 0.1825, "rewards/accuracies": 1.0, "rewards/chosen": 0.04146631807088852, "rewards/margins": 2.0336129665374756, "rewards/rejected": -1.9921468496322632, "step": 41} |
| {"epoch": 0.48, "grad_norm": 1.1893919706344604, "learning_rate": 3.0166935317123824e-06, "logits/chosen": -1.0347564220428467, "logits/rejected": -0.4240405261516571, "logps/chosen": -56.76289367675781, "logps/rejected": -103.66142272949219, "loss": 0.1124, "rewards/accuracies": 1.0, "rewards/chosen": -0.057413943111896515, "rewards/margins": 3.0061051845550537, "rewards/rejected": -3.06351900100708, "step": 42} |
| {"epoch": 0.49142857142857144, "grad_norm": 1.1706154346466064, "learning_rate": 2.9237623356402423e-06, "logits/chosen": -1.0618159770965576, "logits/rejected": -0.327092707157135, "logps/chosen": -32.207794189453125, "logps/rejected": -111.8665771484375, "loss": 0.0903, "rewards/accuracies": 1.0, "rewards/chosen": -0.027263116091489792, "rewards/margins": 3.3231871128082275, "rewards/rejected": -3.350450277328491, "step": 43} |
| {"epoch": 0.5028571428571429, "grad_norm": 1.079911470413208, "learning_rate": 2.8302241041042564e-06, "logits/chosen": -1.1862759590148926, "logits/rejected": -0.7669480443000793, "logps/chosen": -89.4259262084961, "logps/rejected": -127.56314086914062, "loss": 0.0967, "rewards/accuracies": 1.0, "rewards/chosen": 0.14472071826457977, "rewards/margins": 3.0603933334350586, "rewards/rejected": -2.915672540664673, "step": 44} |
| {"epoch": 0.5028571428571429, "eval_logits/chosen": -0.8571964502334595, "eval_logits/rejected": -0.6531373858451843, "eval_logps/chosen": -90.10453033447266, "eval_logps/rejected": -104.75491333007812, "eval_loss": 0.17935897409915924, "eval_rewards/accuracies": 0.9700000286102295, "eval_rewards/chosen": -0.4354174733161926, "eval_rewards/margins": 2.387676954269409, "eval_rewards/rejected": -2.8230948448181152, "eval_runtime": 208.3973, "eval_samples_per_second": 0.96, "eval_steps_per_second": 0.96, "step": 44} |
| {"epoch": 0.5142857142857142, "grad_norm": 1.5324379205703735, "learning_rate": 2.7362128297200784e-06, "logits/chosen": -0.7203244566917419, "logits/rejected": -0.5576686859130859, "logps/chosen": -65.52645874023438, "logps/rejected": -95.54450988769531, "loss": 0.1323, "rewards/accuracies": 1.0, "rewards/chosen": 0.1149684339761734, "rewards/margins": 2.6911990642547607, "rewards/rejected": -2.576230525970459, "step": 45} |
| {"epoch": 0.5257142857142857, "grad_norm": 1.2247161865234375, "learning_rate": 2.6418631827326857e-06, "logits/chosen": -0.9313367009162903, "logits/rejected": -0.8070792555809021, "logps/chosen": -55.876502990722656, "logps/rejected": -119.91787719726562, "loss": 0.1285, "rewards/accuracies": 1.0, "rewards/chosen": 0.06400280445814133, "rewards/margins": 2.901925802230835, "rewards/rejected": -2.837923288345337, "step": 46} |
| {"epoch": 0.5371428571428571, "grad_norm": 1.3425116539001465, "learning_rate": 2.547310318102548e-06, "logits/chosen": -0.9409236907958984, "logits/rejected": -0.7352248430252075, "logps/chosen": -50.5601806640625, "logps/rejected": -86.23294830322266, "loss": 0.1466, "rewards/accuracies": 1.0, "rewards/chosen": 0.09888257831335068, "rewards/margins": 2.470425844192505, "rewards/rejected": -2.3715431690216064, "step": 47} |
| {"epoch": 0.5485714285714286, "grad_norm": 1.4391740560531616, "learning_rate": 2.4526896818974534e-06, "logits/chosen": -0.8374643921852112, "logits/rejected": -0.47213953733444214, "logps/chosen": -50.190616607666016, "logps/rejected": -91.12651824951172, "loss": 0.1023, "rewards/accuracies": 1.0, "rewards/chosen": -0.009993518702685833, "rewards/margins": 2.796506881713867, "rewards/rejected": -2.8065006732940674, "step": 48} |
| {"epoch": 0.56, "grad_norm": 1.5294134616851807, "learning_rate": 2.358136817267315e-06, "logits/chosen": -0.8648539185523987, "logits/rejected": -0.736384391784668, "logps/chosen": -72.85952758789062, "logps/rejected": -83.08409881591797, "loss": 0.1516, "rewards/accuracies": 1.0, "rewards/chosen": 0.09306713193655014, "rewards/margins": 2.5247979164123535, "rewards/rejected": -2.4317305088043213, "step": 49} |
| {"epoch": 0.5714285714285714, "grad_norm": 1.6909253597259521, "learning_rate": 2.263787170279922e-06, "logits/chosen": -1.3599584102630615, "logits/rejected": -1.2676033973693848, "logps/chosen": -78.65673065185547, "logps/rejected": -69.09491729736328, "loss": 0.1557, "rewards/accuracies": 1.0, "rewards/chosen": 0.00439932756125927, "rewards/margins": 2.1822092533111572, "rewards/rejected": -2.177809953689575, "step": 50} |
| {"epoch": 0.5828571428571429, "grad_norm": 1.3136837482452393, "learning_rate": 2.169775895895745e-06, "logits/chosen": -0.7750363349914551, "logits/rejected": -0.7582042217254639, "logps/chosen": -48.27521514892578, "logps/rejected": -105.779541015625, "loss": 0.0944, "rewards/accuracies": 1.0, "rewards/chosen": 0.060845933854579926, "rewards/margins": 3.2103078365325928, "rewards/rejected": -3.1494617462158203, "step": 51} |
| {"epoch": 0.5942857142857143, "grad_norm": 1.304816722869873, "learning_rate": 2.0762376643597586e-06, "logits/chosen": -0.7206559777259827, "logits/rejected": -0.7368943691253662, "logps/chosen": -49.27516174316406, "logps/rejected": -99.28387451171875, "loss": 0.1227, "rewards/accuracies": 1.0, "rewards/chosen": 0.08159955590963364, "rewards/margins": 2.943441390991211, "rewards/rejected": -2.861841917037964, "step": 52} |
| {"epoch": 0.6057142857142858, "grad_norm": 1.1686677932739258, "learning_rate": 1.9833064682876175e-06, "logits/chosen": -0.7927972674369812, "logits/rejected": -0.8789339065551758, "logps/chosen": -62.97304916381836, "logps/rejected": -93.07855224609375, "loss": 0.097, "rewards/accuracies": 1.0, "rewards/chosen": 0.10683320462703705, "rewards/margins": 2.952019453048706, "rewards/rejected": -2.8451859951019287, "step": 53} |
| {"epoch": 0.6171428571428571, "grad_norm": 0.7459912300109863, "learning_rate": 1.8911154307225204e-06, "logits/chosen": -0.7670217752456665, "logits/rejected": -0.7017982006072998, "logps/chosen": -31.30267333984375, "logps/rejected": -110.78553771972656, "loss": 0.0593, "rewards/accuracies": 1.0, "rewards/chosen": 0.003862532787024975, "rewards/margins": 3.5174167156219482, "rewards/rejected": -3.513554096221924, "step": 54} |
| {"epoch": 0.6285714285714286, "grad_norm": 1.5418952703475952, "learning_rate": 1.7997966144377328e-06, "logits/chosen": -0.986474871635437, "logits/rejected": -1.1061036586761475, "logps/chosen": -47.49861526489258, "logps/rejected": -67.53656005859375, "loss": 0.1543, "rewards/accuracies": 1.0, "rewards/chosen": -0.04688534885644913, "rewards/margins": 2.227695941925049, "rewards/rejected": -2.27458119392395, "step": 55} |
| {"epoch": 0.64, "grad_norm": 1.0748093128204346, "learning_rate": 1.7094808327579401e-06, "logits/chosen": -1.324556589126587, "logits/rejected": -0.8223966360092163, "logps/chosen": -36.40914535522461, "logps/rejected": -105.08839416503906, "loss": 0.084, "rewards/accuracies": 1.0, "rewards/chosen": -0.07963553071022034, "rewards/margins": 3.377859354019165, "rewards/rejected": -3.4574952125549316, "step": 56} |
| {"epoch": 0.6514285714285715, "grad_norm": 0.9280484318733215, "learning_rate": 1.6202974621704176e-06, "logits/chosen": -1.450647234916687, "logits/rejected": -0.8040248155593872, "logps/chosen": -68.76046752929688, "logps/rejected": -125.3299331665039, "loss": 0.0635, "rewards/accuracies": 1.0, "rewards/chosen": 0.06499414145946503, "rewards/margins": 3.9410018920898438, "rewards/rejected": -3.876007318496704, "step": 57} |
| {"epoch": 0.6628571428571428, "grad_norm": 0.8896822333335876, "learning_rate": 1.5323742569944573e-06, "logits/chosen": -1.0071394443511963, "logits/rejected": -1.0022106170654297, "logps/chosen": -45.09584426879883, "logps/rejected": -109.64781951904297, "loss": 0.0802, "rewards/accuracies": 1.0, "rewards/chosen": 0.034890901297330856, "rewards/margins": 3.5162737369537354, "rewards/rejected": -3.4813833236694336, "step": 58} |
| {"epoch": 0.6742857142857143, "grad_norm": 0.7682043313980103, "learning_rate": 1.44583716637454e-06, "logits/chosen": -1.0028694868087769, "logits/rejected": -0.813642144203186, "logps/chosen": -66.2764663696289, "logps/rejected": -112.77369689941406, "loss": 0.0576, "rewards/accuracies": 1.0, "rewards/chosen": -0.0501595139503479, "rewards/margins": 3.7721147537231445, "rewards/rejected": -3.822274684906006, "step": 59} |
| {"epoch": 0.6857142857142857, "grad_norm": 1.0216327905654907, "learning_rate": 1.3608101538593965e-06, "logits/chosen": -1.1250649690628052, "logits/rejected": -0.9627130627632141, "logps/chosen": -42.440757751464844, "logps/rejected": -100.12843322753906, "loss": 0.0804, "rewards/accuracies": 1.0, "rewards/chosen": 0.11411059647798538, "rewards/margins": 3.393651247024536, "rewards/rejected": -3.2795403003692627, "step": 60} |
| {"epoch": 0.6971428571428572, "grad_norm": 0.7271348834037781, "learning_rate": 1.277415019825417e-06, "logits/chosen": -0.7807080745697021, "logits/rejected": -0.7796292304992676, "logps/chosen": -33.5434455871582, "logps/rejected": -112.23175048828125, "loss": 0.0711, "rewards/accuracies": 1.0, "rewards/chosen": -0.06058162823319435, "rewards/margins": 3.8745694160461426, "rewards/rejected": -3.9351511001586914, "step": 61} |
| {"epoch": 0.7085714285714285, "grad_norm": 1.7274105548858643, "learning_rate": 1.195771226998789e-06, "logits/chosen": -0.9823087453842163, "logits/rejected": -1.2148367166519165, "logps/chosen": -72.55377960205078, "logps/rejected": -75.32920837402344, "loss": 0.1183, "rewards/accuracies": 1.0, "rewards/chosen": -0.12965497374534607, "rewards/margins": 2.7459590435028076, "rewards/rejected": -2.8756141662597656, "step": 62} |
| {"epoch": 0.72, "grad_norm": 0.9623571038246155, "learning_rate": 1.1159957293262888e-06, "logits/chosen": -1.0808844566345215, "logits/rejected": -0.8635554909706116, "logps/chosen": -49.637996673583984, "logps/rejected": -116.99398803710938, "loss": 0.09, "rewards/accuracies": 1.0, "rewards/chosen": 0.02851293236017227, "rewards/margins": 3.4344334602355957, "rewards/rejected": -3.4059205055236816, "step": 63} |
| {"epoch": 0.7314285714285714, "grad_norm": 1.0066514015197754, "learning_rate": 1.0382028044398823e-06, "logits/chosen": -1.197101354598999, "logits/rejected": -1.1306017637252808, "logps/chosen": -50.98615264892578, "logps/rejected": -120.8044662475586, "loss": 0.0899, "rewards/accuracies": 1.0, "rewards/chosen": -0.035914015024900436, "rewards/margins": 3.1908507347106934, "rewards/rejected": -3.2267649173736572, "step": 64} |
| {"epoch": 0.7428571428571429, "grad_norm": 1.1005553007125854, "learning_rate": 9.625038899551162e-07, "logits/chosen": -1.22350013256073, "logits/rejected": -1.1208860874176025, "logps/chosen": -66.32962036132812, "logps/rejected": -103.34649658203125, "loss": 0.0709, "rewards/accuracies": 1.0, "rewards/chosen": 0.18690410256385803, "rewards/margins": 3.4826159477233887, "rewards/rejected": -3.2957119941711426, "step": 65} |
| {"epoch": 0.7542857142857143, "grad_norm": 0.954483687877655, "learning_rate": 8.890074238378074e-07, "logits/chosen": -0.9665268659591675, "logits/rejected": -0.8029574751853943, "logps/chosen": -43.259857177734375, "logps/rejected": -104.5782241821289, "loss": 0.0993, "rewards/accuracies": 1.0, "rewards/chosen": -0.044777870178222656, "rewards/margins": 3.1982903480529785, "rewards/rejected": -3.243067979812622, "step": 66} |
| {"epoch": 0.7542857142857143, "eval_logits/chosen": -1.1089446544647217, "eval_logits/rejected": -1.1692878007888794, "eval_logps/chosen": -91.48172760009766, "eval_logps/rejected": -110.86624145507812, "eval_loss": 0.14132851362228394, "eval_rewards/accuracies": 0.9700000286102295, "eval_rewards/chosen": -0.573137104511261, "eval_rewards/margins": 2.8610901832580566, "eval_rewards/rejected": -3.4342269897460938, "eval_runtime": 208.8549, "eval_samples_per_second": 0.958, "eval_steps_per_second": 0.958, "step": 66} |
| {"epoch": 0.7657142857142857, "grad_norm": 0.9773061871528625, "learning_rate": 8.178186890677029e-07, "logits/chosen": -1.02699875831604, "logits/rejected": -1.1385068893432617, "logps/chosen": -80.82888793945312, "logps/rejected": -100.25572967529297, "loss": 0.072, "rewards/accuracies": 1.0, "rewards/chosen": 0.0796530619263649, "rewards/margins": 3.1960816383361816, "rewards/rejected": -3.116428852081299, "step": 67} |
| {"epoch": 0.7771428571428571, "grad_norm": 1.2940922975540161, "learning_rate": 7.490396628216237e-07, "logits/chosen": -1.2090729475021362, "logits/rejected": -1.2437564134597778, "logps/chosen": -87.85923767089844, "logps/rejected": -120.87287902832031, "loss": 0.1183, "rewards/accuracies": 1.0, "rewards/chosen": 0.018895722925662994, "rewards/margins": 2.969498634338379, "rewards/rejected": -2.9506030082702637, "step": 68} |
| {"epoch": 0.7885714285714286, "grad_norm": 0.7793561220169067, "learning_rate": 6.827688703921407e-07, "logits/chosen": -1.1516605615615845, "logits/rejected": -1.295521855354309, "logps/chosen": -52.94973373413086, "logps/rejected": -84.64179229736328, "loss": 0.067, "rewards/accuracies": 1.0, "rewards/chosen": 0.18537920713424683, "rewards/margins": 3.4953463077545166, "rewards/rejected": -3.309967041015625, "step": 69} |
| {"epoch": 0.8, "grad_norm": 1.4742248058319092, "learning_rate": 6.191012440510469e-07, "logits/chosen": -1.3784356117248535, "logits/rejected": -1.3366318941116333, "logps/chosen": -57.7204475402832, "logps/rejected": -90.2862548828125, "loss": 0.1197, "rewards/accuracies": 1.0, "rewards/chosen": -0.044269490987062454, "rewards/margins": 2.759215831756592, "rewards/rejected": -2.80348539352417, "step": 70} |
| {"epoch": 0.8114285714285714, "grad_norm": 0.7249897718429565, "learning_rate": 5.581279870597866e-07, "logits/chosen": -1.1907292604446411, "logits/rejected": -0.6756631731987, "logps/chosen": -32.33828353881836, "logps/rejected": -125.02144622802734, "loss": 0.0525, "rewards/accuracies": 1.0, "rewards/chosen": -0.06472505629062653, "rewards/margins": 4.0654168128967285, "rewards/rejected": -4.130141258239746, "step": 71} |
| {"epoch": 0.8228571428571428, "grad_norm": 1.0847841501235962, "learning_rate": 4.999364430216639e-07, "logits/chosen": -1.237339735031128, "logits/rejected": -1.5806419849395752, "logps/chosen": -52.456146240234375, "logps/rejected": -94.31544494628906, "loss": 0.0992, "rewards/accuracies": 1.0, "rewards/chosen": 0.10363951325416565, "rewards/margins": 2.9890761375427246, "rewards/rejected": -2.885436534881592, "step": 72} |
| {"epoch": 0.8342857142857143, "grad_norm": 1.278956651687622, "learning_rate": 4.4460997076297504e-07, "logits/chosen": -1.0774935483932495, "logits/rejected": -1.4760003089904785, "logps/chosen": -43.07012939453125, "logps/rejected": -74.25629425048828, "loss": 0.1131, "rewards/accuracies": 1.0, "rewards/chosen": 0.06534770131111145, "rewards/margins": 2.8206787109375, "rewards/rejected": -2.75533127784729, "step": 73} |
| {"epoch": 0.8457142857142858, "grad_norm": 0.7833404541015625, "learning_rate": 3.922278249222894e-07, "logits/chosen": -1.1311602592468262, "logits/rejected": -1.0242271423339844, "logps/chosen": -53.541568756103516, "logps/rejected": -105.48152923583984, "loss": 0.0774, "rewards/accuracies": 1.0, "rewards/chosen": 0.04792798310518265, "rewards/margins": 3.678433418273926, "rewards/rejected": -3.6305058002471924, "step": 74} |
| {"epoch": 0.8571428571428571, "grad_norm": 1.03605318069458, "learning_rate": 3.4286504241894283e-07, "logits/chosen": -0.9688073992729187, "logits/rejected": -1.2700920104980469, "logps/chosen": -38.00965118408203, "logps/rejected": -92.08466339111328, "loss": 0.1049, "rewards/accuracies": 1.0, "rewards/chosen": -0.01876715011894703, "rewards/margins": 3.16585636138916, "rewards/rejected": -3.1846237182617188, "step": 75} |
| {"epoch": 0.8685714285714285, "grad_norm": 1.319770336151123, "learning_rate": 2.965923349633779e-07, "logits/chosen": -1.3658266067504883, "logits/rejected": -1.3608825206756592, "logps/chosen": -49.00103759765625, "logps/rejected": -117.49503326416016, "loss": 0.0818, "rewards/accuracies": 1.0, "rewards/chosen": -0.09428122639656067, "rewards/margins": 3.61112642288208, "rewards/rejected": -3.7054080963134766, "step": 76} |
| {"epoch": 0.88, "grad_norm": 0.9297468662261963, "learning_rate": 2.53475987763295e-07, "logits/chosen": -1.0338634252548218, "logits/rejected": -1.1930313110351562, "logps/chosen": -67.0118179321289, "logps/rejected": -106.61903381347656, "loss": 0.0727, "rewards/accuracies": 1.0, "rewards/chosen": -0.13288229703903198, "rewards/margins": 3.487334728240967, "rewards/rejected": -3.6202168464660645, "step": 77} |
| {"epoch": 0.8914285714285715, "grad_norm": 1.1066629886627197, "learning_rate": 2.135777645707318e-07, "logits/chosen": -1.384450078010559, "logits/rejected": -1.3552913665771484, "logps/chosen": -52.43301010131836, "logps/rejected": -101.87602996826172, "loss": 0.0989, "rewards/accuracies": 1.0, "rewards/chosen": 0.009377628564834595, "rewards/margins": 3.135222911834717, "rewards/rejected": -3.125845432281494, "step": 78} |
| {"epoch": 0.9028571428571428, "grad_norm": 1.0631781816482544, "learning_rate": 1.7695481920608716e-07, "logits/chosen": -1.1844675540924072, "logits/rejected": -1.0223643779754639, "logps/chosen": -45.87510299682617, "logps/rejected": -102.84295654296875, "loss": 0.0862, "rewards/accuracies": 1.0, "rewards/chosen": -0.05180816724896431, "rewards/margins": 3.2761740684509277, "rewards/rejected": -3.3279824256896973, "step": 79} |
| {"epoch": 0.9142857142857143, "grad_norm": 1.2742042541503906, "learning_rate": 1.4365961368581844e-07, "logits/chosen": -1.2797547578811646, "logits/rejected": -1.6902124881744385, "logps/chosen": -79.47929382324219, "logps/rejected": -78.93382263183594, "loss": 0.1025, "rewards/accuracies": 1.0, "rewards/chosen": -0.0567280538380146, "rewards/margins": 2.9206228256225586, "rewards/rejected": -2.977350950241089, "step": 80} |
| {"epoch": 0.9257142857142857, "grad_norm": 0.8278792500495911, "learning_rate": 1.137398430711123e-07, "logits/chosen": -1.119363784790039, "logits/rejected": -0.9186141490936279, "logps/chosen": -29.899991989135742, "logps/rejected": -105.7519302368164, "loss": 0.0648, "rewards/accuracies": 1.0, "rewards/chosen": 0.031892240047454834, "rewards/margins": 3.5144004821777344, "rewards/rejected": -3.4825081825256348, "step": 81} |
| {"epoch": 0.9371428571428572, "grad_norm": 1.0955795049667358, "learning_rate": 8.723836714516681e-08, "logits/chosen": -1.2281020879745483, "logits/rejected": -1.3817219734191895, "logps/chosen": -73.53196716308594, "logps/rejected": -103.11698913574219, "loss": 0.0811, "rewards/accuracies": 1.0, "rewards/chosen": -0.06516408920288086, "rewards/margins": 3.2024097442626953, "rewards/rejected": -3.267573833465576, "step": 82} |
| {"epoch": 0.9485714285714286, "grad_norm": 0.6436601877212524, "learning_rate": 6.419314901696671e-08, "logits/chosen": -1.2053653001785278, "logits/rejected": -1.208804965019226, "logps/chosen": -28.06381607055664, "logps/rejected": -100.99942016601562, "loss": 0.0586, "rewards/accuracies": 1.0, "rewards/chosen": -0.027196241542696953, "rewards/margins": 3.633328437805176, "rewards/rejected": -3.660524845123291, "step": 83} |
| {"epoch": 0.96, "grad_norm": 0.9888738393783569, "learning_rate": 4.4637200739493514e-08, "logits/chosen": -1.231987714767456, "logits/rejected": -1.143172025680542, "logps/chosen": -57.63927459716797, "logps/rejected": -96.07897186279297, "loss": 0.1077, "rewards/accuracies": 1.0, "rewards/chosen": -0.05573497712612152, "rewards/margins": 3.1897521018981934, "rewards/rejected": -3.2454872131347656, "step": 84} |
| {"epoch": 0.9714285714285714, "grad_norm": 1.0415937900543213, "learning_rate": 2.8598536020278678e-08, "logits/chosen": -1.221508264541626, "logits/rejected": -1.4119690656661987, "logps/chosen": -48.45242691040039, "logps/rejected": -84.4826431274414, "loss": 0.0827, "rewards/accuracies": 1.0, "rewards/chosen": 0.04423363134264946, "rewards/margins": 3.218625545501709, "rewards/rejected": -3.174391984939575, "step": 85} |
| {"epoch": 0.9828571428571429, "grad_norm": 1.0360701084136963, "learning_rate": 1.6100130092037704e-08, "logits/chosen": -1.3318215608596802, "logits/rejected": -1.6143568754196167, "logps/chosen": -45.959354400634766, "logps/rejected": -79.45930480957031, "loss": 0.0892, "rewards/accuracies": 1.0, "rewards/chosen": -0.08850360661745071, "rewards/margins": 2.9804043769836426, "rewards/rejected": -3.068908214569092, "step": 86} |
| {"epoch": 0.9942857142857143, "grad_norm": 0.7161802053451538, "learning_rate": 7.159886800869875e-09, "logits/chosen": -1.1106700897216797, "logits/rejected": -0.9769763946533203, "logps/chosen": -85.0906982421875, "logps/rejected": -132.0802459716797, "loss": 0.0524, "rewards/accuracies": 1.0, "rewards/chosen": -0.0784958153963089, "rewards/margins": 4.037193298339844, "rewards/rejected": -4.115689277648926, "step": 87} |
| {"epoch": 1.0, "grad_norm": 2.9429450035095215, "learning_rate": 1.7906129591713228e-09, "logits/chosen": -1.2418022155761719, "logits/rejected": -0.8798779249191284, "logps/chosen": -90.4301986694336, "logps/rejected": -115.04806518554688, "loss": 0.0987, "rewards/accuracies": 1.0, "rewards/chosen": -0.08904826641082764, "rewards/margins": 3.531545639038086, "rewards/rejected": -3.620594024658203, "step": 88} |
| {"epoch": 1.0, "eval_logits/chosen": -1.1477130651474, "eval_logits/rejected": -1.244410514831543, "eval_logps/chosen": -91.70614624023438, "eval_logps/rejected": -111.82247161865234, "eval_loss": 0.1359921395778656, "eval_rewards/accuracies": 0.9700000286102295, "eval_rewards/chosen": -0.5955771207809448, "eval_rewards/margins": 2.9342737197875977, "eval_rewards/rejected": -3.529850959777832, "eval_runtime": 209.3361, "eval_samples_per_second": 0.955, "eval_steps_per_second": 0.955, "step": 88} |
| {"epoch": 1.0, "step": 88, "total_flos": 0.0, "train_loss": 0.2132695221172815, "train_runtime": 4416.7921, "train_samples_per_second": 0.317, "train_steps_per_second": 0.02} |
|
|
