| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 4.0, |
| "eval_steps": 500, |
| "global_step": 6736, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.009501187648456057, |
| "grad_norm": 54.75, |
| "learning_rate": 9.980997624703087e-07, |
| "logits/chosen": 0.09168118238449097, |
| "logits/rejected": 0.12129797041416168, |
| "logps/chosen": -36.49408721923828, |
| "logps/rejected": -49.25672149658203, |
| "loss": 0.6922, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": 0.008192603476345539, |
| "rewards/margins": 0.0024515376426279545, |
| "rewards/rejected": 0.005741065833717585, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.019002375296912115, |
| "grad_norm": 59.75, |
| "learning_rate": 9.961995249406174e-07, |
| "logits/chosen": 0.08897414803504944, |
| "logits/rejected": 0.1399449110031128, |
| "logps/chosen": -37.52133560180664, |
| "logps/rejected": -51.12624740600586, |
| "loss": 0.6829, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": 0.017915938049554825, |
| "rewards/margins": 0.021237602457404137, |
| "rewards/rejected": -0.003321664407849312, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.028503562945368172, |
| "grad_norm": 63.75, |
| "learning_rate": 9.942992874109262e-07, |
| "logits/chosen": 0.09679778665304184, |
| "logits/rejected": 0.16931985318660736, |
| "logps/chosen": -36.65215301513672, |
| "logps/rejected": -49.573421478271484, |
| "loss": 0.6798, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.03151966631412506, |
| "rewards/margins": 0.027938464656472206, |
| "rewards/rejected": 0.003581203753128648, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.03800475059382423, |
| "grad_norm": 63.25, |
| "learning_rate": 9.923990498812351e-07, |
| "logits/chosen": 0.16125047206878662, |
| "logits/rejected": 0.24480265378952026, |
| "logps/chosen": -38.162559509277344, |
| "logps/rejected": -52.15303039550781, |
| "loss": 0.6703, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": 0.03711671382188797, |
| "rewards/margins": 0.047640666365623474, |
| "rewards/rejected": -0.010523954406380653, |
| "step": 64 |
| }, |
| { |
| "epoch": 0.047505938242280284, |
| "grad_norm": 64.5, |
| "learning_rate": 9.904988123515439e-07, |
| "logits/chosen": 0.06547170132398605, |
| "logits/rejected": 0.13260145485401154, |
| "logps/chosen": -36.432579040527344, |
| "logps/rejected": -48.84739303588867, |
| "loss": 0.6626, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": 0.0562339723110199, |
| "rewards/margins": 0.0634559616446495, |
| "rewards/rejected": -0.00722198560833931, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.057007125890736345, |
| "grad_norm": 55.25, |
| "learning_rate": 9.885985748218526e-07, |
| "logits/chosen": 0.07628384232521057, |
| "logits/rejected": 0.13675282895565033, |
| "logps/chosen": -35.15069580078125, |
| "logps/rejected": -47.46355438232422, |
| "loss": 0.6585, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.06429558247327805, |
| "rewards/margins": 0.0723925530910492, |
| "rewards/rejected": -0.008096965961158276, |
| "step": 96 |
| }, |
| { |
| "epoch": 0.0665083135391924, |
| "grad_norm": 50.75, |
| "learning_rate": 9.866983372921614e-07, |
| "logits/chosen": 0.11520403623580933, |
| "logits/rejected": 0.16309119760990143, |
| "logps/chosen": -36.81483840942383, |
| "logps/rejected": -50.9425163269043, |
| "loss": 0.6533, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": 0.06526956707239151, |
| "rewards/margins": 0.08356323093175888, |
| "rewards/rejected": -0.018293654546141624, |
| "step": 112 |
| }, |
| { |
| "epoch": 0.07600950118764846, |
| "grad_norm": 70.5, |
| "learning_rate": 9.847980997624703e-07, |
| "logits/chosen": 0.08769215643405914, |
| "logits/rejected": 0.17430077493190765, |
| "logps/chosen": -37.91537094116211, |
| "logps/rejected": -52.815677642822266, |
| "loss": 0.6492, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": 0.058046698570251465, |
| "rewards/margins": 0.09332208335399628, |
| "rewards/rejected": -0.035275377333164215, |
| "step": 128 |
| }, |
| { |
| "epoch": 0.0855106888361045, |
| "grad_norm": 56.5, |
| "learning_rate": 9.82897862232779e-07, |
| "logits/chosen": 0.1254298835992813, |
| "logits/rejected": 0.1748633235692978, |
| "logps/chosen": -36.702362060546875, |
| "logps/rejected": -50.70151901245117, |
| "loss": 0.6422, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": 0.06117745488882065, |
| "rewards/margins": 0.10814622789621353, |
| "rewards/rejected": -0.04696878045797348, |
| "step": 144 |
| }, |
| { |
| "epoch": 0.09501187648456057, |
| "grad_norm": 61.75, |
| "learning_rate": 9.809976247030878e-07, |
| "logits/chosen": 0.09889174997806549, |
| "logits/rejected": 0.09486458450555801, |
| "logps/chosen": -37.1884651184082, |
| "logps/rejected": -48.955718994140625, |
| "loss": 0.6406, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": 0.08231806755065918, |
| "rewards/margins": 0.11266255378723145, |
| "rewards/rejected": -0.030344482511281967, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.10451306413301663, |
| "grad_norm": 61.0, |
| "learning_rate": 9.790973871733966e-07, |
| "logits/chosen": 0.12300257384777069, |
| "logits/rejected": 0.1372351050376892, |
| "logps/chosen": -37.07274627685547, |
| "logps/rejected": -51.80104446411133, |
| "loss": 0.6329, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.07423266023397446, |
| "rewards/margins": 0.1288163661956787, |
| "rewards/rejected": -0.05458369851112366, |
| "step": 176 |
| }, |
| { |
| "epoch": 0.11401425178147269, |
| "grad_norm": 56.25, |
| "learning_rate": 9.771971496437053e-07, |
| "logits/chosen": 0.14542920887470245, |
| "logits/rejected": 0.1965111494064331, |
| "logps/chosen": -35.992820739746094, |
| "logps/rejected": -49.162193298339844, |
| "loss": 0.6215, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": 0.09972672164440155, |
| "rewards/margins": 0.1545056849718094, |
| "rewards/rejected": -0.05477897450327873, |
| "step": 192 |
| }, |
| { |
| "epoch": 0.12351543942992874, |
| "grad_norm": 73.5, |
| "learning_rate": 9.752969121140143e-07, |
| "logits/chosen": 0.09255525469779968, |
| "logits/rejected": 0.19868624210357666, |
| "logps/chosen": -35.94331741333008, |
| "logps/rejected": -51.20383071899414, |
| "loss": 0.6101, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.10126903653144836, |
| "rewards/margins": 0.17989249527454376, |
| "rewards/rejected": -0.0786234438419342, |
| "step": 208 |
| }, |
| { |
| "epoch": 0.1330166270783848, |
| "grad_norm": 58.25, |
| "learning_rate": 9.73396674584323e-07, |
| "logits/chosen": 0.13323335349559784, |
| "logits/rejected": 0.2155267596244812, |
| "logps/chosen": -38.20836639404297, |
| "logps/rejected": -52.9131965637207, |
| "loss": 0.6177, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.05846577137708664, |
| "rewards/margins": 0.16329729557037354, |
| "rewards/rejected": -0.1048315167427063, |
| "step": 224 |
| }, |
| { |
| "epoch": 0.14251781472684086, |
| "grad_norm": 54.75, |
| "learning_rate": 9.714964370546317e-07, |
| "logits/chosen": 0.09854070097208023, |
| "logits/rejected": 0.20600098371505737, |
| "logps/chosen": -35.198726654052734, |
| "logps/rejected": -50.93071365356445, |
| "loss": 0.6034, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.0857614204287529, |
| "rewards/margins": 0.19499169290065765, |
| "rewards/rejected": -0.10923026502132416, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.15201900237529692, |
| "grad_norm": 60.25, |
| "learning_rate": 9.695961995249405e-07, |
| "logits/chosen": 0.11161897331476212, |
| "logits/rejected": 0.14389154314994812, |
| "logps/chosen": -37.637821197509766, |
| "logps/rejected": -51.35750198364258, |
| "loss": 0.6147, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": 0.03235001862049103, |
| "rewards/margins": 0.17306238412857056, |
| "rewards/rejected": -0.14071235060691833, |
| "step": 256 |
| }, |
| { |
| "epoch": 0.16152019002375298, |
| "grad_norm": 61.5, |
| "learning_rate": 9.676959619952494e-07, |
| "logits/chosen": 0.045116275548934937, |
| "logits/rejected": 0.12899161875247955, |
| "logps/chosen": -36.590667724609375, |
| "logps/rejected": -52.80071258544922, |
| "loss": 0.6001, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": 0.04269418492913246, |
| "rewards/margins": 0.20594461262226105, |
| "rewards/rejected": -0.16325044631958008, |
| "step": 272 |
| }, |
| { |
| "epoch": 0.171021377672209, |
| "grad_norm": 57.0, |
| "learning_rate": 9.657957244655582e-07, |
| "logits/chosen": 0.10446017980575562, |
| "logits/rejected": 0.1502765715122223, |
| "logps/chosen": -36.7647705078125, |
| "logps/rejected": -50.46417236328125, |
| "loss": 0.5972, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.0390743762254715, |
| "rewards/margins": 0.21257053315639496, |
| "rewards/rejected": -0.17349614202976227, |
| "step": 288 |
| }, |
| { |
| "epoch": 0.18052256532066507, |
| "grad_norm": 55.0, |
| "learning_rate": 9.63895486935867e-07, |
| "logits/chosen": 0.04691855609416962, |
| "logits/rejected": 0.17598745226860046, |
| "logps/chosen": -35.485595703125, |
| "logps/rejected": -51.60365676879883, |
| "loss": 0.582, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": 0.03419807553291321, |
| "rewards/margins": 0.2475091964006424, |
| "rewards/rejected": -0.2133111208677292, |
| "step": 304 |
| }, |
| { |
| "epoch": 0.19002375296912113, |
| "grad_norm": 59.25, |
| "learning_rate": 9.619952494061757e-07, |
| "logits/chosen": 0.15173594653606415, |
| "logits/rejected": 0.22723865509033203, |
| "logps/chosen": -38.54058837890625, |
| "logps/rejected": -53.069339752197266, |
| "loss": 0.5939, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.0007697370601817966, |
| "rewards/margins": 0.22217878699302673, |
| "rewards/rejected": -0.22294853627681732, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.1995249406175772, |
| "grad_norm": 54.5, |
| "learning_rate": 9.600950118764846e-07, |
| "logits/chosen": 0.13400091230869293, |
| "logits/rejected": 0.1886436641216278, |
| "logps/chosen": -37.70890808105469, |
| "logps/rejected": -53.3675651550293, |
| "loss": 0.5775, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.012358499690890312, |
| "rewards/margins": 0.26150980591773987, |
| "rewards/rejected": -0.2738683223724365, |
| "step": 336 |
| }, |
| { |
| "epoch": 0.20902612826603326, |
| "grad_norm": 59.0, |
| "learning_rate": 9.581947743467934e-07, |
| "logits/chosen": 0.15047906339168549, |
| "logits/rejected": 0.2155291885137558, |
| "logps/chosen": -36.07099533081055, |
| "logps/rejected": -51.531551361083984, |
| "loss": 0.577, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": 0.0014424873515963554, |
| "rewards/margins": 0.26147347688674927, |
| "rewards/rejected": -0.26003098487854004, |
| "step": 352 |
| }, |
| { |
| "epoch": 0.21852731591448932, |
| "grad_norm": 61.25, |
| "learning_rate": 9.562945368171021e-07, |
| "logits/chosen": 0.11939611285924911, |
| "logits/rejected": 0.20481155812740326, |
| "logps/chosen": -37.29627990722656, |
| "logps/rejected": -53.07925033569336, |
| "loss": 0.5697, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": 0.005910781677812338, |
| "rewards/margins": 0.28491517901420593, |
| "rewards/rejected": -0.27900439500808716, |
| "step": 368 |
| }, |
| { |
| "epoch": 0.22802850356294538, |
| "grad_norm": 55.75, |
| "learning_rate": 9.543942992874109e-07, |
| "logits/chosen": 0.10844097286462784, |
| "logits/rejected": 0.21932217478752136, |
| "logps/chosen": -37.1666259765625, |
| "logps/rejected": -53.12284851074219, |
| "loss": 0.5754, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.007143110036849976, |
| "rewards/margins": 0.2737848162651062, |
| "rewards/rejected": -0.28092795610427856, |
| "step": 384 |
| }, |
| { |
| "epoch": 0.2375296912114014, |
| "grad_norm": 54.75, |
| "learning_rate": 9.524940617577196e-07, |
| "logits/chosen": 0.10385459661483765, |
| "logits/rejected": 0.13318760693073273, |
| "logps/chosen": -35.50322341918945, |
| "logps/rejected": -50.462791442871094, |
| "loss": 0.5679, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.0004334240220487118, |
| "rewards/margins": 0.2884422540664673, |
| "rewards/rejected": -0.28800880908966064, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.24703087885985747, |
| "grad_norm": 54.25, |
| "learning_rate": 9.505938242280285e-07, |
| "logits/chosen": 0.08173692971467972, |
| "logits/rejected": 0.14498800039291382, |
| "logps/chosen": -35.70138931274414, |
| "logps/rejected": -54.012001037597656, |
| "loss": 0.5657, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.03053681179881096, |
| "rewards/margins": 0.29130610823631287, |
| "rewards/rejected": -0.3218429386615753, |
| "step": 416 |
| }, |
| { |
| "epoch": 0.25653206650831356, |
| "grad_norm": 63.75, |
| "learning_rate": 9.486935866983372e-07, |
| "logits/chosen": 0.08363573253154755, |
| "logits/rejected": 0.18085773289203644, |
| "logps/chosen": -39.11247634887695, |
| "logps/rejected": -55.508888244628906, |
| "loss": 0.5693, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.05329626053571701, |
| "rewards/margins": 0.2859145998954773, |
| "rewards/rejected": -0.3392108976840973, |
| "step": 432 |
| }, |
| { |
| "epoch": 0.2660332541567696, |
| "grad_norm": 65.5, |
| "learning_rate": 9.467933491686461e-07, |
| "logits/chosen": 0.07687534391880035, |
| "logits/rejected": 0.2082725167274475, |
| "logps/chosen": -37.038570404052734, |
| "logps/rejected": -54.50634765625, |
| "loss": 0.5379, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.019455045461654663, |
| "rewards/margins": 0.36107519268989563, |
| "rewards/rejected": -0.3805302679538727, |
| "step": 448 |
| }, |
| { |
| "epoch": 0.2755344418052256, |
| "grad_norm": 63.25, |
| "learning_rate": 9.448931116389548e-07, |
| "logits/chosen": 0.10213326662778854, |
| "logits/rejected": 0.20626387000083923, |
| "logps/chosen": -37.784019470214844, |
| "logps/rejected": -54.457462310791016, |
| "loss": 0.5493, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -0.04002974182367325, |
| "rewards/margins": 0.3406682312488556, |
| "rewards/rejected": -0.380698025226593, |
| "step": 464 |
| }, |
| { |
| "epoch": 0.2850356294536817, |
| "grad_norm": 61.75, |
| "learning_rate": 9.429928741092636e-07, |
| "logits/chosen": 0.13567008078098297, |
| "logits/rejected": 0.24827228486537933, |
| "logps/chosen": -38.69795227050781, |
| "logps/rejected": -54.621788024902344, |
| "loss": 0.5563, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.040405262261629105, |
| "rewards/margins": 0.3285280168056488, |
| "rewards/rejected": -0.3689332604408264, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.29453681710213775, |
| "grad_norm": 58.0, |
| "learning_rate": 9.410926365795724e-07, |
| "logits/chosen": 0.10663773119449615, |
| "logits/rejected": 0.16733931005001068, |
| "logps/chosen": -38.08431625366211, |
| "logps/rejected": -54.35865020751953, |
| "loss": 0.5339, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.04221474006772041, |
| "rewards/margins": 0.37557515501976013, |
| "rewards/rejected": -0.41778990626335144, |
| "step": 496 |
| }, |
| { |
| "epoch": 0.30403800475059384, |
| "grad_norm": 70.0, |
| "learning_rate": 9.391923990498812e-07, |
| "logits/chosen": 0.1082092821598053, |
| "logits/rejected": 0.23803919553756714, |
| "logps/chosen": -38.825382232666016, |
| "logps/rejected": -55.53346252441406, |
| "loss": 0.5598, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -0.09616291522979736, |
| "rewards/margins": 0.31281211972236633, |
| "rewards/rejected": -0.4089750051498413, |
| "step": 512 |
| }, |
| { |
| "epoch": 0.31353919239904987, |
| "grad_norm": 50.25, |
| "learning_rate": 9.3729216152019e-07, |
| "logits/chosen": 0.07085791230201721, |
| "logits/rejected": 0.15105192363262177, |
| "logps/chosen": -39.11968994140625, |
| "logps/rejected": -55.4873046875, |
| "loss": 0.5426, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.10313617438077927, |
| "rewards/margins": 0.351255863904953, |
| "rewards/rejected": -0.45439204573631287, |
| "step": 528 |
| }, |
| { |
| "epoch": 0.32304038004750596, |
| "grad_norm": 51.25, |
| "learning_rate": 9.353919239904988e-07, |
| "logits/chosen": 0.10827435553073883, |
| "logits/rejected": 0.17179493606090546, |
| "logps/chosen": -37.49391174316406, |
| "logps/rejected": -55.33747100830078, |
| "loss": 0.5326, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.05370311439037323, |
| "rewards/margins": 0.38469013571739197, |
| "rewards/rejected": -0.438393235206604, |
| "step": 544 |
| }, |
| { |
| "epoch": 0.332541567695962, |
| "grad_norm": 65.5, |
| "learning_rate": 9.334916864608076e-07, |
| "logits/chosen": 0.14238286018371582, |
| "logits/rejected": 0.19145098328590393, |
| "logps/chosen": -37.788291931152344, |
| "logps/rejected": -51.97111892700195, |
| "loss": 0.553, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -0.08003263175487518, |
| "rewards/margins": 0.33586975932121277, |
| "rewards/rejected": -0.41590237617492676, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.342042755344418, |
| "grad_norm": 52.75, |
| "learning_rate": 9.315914489311163e-07, |
| "logits/chosen": 0.16290903091430664, |
| "logits/rejected": 0.1991579532623291, |
| "logps/chosen": -37.21266174316406, |
| "logps/rejected": -55.373538970947266, |
| "loss": 0.524, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.07777812331914902, |
| "rewards/margins": 0.40286707878112793, |
| "rewards/rejected": -0.48064517974853516, |
| "step": 576 |
| }, |
| { |
| "epoch": 0.3515439429928741, |
| "grad_norm": 51.75, |
| "learning_rate": 9.296912114014252e-07, |
| "logits/chosen": 0.08858010172843933, |
| "logits/rejected": 0.16938555240631104, |
| "logps/chosen": -37.08319091796875, |
| "logps/rejected": -53.559356689453125, |
| "loss": 0.5302, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.08191077411174774, |
| "rewards/margins": 0.3878284990787506, |
| "rewards/rejected": -0.46973931789398193, |
| "step": 592 |
| }, |
| { |
| "epoch": 0.36104513064133015, |
| "grad_norm": 57.0, |
| "learning_rate": 9.277909738717339e-07, |
| "logits/chosen": 0.1242557018995285, |
| "logits/rejected": 0.12163582444190979, |
| "logps/chosen": -40.13254165649414, |
| "logps/rejected": -55.82794952392578, |
| "loss": 0.5285, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.10593372583389282, |
| "rewards/margins": 0.4058433473110199, |
| "rewards/rejected": -0.5117770433425903, |
| "step": 608 |
| }, |
| { |
| "epoch": 0.37054631828978624, |
| "grad_norm": 54.25, |
| "learning_rate": 9.258907363420428e-07, |
| "logits/chosen": 0.12377818673849106, |
| "logits/rejected": 0.218975231051445, |
| "logps/chosen": -38.33209228515625, |
| "logps/rejected": -54.284889221191406, |
| "loss": 0.5255, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.10532137006521225, |
| "rewards/margins": 0.40542417764663696, |
| "rewards/rejected": -0.5107455253601074, |
| "step": 624 |
| }, |
| { |
| "epoch": 0.38004750593824227, |
| "grad_norm": 61.25, |
| "learning_rate": 9.239904988123515e-07, |
| "logits/chosen": 0.14407610893249512, |
| "logits/rejected": 0.13527539372444153, |
| "logps/chosen": -39.09244918823242, |
| "logps/rejected": -56.88185501098633, |
| "loss": 0.5161, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.12786737084388733, |
| "rewards/margins": 0.4305534064769745, |
| "rewards/rejected": -0.5584207773208618, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.38954869358669836, |
| "grad_norm": 60.5, |
| "learning_rate": 9.220902612826604e-07, |
| "logits/chosen": 0.08770464360713959, |
| "logits/rejected": 0.1615772247314453, |
| "logps/chosen": -40.214927673339844, |
| "logps/rejected": -57.730621337890625, |
| "loss": 0.4933, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.10945181548595428, |
| "rewards/margins": 0.4860302805900574, |
| "rewards/rejected": -0.5954821109771729, |
| "step": 656 |
| }, |
| { |
| "epoch": 0.3990498812351544, |
| "grad_norm": 49.5, |
| "learning_rate": 9.201900237529691e-07, |
| "logits/chosen": 0.1510995626449585, |
| "logits/rejected": 0.20847982168197632, |
| "logps/chosen": -37.78511428833008, |
| "logps/rejected": -55.67912292480469, |
| "loss": 0.4915, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.07964207231998444, |
| "rewards/margins": 0.5031406283378601, |
| "rewards/rejected": -0.5827827453613281, |
| "step": 672 |
| }, |
| { |
| "epoch": 0.4085510688836104, |
| "grad_norm": 45.75, |
| "learning_rate": 9.18289786223278e-07, |
| "logits/chosen": 0.08455921709537506, |
| "logits/rejected": 0.19820302724838257, |
| "logps/chosen": -37.23653793334961, |
| "logps/rejected": -52.45553207397461, |
| "loss": 0.5195, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.10724642127752304, |
| "rewards/margins": 0.42769795656204224, |
| "rewards/rejected": -0.5349443554878235, |
| "step": 688 |
| }, |
| { |
| "epoch": 0.4180522565320665, |
| "grad_norm": 49.75, |
| "learning_rate": 9.163895486935866e-07, |
| "logits/chosen": 0.0849796012043953, |
| "logits/rejected": 0.1387515664100647, |
| "logps/chosen": -36.640594482421875, |
| "logps/rejected": -56.54729080200195, |
| "loss": 0.4855, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.07652697712182999, |
| "rewards/margins": 0.5180978775024414, |
| "rewards/rejected": -0.5946248173713684, |
| "step": 704 |
| }, |
| { |
| "epoch": 0.42755344418052255, |
| "grad_norm": 50.25, |
| "learning_rate": 9.144893111638954e-07, |
| "logits/chosen": 0.1335923671722412, |
| "logits/rejected": 0.12739677727222443, |
| "logps/chosen": -38.18653106689453, |
| "logps/rejected": -56.48445510864258, |
| "loss": 0.4906, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.10945376753807068, |
| "rewards/margins": 0.4993862211704254, |
| "rewards/rejected": -0.6088399887084961, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.43705463182897863, |
| "grad_norm": 47.75, |
| "learning_rate": 9.125890736342042e-07, |
| "logits/chosen": 0.05496565252542496, |
| "logits/rejected": 0.15505348145961761, |
| "logps/chosen": -38.44123077392578, |
| "logps/rejected": -58.174560546875, |
| "loss": 0.4734, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.1328837275505066, |
| "rewards/margins": 0.5390680432319641, |
| "rewards/rejected": -0.6719517707824707, |
| "step": 736 |
| }, |
| { |
| "epoch": 0.44655581947743467, |
| "grad_norm": 51.0, |
| "learning_rate": 9.106888361045129e-07, |
| "logits/chosen": 0.17514106631278992, |
| "logits/rejected": 0.21084193885326385, |
| "logps/chosen": -39.44230270385742, |
| "logps/rejected": -55.33711624145508, |
| "loss": 0.4927, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.13082744181156158, |
| "rewards/margins": 0.4961455464363098, |
| "rewards/rejected": -0.626973032951355, |
| "step": 752 |
| }, |
| { |
| "epoch": 0.45605700712589076, |
| "grad_norm": 53.5, |
| "learning_rate": 9.087885985748218e-07, |
| "logits/chosen": 0.05614431947469711, |
| "logits/rejected": 0.16487029194831848, |
| "logps/chosen": -38.37213897705078, |
| "logps/rejected": -56.517974853515625, |
| "loss": 0.4808, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.13590717315673828, |
| "rewards/margins": 0.5437510013580322, |
| "rewards/rejected": -0.6796582341194153, |
| "step": 768 |
| }, |
| { |
| "epoch": 0.4655581947743468, |
| "grad_norm": 56.5, |
| "learning_rate": 9.068883610451305e-07, |
| "logits/chosen": 0.09643438458442688, |
| "logits/rejected": 0.1764623522758484, |
| "logps/chosen": -37.71235275268555, |
| "logps/rejected": -55.301570892333984, |
| "loss": 0.4736, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.10847848653793335, |
| "rewards/margins": 0.5602878928184509, |
| "rewards/rejected": -0.6687663793563843, |
| "step": 784 |
| }, |
| { |
| "epoch": 0.4750593824228028, |
| "grad_norm": 50.5, |
| "learning_rate": 9.049881235154394e-07, |
| "logits/chosen": 0.10280097275972366, |
| "logits/rejected": 0.24883826076984406, |
| "logps/chosen": -37.97513198852539, |
| "logps/rejected": -55.975555419921875, |
| "loss": 0.4709, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.1473047137260437, |
| "rewards/margins": 0.5804386734962463, |
| "rewards/rejected": -0.72774338722229, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.4845605700712589, |
| "grad_norm": 57.0, |
| "learning_rate": 9.030878859857481e-07, |
| "logits/chosen": 0.13920438289642334, |
| "logits/rejected": 0.2100549191236496, |
| "logps/chosen": -38.78316116333008, |
| "logps/rejected": -57.39037322998047, |
| "loss": 0.4858, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.19835878908634186, |
| "rewards/margins": 0.5184602737426758, |
| "rewards/rejected": -0.716819167137146, |
| "step": 816 |
| }, |
| { |
| "epoch": 0.49406175771971494, |
| "grad_norm": 47.25, |
| "learning_rate": 9.01187648456057e-07, |
| "logits/chosen": 0.04185810685157776, |
| "logits/rejected": 0.16661542654037476, |
| "logps/chosen": -38.728607177734375, |
| "logps/rejected": -56.995845794677734, |
| "loss": 0.4705, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.18127599358558655, |
| "rewards/margins": 0.5616625547409058, |
| "rewards/rejected": -0.7429385185241699, |
| "step": 832 |
| }, |
| { |
| "epoch": 0.503562945368171, |
| "grad_norm": 55.5, |
| "learning_rate": 8.992874109263657e-07, |
| "logits/chosen": 0.09122167527675629, |
| "logits/rejected": 0.22163383662700653, |
| "logps/chosen": -37.88745880126953, |
| "logps/rejected": -58.1086311340332, |
| "loss": 0.502, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.20832395553588867, |
| "rewards/margins": 0.4904058575630188, |
| "rewards/rejected": -0.6987298130989075, |
| "step": 848 |
| }, |
| { |
| "epoch": 0.5130641330166271, |
| "grad_norm": 53.75, |
| "learning_rate": 8.973871733966746e-07, |
| "logits/chosen": 0.0702139213681221, |
| "logits/rejected": 0.1391507089138031, |
| "logps/chosen": -40.5086784362793, |
| "logps/rejected": -56.50019454956055, |
| "loss": 0.4854, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.26017051935195923, |
| "rewards/margins": 0.5463624000549316, |
| "rewards/rejected": -0.8065329194068909, |
| "step": 864 |
| }, |
| { |
| "epoch": 0.5225653206650831, |
| "grad_norm": 46.0, |
| "learning_rate": 8.954869358669833e-07, |
| "logits/chosen": 0.1008288711309433, |
| "logits/rejected": 0.20593173801898956, |
| "logps/chosen": -38.77046585083008, |
| "logps/rejected": -56.287872314453125, |
| "loss": 0.4734, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.18773072957992554, |
| "rewards/margins": 0.567348301410675, |
| "rewards/rejected": -0.7550791501998901, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.5320665083135392, |
| "grad_norm": 56.0, |
| "learning_rate": 8.935866983372922e-07, |
| "logits/chosen": 0.061835043132305145, |
| "logits/rejected": 0.16514989733695984, |
| "logps/chosen": -40.11195373535156, |
| "logps/rejected": -58.53046798706055, |
| "loss": 0.4531, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.1927071213722229, |
| "rewards/margins": 0.6243357062339783, |
| "rewards/rejected": -0.817042887210846, |
| "step": 896 |
| }, |
| { |
| "epoch": 0.5415676959619953, |
| "grad_norm": 49.5, |
| "learning_rate": 8.916864608076009e-07, |
| "logits/chosen": 0.14350858330726624, |
| "logits/rejected": 0.1577494591474533, |
| "logps/chosen": -39.235504150390625, |
| "logps/rejected": -58.71210861206055, |
| "loss": 0.4787, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.22734667360782623, |
| "rewards/margins": 0.5474963784217834, |
| "rewards/rejected": -0.7748430967330933, |
| "step": 912 |
| }, |
| { |
| "epoch": 0.5510688836104513, |
| "grad_norm": 38.25, |
| "learning_rate": 8.897862232779097e-07, |
| "logits/chosen": 0.07540839910507202, |
| "logits/rejected": 0.12306182831525803, |
| "logps/chosen": -37.44085693359375, |
| "logps/rejected": -57.793243408203125, |
| "loss": 0.4301, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.15863735973834991, |
| "rewards/margins": 0.693757176399231, |
| "rewards/rejected": -0.8523945808410645, |
| "step": 928 |
| }, |
| { |
| "epoch": 0.5605700712589073, |
| "grad_norm": 47.25, |
| "learning_rate": 8.878859857482185e-07, |
| "logits/chosen": 0.005182682536542416, |
| "logits/rejected": 0.1266525387763977, |
| "logps/chosen": -38.00994873046875, |
| "logps/rejected": -57.80335235595703, |
| "loss": 0.4626, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.16963782906532288, |
| "rewards/margins": 0.6042665839195251, |
| "rewards/rejected": -0.7739044427871704, |
| "step": 944 |
| }, |
| { |
| "epoch": 0.5700712589073634, |
| "grad_norm": 55.5, |
| "learning_rate": 8.859857482185272e-07, |
| "logits/chosen": 0.11473742127418518, |
| "logits/rejected": 0.16739153861999512, |
| "logps/chosen": -41.33670425415039, |
| "logps/rejected": -59.52238082885742, |
| "loss": 0.4705, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.25219887495040894, |
| "rewards/margins": 0.5897648334503174, |
| "rewards/rejected": -0.8419637680053711, |
| "step": 960 |
| }, |
| { |
| "epoch": 0.5795724465558195, |
| "grad_norm": 46.5, |
| "learning_rate": 8.840855106888361e-07, |
| "logits/chosen": 0.03330547362565994, |
| "logits/rejected": 0.15450525283813477, |
| "logps/chosen": -38.50295639038086, |
| "logps/rejected": -55.91643142700195, |
| "loss": 0.4555, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.1985245943069458, |
| "rewards/margins": 0.6212279200553894, |
| "rewards/rejected": -0.8197525143623352, |
| "step": 976 |
| }, |
| { |
| "epoch": 0.5890736342042755, |
| "grad_norm": 49.25, |
| "learning_rate": 8.821852731591448e-07, |
| "logits/chosen": 0.10139049589633942, |
| "logits/rejected": 0.19957150518894196, |
| "logps/chosen": -39.53318405151367, |
| "logps/rejected": -60.60167694091797, |
| "loss": 0.4703, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.23673467338085175, |
| "rewards/margins": 0.5786986351013184, |
| "rewards/rejected": -0.8154332637786865, |
| "step": 992 |
| }, |
| { |
| "epoch": 0.5985748218527316, |
| "grad_norm": 42.5, |
| "learning_rate": 8.802850356294537e-07, |
| "logits/chosen": 0.08935532718896866, |
| "logits/rejected": 0.18748882412910461, |
| "logps/chosen": -39.9503059387207, |
| "logps/rejected": -58.329071044921875, |
| "loss": 0.4571, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.23434796929359436, |
| "rewards/margins": 0.6296550631523132, |
| "rewards/rejected": -0.86400306224823, |
| "step": 1008 |
| }, |
| { |
| "epoch": 0.6080760095011877, |
| "grad_norm": 48.0, |
| "learning_rate": 8.783847980997624e-07, |
| "logits/chosen": 0.12793917953968048, |
| "logits/rejected": 0.26207882165908813, |
| "logps/chosen": -40.79100036621094, |
| "logps/rejected": -63.207977294921875, |
| "loss": 0.4433, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.2365538626909256, |
| "rewards/margins": 0.6767521500587463, |
| "rewards/rejected": -0.9133059978485107, |
| "step": 1024 |
| }, |
| { |
| "epoch": 0.6175771971496437, |
| "grad_norm": 44.75, |
| "learning_rate": 8.764845605700713e-07, |
| "logits/chosen": 0.07575452327728271, |
| "logits/rejected": 0.13339047133922577, |
| "logps/chosen": -38.435760498046875, |
| "logps/rejected": -56.613590240478516, |
| "loss": 0.4691, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.20284534990787506, |
| "rewards/margins": 0.6009599566459656, |
| "rewards/rejected": -0.8038052916526794, |
| "step": 1040 |
| }, |
| { |
| "epoch": 0.6270783847980997, |
| "grad_norm": 49.75, |
| "learning_rate": 8.7458432304038e-07, |
| "logits/chosen": 0.08708982169628143, |
| "logits/rejected": 0.1805320382118225, |
| "logps/chosen": -38.411643981933594, |
| "logps/rejected": -62.70646286010742, |
| "loss": 0.4134, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.2167506217956543, |
| "rewards/margins": 0.7745501399040222, |
| "rewards/rejected": -0.9913008213043213, |
| "step": 1056 |
| }, |
| { |
| "epoch": 0.6365795724465558, |
| "grad_norm": 74.5, |
| "learning_rate": 8.726840855106889e-07, |
| "logits/chosen": 0.10599172860383987, |
| "logits/rejected": 0.17720839381217957, |
| "logps/chosen": -41.43864059448242, |
| "logps/rejected": -60.35655212402344, |
| "loss": 0.4606, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.24390998482704163, |
| "rewards/margins": 0.6325302124023438, |
| "rewards/rejected": -0.8764402270317078, |
| "step": 1072 |
| }, |
| { |
| "epoch": 0.6460807600950119, |
| "grad_norm": 54.5, |
| "learning_rate": 8.707838479809976e-07, |
| "logits/chosen": 0.1116759404540062, |
| "logits/rejected": 0.12336824834346771, |
| "logps/chosen": -40.634971618652344, |
| "logps/rejected": -57.01872253417969, |
| "loss": 0.4681, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.251184344291687, |
| "rewards/margins": 0.6177032589912415, |
| "rewards/rejected": -0.8688876628875732, |
| "step": 1088 |
| }, |
| { |
| "epoch": 0.6555819477434679, |
| "grad_norm": 40.0, |
| "learning_rate": 8.688836104513065e-07, |
| "logits/chosen": 0.05607762932777405, |
| "logits/rejected": 0.16543659567832947, |
| "logps/chosen": -40.00341796875, |
| "logps/rejected": -58.64830780029297, |
| "loss": 0.4405, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.2381763756275177, |
| "rewards/margins": 0.6710238456726074, |
| "rewards/rejected": -0.909200131893158, |
| "step": 1104 |
| }, |
| { |
| "epoch": 0.665083135391924, |
| "grad_norm": 50.25, |
| "learning_rate": 8.669833729216152e-07, |
| "logits/chosen": 0.049548353999853134, |
| "logits/rejected": 0.13943257927894592, |
| "logps/chosen": -39.979637145996094, |
| "logps/rejected": -59.84418487548828, |
| "loss": 0.4359, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.2502007484436035, |
| "rewards/margins": 0.7026790380477905, |
| "rewards/rejected": -0.9528796672821045, |
| "step": 1120 |
| }, |
| { |
| "epoch": 0.6745843230403801, |
| "grad_norm": 46.5, |
| "learning_rate": 8.65083135391924e-07, |
| "logits/chosen": 0.08080411702394485, |
| "logits/rejected": 0.15538814663887024, |
| "logps/chosen": -37.723243713378906, |
| "logps/rejected": -58.846256256103516, |
| "loss": 0.422, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.18230988085269928, |
| "rewards/margins": 0.7491470575332642, |
| "rewards/rejected": -0.931456983089447, |
| "step": 1136 |
| }, |
| { |
| "epoch": 0.684085510688836, |
| "grad_norm": 38.5, |
| "learning_rate": 8.631828978622328e-07, |
| "logits/chosen": 0.10914994031190872, |
| "logits/rejected": 0.22802668809890747, |
| "logps/chosen": -40.43394088745117, |
| "logps/rejected": -58.36912536621094, |
| "loss": 0.4662, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.2760140001773834, |
| "rewards/margins": 0.613175094127655, |
| "rewards/rejected": -0.8891890048980713, |
| "step": 1152 |
| }, |
| { |
| "epoch": 0.6935866983372921, |
| "grad_norm": 50.0, |
| "learning_rate": 8.612826603325414e-07, |
| "logits/chosen": 0.09185300767421722, |
| "logits/rejected": 0.1087762862443924, |
| "logps/chosen": -38.46809768676758, |
| "logps/rejected": -57.867942810058594, |
| "loss": 0.4468, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.26649224758148193, |
| "rewards/margins": 0.6601444482803345, |
| "rewards/rejected": -0.9266366362571716, |
| "step": 1168 |
| }, |
| { |
| "epoch": 0.7030878859857482, |
| "grad_norm": 48.75, |
| "learning_rate": 8.593824228028503e-07, |
| "logits/chosen": 0.09283562749624252, |
| "logits/rejected": 0.1531636118888855, |
| "logps/chosen": -41.238792419433594, |
| "logps/rejected": -59.725738525390625, |
| "loss": 0.447, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.30539485812187195, |
| "rewards/margins": 0.671943187713623, |
| "rewards/rejected": -0.9773380160331726, |
| "step": 1184 |
| }, |
| { |
| "epoch": 0.7125890736342043, |
| "grad_norm": 54.25, |
| "learning_rate": 8.57482185273159e-07, |
| "logits/chosen": 0.17938324809074402, |
| "logits/rejected": 0.21232807636260986, |
| "logps/chosen": -39.86995315551758, |
| "logps/rejected": -58.8725471496582, |
| "loss": 0.4251, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.25202998518943787, |
| "rewards/margins": 0.7341017723083496, |
| "rewards/rejected": -0.9861317873001099, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.7220902612826603, |
| "grad_norm": 54.25, |
| "learning_rate": 8.555819477434679e-07, |
| "logits/chosen": 0.0566461905837059, |
| "logits/rejected": 0.12666890025138855, |
| "logps/chosen": -39.986106872558594, |
| "logps/rejected": -61.40876388549805, |
| "loss": 0.4067, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.2511664927005768, |
| "rewards/margins": 0.7968044281005859, |
| "rewards/rejected": -1.0479708909988403, |
| "step": 1216 |
| }, |
| { |
| "epoch": 0.7315914489311164, |
| "grad_norm": 58.75, |
| "learning_rate": 8.536817102137766e-07, |
| "logits/chosen": 0.08908773213624954, |
| "logits/rejected": 0.13832062482833862, |
| "logps/chosen": -39.75481414794922, |
| "logps/rejected": -60.21009826660156, |
| "loss": 0.4281, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.24241387844085693, |
| "rewards/margins": 0.7212178707122803, |
| "rewards/rejected": -0.9636316895484924, |
| "step": 1232 |
| }, |
| { |
| "epoch": 0.7410926365795725, |
| "grad_norm": 52.5, |
| "learning_rate": 8.517814726840855e-07, |
| "logits/chosen": 0.0984007716178894, |
| "logits/rejected": 0.15883538126945496, |
| "logps/chosen": -38.90775680541992, |
| "logps/rejected": -59.53275680541992, |
| "loss": 0.4418, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.27383100986480713, |
| "rewards/margins": 0.7038523554801941, |
| "rewards/rejected": -0.9776833653450012, |
| "step": 1248 |
| }, |
| { |
| "epoch": 0.7505938242280285, |
| "grad_norm": 41.25, |
| "learning_rate": 8.498812351543942e-07, |
| "logits/chosen": 0.11719369888305664, |
| "logits/rejected": 0.17845718562602997, |
| "logps/chosen": -38.104705810546875, |
| "logps/rejected": -61.452178955078125, |
| "loss": 0.3903, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.20659206807613373, |
| "rewards/margins": 0.855985701084137, |
| "rewards/rejected": -1.062577724456787, |
| "step": 1264 |
| }, |
| { |
| "epoch": 0.7600950118764845, |
| "grad_norm": 45.75, |
| "learning_rate": 8.479809976247031e-07, |
| "logits/chosen": 0.0768904983997345, |
| "logits/rejected": 0.20530077815055847, |
| "logps/chosen": -39.765953063964844, |
| "logps/rejected": -61.358497619628906, |
| "loss": 0.4049, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.2901615798473358, |
| "rewards/margins": 0.8085931539535522, |
| "rewards/rejected": -1.098754644393921, |
| "step": 1280 |
| }, |
| { |
| "epoch": 0.7695961995249406, |
| "grad_norm": 41.5, |
| "learning_rate": 8.460807600950118e-07, |
| "logits/chosen": 0.1819642186164856, |
| "logits/rejected": 0.20391146838665009, |
| "logps/chosen": -41.09254455566406, |
| "logps/rejected": -62.414024353027344, |
| "loss": 0.4161, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.2619169354438782, |
| "rewards/margins": 0.7796422243118286, |
| "rewards/rejected": -1.0415592193603516, |
| "step": 1296 |
| }, |
| { |
| "epoch": 0.7790973871733967, |
| "grad_norm": 45.75, |
| "learning_rate": 8.441805225653206e-07, |
| "logits/chosen": 0.07217932492494583, |
| "logits/rejected": 0.15043236315250397, |
| "logps/chosen": -41.25364685058594, |
| "logps/rejected": -59.63226318359375, |
| "loss": 0.4309, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3054697513580322, |
| "rewards/margins": 0.7218286395072937, |
| "rewards/rejected": -1.0272983312606812, |
| "step": 1312 |
| }, |
| { |
| "epoch": 0.7885985748218527, |
| "grad_norm": 62.75, |
| "learning_rate": 8.422802850356294e-07, |
| "logits/chosen": 0.10031426697969437, |
| "logits/rejected": 0.197869673371315, |
| "logps/chosen": -40.79019546508789, |
| "logps/rejected": -61.019493103027344, |
| "loss": 0.4362, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.2853235602378845, |
| "rewards/margins": 0.7121058106422424, |
| "rewards/rejected": -0.9974292516708374, |
| "step": 1328 |
| }, |
| { |
| "epoch": 0.7980997624703088, |
| "grad_norm": 54.0, |
| "learning_rate": 8.403800475059381e-07, |
| "logits/chosen": 0.044592004269361496, |
| "logits/rejected": 0.19066573679447174, |
| "logps/chosen": -40.317413330078125, |
| "logps/rejected": -58.56877899169922, |
| "loss": 0.4502, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.30221185088157654, |
| "rewards/margins": 0.6648804545402527, |
| "rewards/rejected": -0.9670923352241516, |
| "step": 1344 |
| }, |
| { |
| "epoch": 0.8076009501187649, |
| "grad_norm": 40.75, |
| "learning_rate": 8.38479809976247e-07, |
| "logits/chosen": 0.06595605611801147, |
| "logits/rejected": 0.18088053166866302, |
| "logps/chosen": -39.756439208984375, |
| "logps/rejected": -63.843971252441406, |
| "loss": 0.3955, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.2804669737815857, |
| "rewards/margins": 0.8086118102073669, |
| "rewards/rejected": -1.0890789031982422, |
| "step": 1360 |
| }, |
| { |
| "epoch": 0.8171021377672208, |
| "grad_norm": 48.0, |
| "learning_rate": 8.365795724465557e-07, |
| "logits/chosen": 0.1287180781364441, |
| "logits/rejected": 0.22206488251686096, |
| "logps/chosen": -41.125389099121094, |
| "logps/rejected": -61.68186950683594, |
| "loss": 0.4428, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.30739057064056396, |
| "rewards/margins": 0.7095006704330444, |
| "rewards/rejected": -1.0168912410736084, |
| "step": 1376 |
| }, |
| { |
| "epoch": 0.8266033254156769, |
| "grad_norm": 52.25, |
| "learning_rate": 8.346793349168646e-07, |
| "logits/chosen": 0.053852953016757965, |
| "logits/rejected": 0.13252206146717072, |
| "logps/chosen": -40.2414436340332, |
| "logps/rejected": -61.29638671875, |
| "loss": 0.4071, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.3235268294811249, |
| "rewards/margins": 0.811761736869812, |
| "rewards/rejected": -1.1352884769439697, |
| "step": 1392 |
| }, |
| { |
| "epoch": 0.836104513064133, |
| "grad_norm": 56.0, |
| "learning_rate": 8.327790973871733e-07, |
| "logits/chosen": 0.0468582957983017, |
| "logits/rejected": 0.11399667710065842, |
| "logps/chosen": -39.64308547973633, |
| "logps/rejected": -59.60944747924805, |
| "loss": 0.4147, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.2946662902832031, |
| "rewards/margins": 0.7918639183044434, |
| "rewards/rejected": -1.086530089378357, |
| "step": 1408 |
| }, |
| { |
| "epoch": 0.8456057007125891, |
| "grad_norm": 60.5, |
| "learning_rate": 8.308788598574822e-07, |
| "logits/chosen": 0.09335757046937943, |
| "logits/rejected": 0.10522627085447311, |
| "logps/chosen": -42.684288024902344, |
| "logps/rejected": -61.48929214477539, |
| "loss": 0.4301, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3570723533630371, |
| "rewards/margins": 0.7129464745521545, |
| "rewards/rejected": -1.0700188875198364, |
| "step": 1424 |
| }, |
| { |
| "epoch": 0.8551068883610451, |
| "grad_norm": 51.75, |
| "learning_rate": 8.289786223277909e-07, |
| "logits/chosen": 0.07347086817026138, |
| "logits/rejected": 0.15308743715286255, |
| "logps/chosen": -41.02301788330078, |
| "logps/rejected": -61.934791564941406, |
| "loss": 0.4048, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.29406049847602844, |
| "rewards/margins": 0.8210353255271912, |
| "rewards/rejected": -1.1150959730148315, |
| "step": 1440 |
| }, |
| { |
| "epoch": 0.8646080760095012, |
| "grad_norm": 45.0, |
| "learning_rate": 8.270783847980998e-07, |
| "logits/chosen": 0.0793566182255745, |
| "logits/rejected": 0.17386886477470398, |
| "logps/chosen": -37.58202362060547, |
| "logps/rejected": -59.483882904052734, |
| "loss": 0.4083, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.2546355128288269, |
| "rewards/margins": 0.7988392114639282, |
| "rewards/rejected": -1.0534747838974, |
| "step": 1456 |
| }, |
| { |
| "epoch": 0.8741092636579573, |
| "grad_norm": 51.5, |
| "learning_rate": 8.251781472684085e-07, |
| "logits/chosen": 0.06765860319137573, |
| "logits/rejected": 0.17789000272750854, |
| "logps/chosen": -38.726783752441406, |
| "logps/rejected": -60.934757232666016, |
| "loss": 0.4203, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.3265742063522339, |
| "rewards/margins": 0.7777712345123291, |
| "rewards/rejected": -1.1043453216552734, |
| "step": 1472 |
| }, |
| { |
| "epoch": 0.8836104513064132, |
| "grad_norm": 56.5, |
| "learning_rate": 8.232779097387174e-07, |
| "logits/chosen": 0.05681402608752251, |
| "logits/rejected": 0.1788835972547531, |
| "logps/chosen": -38.938507080078125, |
| "logps/rejected": -61.65933609008789, |
| "loss": 0.3918, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.25624170899391174, |
| "rewards/margins": 0.8536828756332397, |
| "rewards/rejected": -1.1099246740341187, |
| "step": 1488 |
| }, |
| { |
| "epoch": 0.8931116389548693, |
| "grad_norm": 49.25, |
| "learning_rate": 8.213776722090261e-07, |
| "logits/chosen": 0.057860612869262695, |
| "logits/rejected": 0.14475533366203308, |
| "logps/chosen": -37.889766693115234, |
| "logps/rejected": -61.03107452392578, |
| "loss": 0.3916, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.23101815581321716, |
| "rewards/margins": 0.8708871603012085, |
| "rewards/rejected": -1.101905345916748, |
| "step": 1504 |
| }, |
| { |
| "epoch": 0.9026128266033254, |
| "grad_norm": 48.5, |
| "learning_rate": 8.194774346793349e-07, |
| "logits/chosen": 0.07886646687984467, |
| "logits/rejected": 0.19103488326072693, |
| "logps/chosen": -40.67872619628906, |
| "logps/rejected": -61.686668395996094, |
| "loss": 0.4185, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.3377780318260193, |
| "rewards/margins": 0.795622706413269, |
| "rewards/rejected": -1.133400797843933, |
| "step": 1520 |
| }, |
| { |
| "epoch": 0.9121140142517815, |
| "grad_norm": 44.0, |
| "learning_rate": 8.175771971496437e-07, |
| "logits/chosen": 0.0845412164926529, |
| "logits/rejected": 0.16773808002471924, |
| "logps/chosen": -40.12324523925781, |
| "logps/rejected": -59.303245544433594, |
| "loss": 0.4103, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.3262190520763397, |
| "rewards/margins": 0.8095906376838684, |
| "rewards/rejected": -1.1358096599578857, |
| "step": 1536 |
| }, |
| { |
| "epoch": 0.9216152019002375, |
| "grad_norm": 46.0, |
| "learning_rate": 8.156769596199525e-07, |
| "logits/chosen": 0.08066831529140472, |
| "logits/rejected": 0.17000192403793335, |
| "logps/chosen": -40.7078742980957, |
| "logps/rejected": -62.315185546875, |
| "loss": 0.3972, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.28264638781547546, |
| "rewards/margins": 0.8620734214782715, |
| "rewards/rejected": -1.1447197198867798, |
| "step": 1552 |
| }, |
| { |
| "epoch": 0.9311163895486936, |
| "grad_norm": 37.75, |
| "learning_rate": 8.137767220902613e-07, |
| "logits/chosen": 0.08594940602779388, |
| "logits/rejected": 0.22140294313430786, |
| "logps/chosen": -38.64836120605469, |
| "logps/rejected": -62.625946044921875, |
| "loss": 0.376, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.2832408845424652, |
| "rewards/margins": 0.9163691997528076, |
| "rewards/rejected": -1.1996098756790161, |
| "step": 1568 |
| }, |
| { |
| "epoch": 0.9406175771971497, |
| "grad_norm": 61.5, |
| "learning_rate": 8.1187648456057e-07, |
| "logits/chosen": 0.07032456994056702, |
| "logits/rejected": 0.17861232161521912, |
| "logps/chosen": -39.57463836669922, |
| "logps/rejected": -60.60888671875, |
| "loss": 0.4058, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.35363835096359253, |
| "rewards/margins": 0.8107983469963074, |
| "rewards/rejected": -1.1644368171691895, |
| "step": 1584 |
| }, |
| { |
| "epoch": 0.9501187648456056, |
| "grad_norm": 52.25, |
| "learning_rate": 8.099762470308789e-07, |
| "logits/chosen": 0.08332876116037369, |
| "logits/rejected": 0.1855737268924713, |
| "logps/chosen": -38.6751594543457, |
| "logps/rejected": -59.24980163574219, |
| "loss": 0.4035, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3069637417793274, |
| "rewards/margins": 0.8082422614097595, |
| "rewards/rejected": -1.115206003189087, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.9596199524940617, |
| "grad_norm": 45.75, |
| "learning_rate": 8.080760095011876e-07, |
| "logits/chosen": 0.12488888204097748, |
| "logits/rejected": 0.14733757078647614, |
| "logps/chosen": -40.226043701171875, |
| "logps/rejected": -58.75604248046875, |
| "loss": 0.4261, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.35558950901031494, |
| "rewards/margins": 0.7374365925788879, |
| "rewards/rejected": -1.0930261611938477, |
| "step": 1616 |
| }, |
| { |
| "epoch": 0.9691211401425178, |
| "grad_norm": 43.5, |
| "learning_rate": 8.061757719714965e-07, |
| "logits/chosen": 0.04622222110629082, |
| "logits/rejected": 0.1545945107936859, |
| "logps/chosen": -39.63484573364258, |
| "logps/rejected": -61.277587890625, |
| "loss": 0.3831, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3327168822288513, |
| "rewards/margins": 0.8751581311225891, |
| "rewards/rejected": -1.2078750133514404, |
| "step": 1632 |
| }, |
| { |
| "epoch": 0.9786223277909739, |
| "grad_norm": 54.0, |
| "learning_rate": 8.042755344418051e-07, |
| "logits/chosen": 0.09918095171451569, |
| "logits/rejected": 0.18718725442886353, |
| "logps/chosen": -42.15406799316406, |
| "logps/rejected": -62.292816162109375, |
| "loss": 0.4012, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.3841949701309204, |
| "rewards/margins": 0.8375911116600037, |
| "rewards/rejected": -1.2217860221862793, |
| "step": 1648 |
| }, |
| { |
| "epoch": 0.9881235154394299, |
| "grad_norm": 47.0, |
| "learning_rate": 8.02375296912114e-07, |
| "logits/chosen": 0.06581688672304153, |
| "logits/rejected": 0.16606493294239044, |
| "logps/chosen": -40.35301971435547, |
| "logps/rejected": -60.86550521850586, |
| "loss": 0.4171, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.3261529803276062, |
| "rewards/margins": 0.7935004830360413, |
| "rewards/rejected": -1.1196534633636475, |
| "step": 1664 |
| }, |
| { |
| "epoch": 0.997624703087886, |
| "grad_norm": 46.75, |
| "learning_rate": 8.004750593824227e-07, |
| "logits/chosen": 0.08406564593315125, |
| "logits/rejected": 0.1631183624267578, |
| "logps/chosen": -42.632301330566406, |
| "logps/rejected": -66.31587219238281, |
| "loss": 0.3864, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3757409453392029, |
| "rewards/margins": 0.8852084875106812, |
| "rewards/rejected": -1.2609493732452393, |
| "step": 1680 |
| }, |
| { |
| "epoch": 1.007125890736342, |
| "grad_norm": 53.5, |
| "learning_rate": 7.985748218527315e-07, |
| "logits/chosen": 0.04815902188420296, |
| "logits/rejected": 0.10080163925886154, |
| "logps/chosen": -40.61830520629883, |
| "logps/rejected": -62.35159683227539, |
| "loss": 0.3894, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.3303859233856201, |
| "rewards/margins": 0.8965721726417542, |
| "rewards/rejected": -1.2269580364227295, |
| "step": 1696 |
| }, |
| { |
| "epoch": 1.0166270783847982, |
| "grad_norm": 58.5, |
| "learning_rate": 7.966745843230403e-07, |
| "logits/chosen": 0.07239064574241638, |
| "logits/rejected": 0.1947220116853714, |
| "logps/chosen": -42.951637268066406, |
| "logps/rejected": -64.53704071044922, |
| "loss": 0.4144, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.397784024477005, |
| "rewards/margins": 0.7604966759681702, |
| "rewards/rejected": -1.158280611038208, |
| "step": 1712 |
| }, |
| { |
| "epoch": 1.0261282660332542, |
| "grad_norm": 59.75, |
| "learning_rate": 7.947743467933491e-07, |
| "logits/chosen": 0.08597906678915024, |
| "logits/rejected": 0.14526161551475525, |
| "logps/chosen": -40.39187240600586, |
| "logps/rejected": -64.09424591064453, |
| "loss": 0.3834, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.33142587542533875, |
| "rewards/margins": 0.926814079284668, |
| "rewards/rejected": -1.258239984512329, |
| "step": 1728 |
| }, |
| { |
| "epoch": 1.03562945368171, |
| "grad_norm": 40.5, |
| "learning_rate": 7.928741092636579e-07, |
| "logits/chosen": 0.0840529203414917, |
| "logits/rejected": 0.18427981436252594, |
| "logps/chosen": -42.51103591918945, |
| "logps/rejected": -64.92820739746094, |
| "loss": 0.4181, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.39203089475631714, |
| "rewards/margins": 0.7877532839775085, |
| "rewards/rejected": -1.1797842979431152, |
| "step": 1744 |
| }, |
| { |
| "epoch": 1.0451306413301662, |
| "grad_norm": 56.5, |
| "learning_rate": 7.909738717339667e-07, |
| "logits/chosen": 0.049816541373729706, |
| "logits/rejected": 0.143840953707695, |
| "logps/chosen": -41.118350982666016, |
| "logps/rejected": -61.427818298339844, |
| "loss": 0.4304, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.38534730672836304, |
| "rewards/margins": 0.7327947020530701, |
| "rewards/rejected": -1.118142008781433, |
| "step": 1760 |
| }, |
| { |
| "epoch": 1.0546318289786223, |
| "grad_norm": 59.0, |
| "learning_rate": 7.890736342042755e-07, |
| "logits/chosen": 0.09707099944353104, |
| "logits/rejected": 0.10315439105033875, |
| "logps/chosen": -39.37982940673828, |
| "logps/rejected": -61.087913513183594, |
| "loss": 0.4052, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.30062562227249146, |
| "rewards/margins": 0.834286630153656, |
| "rewards/rejected": -1.1349122524261475, |
| "step": 1776 |
| }, |
| { |
| "epoch": 1.0641330166270784, |
| "grad_norm": 46.5, |
| "learning_rate": 7.871733966745842e-07, |
| "logits/chosen": -0.01599489152431488, |
| "logits/rejected": 0.1043221652507782, |
| "logps/chosen": -42.238426208496094, |
| "logps/rejected": -64.75145721435547, |
| "loss": 0.389, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.383517324924469, |
| "rewards/margins": 0.8845040202140808, |
| "rewards/rejected": -1.2680213451385498, |
| "step": 1792 |
| }, |
| { |
| "epoch": 1.0736342042755345, |
| "grad_norm": 60.5, |
| "learning_rate": 7.852731591448931e-07, |
| "logits/chosen": 0.04319116473197937, |
| "logits/rejected": 0.10541323572397232, |
| "logps/chosen": -40.8631706237793, |
| "logps/rejected": -59.777008056640625, |
| "loss": 0.3936, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.30645447969436646, |
| "rewards/margins": 0.8425821661949158, |
| "rewards/rejected": -1.1490366458892822, |
| "step": 1808 |
| }, |
| { |
| "epoch": 1.0831353919239906, |
| "grad_norm": 48.75, |
| "learning_rate": 7.833729216152018e-07, |
| "logits/chosen": 0.10437559336423874, |
| "logits/rejected": 0.1490824967622757, |
| "logps/chosen": -39.50224304199219, |
| "logps/rejected": -62.01877975463867, |
| "loss": 0.3812, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3324194550514221, |
| "rewards/margins": 0.897403359413147, |
| "rewards/rejected": -1.2298228740692139, |
| "step": 1824 |
| }, |
| { |
| "epoch": 1.0926365795724466, |
| "grad_norm": 48.75, |
| "learning_rate": 7.814726840855107e-07, |
| "logits/chosen": -0.01841648668050766, |
| "logits/rejected": 0.10108716785907745, |
| "logps/chosen": -39.490970611572266, |
| "logps/rejected": -59.66122055053711, |
| "loss": 0.3821, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3041851818561554, |
| "rewards/margins": 0.8931726813316345, |
| "rewards/rejected": -1.1973577737808228, |
| "step": 1840 |
| }, |
| { |
| "epoch": 1.1021377672209025, |
| "grad_norm": 44.0, |
| "learning_rate": 7.795724465558194e-07, |
| "logits/chosen": 0.08540444076061249, |
| "logits/rejected": 0.18863236904144287, |
| "logps/chosen": -41.12372589111328, |
| "logps/rejected": -62.18980026245117, |
| "loss": 0.3981, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.3796708285808563, |
| "rewards/margins": 0.852729320526123, |
| "rewards/rejected": -1.2324001789093018, |
| "step": 1856 |
| }, |
| { |
| "epoch": 1.1116389548693586, |
| "grad_norm": 51.0, |
| "learning_rate": 7.776722090261282e-07, |
| "logits/chosen": 0.14734028279781342, |
| "logits/rejected": 0.21354371309280396, |
| "logps/chosen": -43.433292388916016, |
| "logps/rejected": -66.97840118408203, |
| "loss": 0.3905, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.40543994307518005, |
| "rewards/margins": 0.8719741702079773, |
| "rewards/rejected": -1.277414083480835, |
| "step": 1872 |
| }, |
| { |
| "epoch": 1.1211401425178147, |
| "grad_norm": 40.5, |
| "learning_rate": 7.75771971496437e-07, |
| "logits/chosen": 0.04771037772297859, |
| "logits/rejected": 0.14470888674259186, |
| "logps/chosen": -40.52289962768555, |
| "logps/rejected": -63.578758239746094, |
| "loss": 0.3997, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3288933336734772, |
| "rewards/margins": 0.8785545229911804, |
| "rewards/rejected": -1.20744788646698, |
| "step": 1888 |
| }, |
| { |
| "epoch": 1.1306413301662708, |
| "grad_norm": 56.25, |
| "learning_rate": 7.738717339667458e-07, |
| "logits/chosen": 0.02750781551003456, |
| "logits/rejected": 0.12446936219930649, |
| "logps/chosen": -39.09437561035156, |
| "logps/rejected": -60.32615661621094, |
| "loss": 0.3828, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.29237014055252075, |
| "rewards/margins": 0.9157527089118958, |
| "rewards/rejected": -1.208122730255127, |
| "step": 1904 |
| }, |
| { |
| "epoch": 1.1401425178147269, |
| "grad_norm": 50.25, |
| "learning_rate": 7.719714964370546e-07, |
| "logits/chosen": -0.0005194246768951416, |
| "logits/rejected": 0.11599244177341461, |
| "logps/chosen": -38.9823112487793, |
| "logps/rejected": -60.105751037597656, |
| "loss": 0.3918, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3398272395133972, |
| "rewards/margins": 0.8596587181091309, |
| "rewards/rejected": -1.1994860172271729, |
| "step": 1920 |
| }, |
| { |
| "epoch": 1.149643705463183, |
| "grad_norm": 52.25, |
| "learning_rate": 7.700712589073634e-07, |
| "logits/chosen": 0.14513945579528809, |
| "logits/rejected": 0.1485050767660141, |
| "logps/chosen": -39.983741760253906, |
| "logps/rejected": -62.480751037597656, |
| "loss": 0.3717, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3441847264766693, |
| "rewards/margins": 0.9267476201057434, |
| "rewards/rejected": -1.2709323167800903, |
| "step": 1936 |
| }, |
| { |
| "epoch": 1.159144893111639, |
| "grad_norm": 56.25, |
| "learning_rate": 7.681710213776722e-07, |
| "logits/chosen": 0.0954248383641243, |
| "logits/rejected": 0.15733623504638672, |
| "logps/chosen": -40.76930618286133, |
| "logps/rejected": -63.525718688964844, |
| "loss": 0.391, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.36353814601898193, |
| "rewards/margins": 0.9048362970352173, |
| "rewards/rejected": -1.2683744430541992, |
| "step": 1952 |
| }, |
| { |
| "epoch": 1.168646080760095, |
| "grad_norm": 55.5, |
| "learning_rate": 7.66270783847981e-07, |
| "logits/chosen": 0.08181347697973251, |
| "logits/rejected": 0.12661109864711761, |
| "logps/chosen": -41.755645751953125, |
| "logps/rejected": -61.083946228027344, |
| "loss": 0.3934, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3673307001590729, |
| "rewards/margins": 0.8563836812973022, |
| "rewards/rejected": -1.2237144708633423, |
| "step": 1968 |
| }, |
| { |
| "epoch": 1.178147268408551, |
| "grad_norm": 52.0, |
| "learning_rate": 7.643705463182898e-07, |
| "logits/chosen": 0.042737413197755814, |
| "logits/rejected": 0.1984286606311798, |
| "logps/chosen": -40.4138298034668, |
| "logps/rejected": -65.36979675292969, |
| "loss": 0.3825, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.35838350653648376, |
| "rewards/margins": 0.8961877822875977, |
| "rewards/rejected": -1.2545711994171143, |
| "step": 1984 |
| }, |
| { |
| "epoch": 1.187648456057007, |
| "grad_norm": 52.0, |
| "learning_rate": 7.624703087885986e-07, |
| "logits/chosen": 0.06372962146997452, |
| "logits/rejected": 0.20503537356853485, |
| "logps/chosen": -39.09539794921875, |
| "logps/rejected": -65.76801300048828, |
| "loss": 0.3814, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.301858514547348, |
| "rewards/margins": 0.8968943953514099, |
| "rewards/rejected": -1.1987528800964355, |
| "step": 2000 |
| }, |
| { |
| "epoch": 1.1971496437054632, |
| "grad_norm": 44.0, |
| "learning_rate": 7.605700712589074e-07, |
| "logits/chosen": 0.09966941177845001, |
| "logits/rejected": 0.17389996349811554, |
| "logps/chosen": -44.16709899902344, |
| "logps/rejected": -61.92054748535156, |
| "loss": 0.4317, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -0.43170446157455444, |
| "rewards/margins": 0.799846887588501, |
| "rewards/rejected": -1.2315512895584106, |
| "step": 2016 |
| }, |
| { |
| "epoch": 1.2066508313539193, |
| "grad_norm": 41.0, |
| "learning_rate": 7.586698337292161e-07, |
| "logits/chosen": 0.0820382758975029, |
| "logits/rejected": 0.1380407214164734, |
| "logps/chosen": -40.85121536254883, |
| "logps/rejected": -61.79035568237305, |
| "loss": 0.3944, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.33426377177238464, |
| "rewards/margins": 0.8878821730613708, |
| "rewards/rejected": -1.222145915031433, |
| "step": 2032 |
| }, |
| { |
| "epoch": 1.2161520190023754, |
| "grad_norm": 49.25, |
| "learning_rate": 7.567695961995249e-07, |
| "logits/chosen": 0.07470327615737915, |
| "logits/rejected": 0.17876245081424713, |
| "logps/chosen": -40.400997161865234, |
| "logps/rejected": -62.64784240722656, |
| "loss": 0.3868, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3647741377353668, |
| "rewards/margins": 0.8964493274688721, |
| "rewards/rejected": -1.261223554611206, |
| "step": 2048 |
| }, |
| { |
| "epoch": 1.2256532066508314, |
| "grad_norm": 59.25, |
| "learning_rate": 7.548693586698337e-07, |
| "logits/chosen": 0.09447329491376877, |
| "logits/rejected": 0.20229777693748474, |
| "logps/chosen": -41.54475021362305, |
| "logps/rejected": -62.25804138183594, |
| "loss": 0.411, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.39685580134391785, |
| "rewards/margins": 0.8458825945854187, |
| "rewards/rejected": -1.2427384853363037, |
| "step": 2064 |
| }, |
| { |
| "epoch": 1.2351543942992875, |
| "grad_norm": 46.75, |
| "learning_rate": 7.529691211401425e-07, |
| "logits/chosen": 0.005977040156722069, |
| "logits/rejected": 0.12502533197402954, |
| "logps/chosen": -39.64662170410156, |
| "logps/rejected": -61.320465087890625, |
| "loss": 0.3843, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.35564541816711426, |
| "rewards/margins": 0.9019960165023804, |
| "rewards/rejected": -1.2576414346694946, |
| "step": 2080 |
| }, |
| { |
| "epoch": 1.2446555819477434, |
| "grad_norm": 56.75, |
| "learning_rate": 7.510688836104513e-07, |
| "logits/chosen": 0.08646165579557419, |
| "logits/rejected": 0.19107480347156525, |
| "logps/chosen": -41.432594299316406, |
| "logps/rejected": -63.792335510253906, |
| "loss": 0.4067, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.4125351309776306, |
| "rewards/margins": 0.8356191515922546, |
| "rewards/rejected": -1.2481542825698853, |
| "step": 2096 |
| }, |
| { |
| "epoch": 1.2541567695961995, |
| "grad_norm": 50.0, |
| "learning_rate": 7.4916864608076e-07, |
| "logits/chosen": 0.0462346225976944, |
| "logits/rejected": 0.18776997923851013, |
| "logps/chosen": -39.760746002197266, |
| "logps/rejected": -62.441749572753906, |
| "loss": 0.3845, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.30308061838150024, |
| "rewards/margins": 0.87934410572052, |
| "rewards/rejected": -1.182424783706665, |
| "step": 2112 |
| }, |
| { |
| "epoch": 1.2636579572446556, |
| "grad_norm": 46.25, |
| "learning_rate": 7.472684085510688e-07, |
| "logits/chosen": 0.08780578523874283, |
| "logits/rejected": 0.16588638722896576, |
| "logps/chosen": -41.076515197753906, |
| "logps/rejected": -62.57638931274414, |
| "loss": 0.3855, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3240537941455841, |
| "rewards/margins": 0.8670827150344849, |
| "rewards/rejected": -1.191136360168457, |
| "step": 2128 |
| }, |
| { |
| "epoch": 1.2731591448931117, |
| "grad_norm": 41.0, |
| "learning_rate": 7.453681710213776e-07, |
| "logits/chosen": 0.11139998584985733, |
| "logits/rejected": 0.18487989902496338, |
| "logps/chosen": -40.500728607177734, |
| "logps/rejected": -61.71664810180664, |
| "loss": 0.4028, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.3737491965293884, |
| "rewards/margins": 0.8448764085769653, |
| "rewards/rejected": -1.218625545501709, |
| "step": 2144 |
| }, |
| { |
| "epoch": 1.2826603325415677, |
| "grad_norm": 51.25, |
| "learning_rate": 7.434679334916864e-07, |
| "logits/chosen": 0.05119692161679268, |
| "logits/rejected": 0.12550300359725952, |
| "logps/chosen": -43.28956604003906, |
| "logps/rejected": -63.49314880371094, |
| "loss": 0.3864, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.43977251648902893, |
| "rewards/margins": 0.8914352655410767, |
| "rewards/rejected": -1.3312077522277832, |
| "step": 2160 |
| }, |
| { |
| "epoch": 1.2921615201900236, |
| "grad_norm": 66.0, |
| "learning_rate": 7.415676959619952e-07, |
| "logits/chosen": 0.023000139743089676, |
| "logits/rejected": 0.10380115360021591, |
| "logps/chosen": -40.769859313964844, |
| "logps/rejected": -61.942955017089844, |
| "loss": 0.3927, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3861158490180969, |
| "rewards/margins": 0.8635023832321167, |
| "rewards/rejected": -1.2496182918548584, |
| "step": 2176 |
| }, |
| { |
| "epoch": 1.3016627078384797, |
| "grad_norm": 56.0, |
| "learning_rate": 7.39667458432304e-07, |
| "logits/chosen": 0.08992882817983627, |
| "logits/rejected": 0.16640856862068176, |
| "logps/chosen": -41.653472900390625, |
| "logps/rejected": -65.48009490966797, |
| "loss": 0.3645, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3618718385696411, |
| "rewards/margins": 0.9549592733383179, |
| "rewards/rejected": -1.3168312311172485, |
| "step": 2192 |
| }, |
| { |
| "epoch": 1.3111638954869358, |
| "grad_norm": 48.5, |
| "learning_rate": 7.377672209026128e-07, |
| "logits/chosen": 0.022954029962420464, |
| "logits/rejected": 0.1399441808462143, |
| "logps/chosen": -40.20348358154297, |
| "logps/rejected": -62.03968811035156, |
| "loss": 0.3814, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.33127692341804504, |
| "rewards/margins": 0.9193905591964722, |
| "rewards/rejected": -1.250667691230774, |
| "step": 2208 |
| }, |
| { |
| "epoch": 1.3206650831353919, |
| "grad_norm": 45.0, |
| "learning_rate": 7.358669833729216e-07, |
| "logits/chosen": 0.05273166671395302, |
| "logits/rejected": 0.10670921206474304, |
| "logps/chosen": -40.3055419921875, |
| "logps/rejected": -60.818115234375, |
| "loss": 0.4315, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.36937782168388367, |
| "rewards/margins": 0.7719258666038513, |
| "rewards/rejected": -1.1413036584854126, |
| "step": 2224 |
| }, |
| { |
| "epoch": 1.330166270783848, |
| "grad_norm": 46.5, |
| "learning_rate": 7.339667458432304e-07, |
| "logits/chosen": 0.020988432690501213, |
| "logits/rejected": 0.13837015628814697, |
| "logps/chosen": -38.807682037353516, |
| "logps/rejected": -61.88154983520508, |
| "loss": 0.3805, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.2889925241470337, |
| "rewards/margins": 0.9347207546234131, |
| "rewards/rejected": -1.2237131595611572, |
| "step": 2240 |
| }, |
| { |
| "epoch": 1.339667458432304, |
| "grad_norm": 49.25, |
| "learning_rate": 7.320665083135391e-07, |
| "logits/chosen": 0.09183872491121292, |
| "logits/rejected": 0.19175560772418976, |
| "logps/chosen": -42.616172790527344, |
| "logps/rejected": -64.80868530273438, |
| "loss": 0.3724, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3987230360507965, |
| "rewards/margins": 0.9598705768585205, |
| "rewards/rejected": -1.3585937023162842, |
| "step": 2256 |
| }, |
| { |
| "epoch": 1.3491686460807601, |
| "grad_norm": 56.75, |
| "learning_rate": 7.301662707838479e-07, |
| "logits/chosen": 0.04315632954239845, |
| "logits/rejected": 0.12694051861763, |
| "logps/chosen": -42.23524856567383, |
| "logps/rejected": -63.53181076049805, |
| "loss": 0.4178, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.45748934149742126, |
| "rewards/margins": 0.7994433045387268, |
| "rewards/rejected": -1.2569326162338257, |
| "step": 2272 |
| }, |
| { |
| "epoch": 1.3586698337292162, |
| "grad_norm": 51.75, |
| "learning_rate": 7.282660332541567e-07, |
| "logits/chosen": 0.05002054572105408, |
| "logits/rejected": 0.10560965538024902, |
| "logps/chosen": -41.06108474731445, |
| "logps/rejected": -61.318199157714844, |
| "loss": 0.3899, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.39432668685913086, |
| "rewards/margins": 0.906670331954956, |
| "rewards/rejected": -1.300997018814087, |
| "step": 2288 |
| }, |
| { |
| "epoch": 1.3681710213776723, |
| "grad_norm": 59.0, |
| "learning_rate": 7.263657957244655e-07, |
| "logits/chosen": 0.0978277325630188, |
| "logits/rejected": 0.1469835340976715, |
| "logps/chosen": -40.50400161743164, |
| "logps/rejected": -64.60050964355469, |
| "loss": 0.3702, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.39239317178726196, |
| "rewards/margins": 0.9568536281585693, |
| "rewards/rejected": -1.3492467403411865, |
| "step": 2304 |
| }, |
| { |
| "epoch": 1.3776722090261282, |
| "grad_norm": 50.25, |
| "learning_rate": 7.244655581947743e-07, |
| "logits/chosen": 0.0486019104719162, |
| "logits/rejected": 0.17572590708732605, |
| "logps/chosen": -40.24808883666992, |
| "logps/rejected": -61.60661697387695, |
| "loss": 0.4009, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3608768582344055, |
| "rewards/margins": 0.831383466720581, |
| "rewards/rejected": -1.1922603845596313, |
| "step": 2320 |
| }, |
| { |
| "epoch": 1.3871733966745843, |
| "grad_norm": 54.25, |
| "learning_rate": 7.225653206650831e-07, |
| "logits/chosen": 0.03973031044006348, |
| "logits/rejected": 0.14902335405349731, |
| "logps/chosen": -39.89193344116211, |
| "logps/rejected": -61.921695709228516, |
| "loss": 0.3674, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3707820177078247, |
| "rewards/margins": 0.9807763695716858, |
| "rewards/rejected": -1.3515583276748657, |
| "step": 2336 |
| }, |
| { |
| "epoch": 1.3966745843230404, |
| "grad_norm": 45.25, |
| "learning_rate": 7.206650831353919e-07, |
| "logits/chosen": 0.01807180978357792, |
| "logits/rejected": 0.14278706908226013, |
| "logps/chosen": -39.26378631591797, |
| "logps/rejected": -60.40557861328125, |
| "loss": 0.3723, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.36147797107696533, |
| "rewards/margins": 0.9218441247940063, |
| "rewards/rejected": -1.2833220958709717, |
| "step": 2352 |
| }, |
| { |
| "epoch": 1.4061757719714965, |
| "grad_norm": 75.5, |
| "learning_rate": 7.187648456057007e-07, |
| "logits/chosen": 0.07302900403738022, |
| "logits/rejected": 0.14269563555717468, |
| "logps/chosen": -41.234161376953125, |
| "logps/rejected": -66.46778869628906, |
| "loss": 0.3693, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.39566075801849365, |
| "rewards/margins": 0.9585368633270264, |
| "rewards/rejected": -1.35419762134552, |
| "step": 2368 |
| }, |
| { |
| "epoch": 1.4156769596199525, |
| "grad_norm": 40.75, |
| "learning_rate": 7.168646080760095e-07, |
| "logits/chosen": 0.051808636635541916, |
| "logits/rejected": 0.21229980885982513, |
| "logps/chosen": -40.08659744262695, |
| "logps/rejected": -61.43524932861328, |
| "loss": 0.3833, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.37025976181030273, |
| "rewards/margins": 0.8906149864196777, |
| "rewards/rejected": -1.2608747482299805, |
| "step": 2384 |
| }, |
| { |
| "epoch": 1.4251781472684084, |
| "grad_norm": 44.75, |
| "learning_rate": 7.149643705463183e-07, |
| "logits/chosen": 0.0380956195294857, |
| "logits/rejected": 0.13877098262310028, |
| "logps/chosen": -42.67876052856445, |
| "logps/rejected": -63.79023361206055, |
| "loss": 0.3799, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3804953396320343, |
| "rewards/margins": 0.9205693006515503, |
| "rewards/rejected": -1.3010647296905518, |
| "step": 2400 |
| }, |
| { |
| "epoch": 1.4346793349168645, |
| "grad_norm": 51.5, |
| "learning_rate": 7.130641330166271e-07, |
| "logits/chosen": 0.08207520097494125, |
| "logits/rejected": 0.14595362544059753, |
| "logps/chosen": -42.30644989013672, |
| "logps/rejected": -63.68936538696289, |
| "loss": 0.3859, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4032093584537506, |
| "rewards/margins": 0.8909638524055481, |
| "rewards/rejected": -1.294173240661621, |
| "step": 2416 |
| }, |
| { |
| "epoch": 1.4441805225653206, |
| "grad_norm": 47.0, |
| "learning_rate": 7.111638954869358e-07, |
| "logits/chosen": 0.017898384481668472, |
| "logits/rejected": 0.18962475657463074, |
| "logps/chosen": -41.686912536621094, |
| "logps/rejected": -62.23005294799805, |
| "loss": 0.3961, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3815171718597412, |
| "rewards/margins": 0.8675276637077332, |
| "rewards/rejected": -1.2490447759628296, |
| "step": 2432 |
| }, |
| { |
| "epoch": 1.4536817102137767, |
| "grad_norm": 46.5, |
| "learning_rate": 7.092636579572447e-07, |
| "logits/chosen": 0.07278081774711609, |
| "logits/rejected": 0.13306282460689545, |
| "logps/chosen": -39.96598815917969, |
| "logps/rejected": -62.889739990234375, |
| "loss": 0.3467, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.3247521221637726, |
| "rewards/margins": 1.0032382011413574, |
| "rewards/rejected": -1.3279902935028076, |
| "step": 2448 |
| }, |
| { |
| "epoch": 1.4631828978622328, |
| "grad_norm": 46.75, |
| "learning_rate": 7.073634204275534e-07, |
| "logits/chosen": 0.028041554614901543, |
| "logits/rejected": 0.126488596200943, |
| "logps/chosen": -42.11606216430664, |
| "logps/rejected": -60.70077896118164, |
| "loss": 0.4265, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.48028531670570374, |
| "rewards/margins": 0.801145076751709, |
| "rewards/rejected": -1.2814303636550903, |
| "step": 2464 |
| }, |
| { |
| "epoch": 1.4726840855106889, |
| "grad_norm": 52.75, |
| "learning_rate": 7.054631828978623e-07, |
| "logits/chosen": 0.05731602758169174, |
| "logits/rejected": 0.13755974173545837, |
| "logps/chosen": -39.297462463378906, |
| "logps/rejected": -62.45978546142578, |
| "loss": 0.373, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.31363803148269653, |
| "rewards/margins": 0.9406128525733948, |
| "rewards/rejected": -1.2542507648468018, |
| "step": 2480 |
| }, |
| { |
| "epoch": 1.482185273159145, |
| "grad_norm": 42.0, |
| "learning_rate": 7.03562945368171e-07, |
| "logits/chosen": 0.0816030278801918, |
| "logits/rejected": 0.14841099083423615, |
| "logps/chosen": -41.25545883178711, |
| "logps/rejected": -61.70344924926758, |
| "loss": 0.4112, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.4528937041759491, |
| "rewards/margins": 0.8154016733169556, |
| "rewards/rejected": -1.268295407295227, |
| "step": 2496 |
| }, |
| { |
| "epoch": 1.491686460807601, |
| "grad_norm": 47.75, |
| "learning_rate": 7.016627078384798e-07, |
| "logits/chosen": 0.07149530947208405, |
| "logits/rejected": 0.17929969727993011, |
| "logps/chosen": -41.5832633972168, |
| "logps/rejected": -67.25575256347656, |
| "loss": 0.3869, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.41033345460891724, |
| "rewards/margins": 0.9050929546356201, |
| "rewards/rejected": -1.3154264688491821, |
| "step": 2512 |
| }, |
| { |
| "epoch": 1.5011876484560571, |
| "grad_norm": 43.5, |
| "learning_rate": 6.997624703087886e-07, |
| "logits/chosen": 0.07096850126981735, |
| "logits/rejected": 0.14748625457286835, |
| "logps/chosen": -42.1703987121582, |
| "logps/rejected": -63.72838592529297, |
| "loss": 0.3933, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.43840450048446655, |
| "rewards/margins": 0.9121676683425903, |
| "rewards/rejected": -1.3505722284317017, |
| "step": 2528 |
| }, |
| { |
| "epoch": 1.5106888361045132, |
| "grad_norm": 40.25, |
| "learning_rate": 6.978622327790974e-07, |
| "logits/chosen": 0.07429444789886475, |
| "logits/rejected": 0.169959157705307, |
| "logps/chosen": -42.15153121948242, |
| "logps/rejected": -67.66064453125, |
| "loss": 0.3566, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.385272741317749, |
| "rewards/margins": 1.010432481765747, |
| "rewards/rejected": -1.3957051038742065, |
| "step": 2544 |
| }, |
| { |
| "epoch": 1.520190023752969, |
| "grad_norm": 46.5, |
| "learning_rate": 6.959619952494062e-07, |
| "logits/chosen": 0.01588086411356926, |
| "logits/rejected": 0.1419978141784668, |
| "logps/chosen": -42.120887756347656, |
| "logps/rejected": -65.12626647949219, |
| "loss": 0.3811, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.4491952657699585, |
| "rewards/margins": 0.9161649942398071, |
| "rewards/rejected": -1.3653602600097656, |
| "step": 2560 |
| }, |
| { |
| "epoch": 1.5296912114014252, |
| "grad_norm": 45.25, |
| "learning_rate": 6.94061757719715e-07, |
| "logits/chosen": 0.09590751677751541, |
| "logits/rejected": 0.20653869211673737, |
| "logps/chosen": -43.40055465698242, |
| "logps/rejected": -65.3685302734375, |
| "loss": 0.3946, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.42235833406448364, |
| "rewards/margins": 0.85853511095047, |
| "rewards/rejected": -1.2808934450149536, |
| "step": 2576 |
| }, |
| { |
| "epoch": 1.5391923990498813, |
| "grad_norm": 48.5, |
| "learning_rate": 6.921615201900237e-07, |
| "logits/chosen": 0.10696500539779663, |
| "logits/rejected": 0.1343913972377777, |
| "logps/chosen": -42.464046478271484, |
| "logps/rejected": -63.380943298339844, |
| "loss": 0.4106, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.44905325770378113, |
| "rewards/margins": 0.8345274925231934, |
| "rewards/rejected": -1.2835807800292969, |
| "step": 2592 |
| }, |
| { |
| "epoch": 1.5486935866983373, |
| "grad_norm": 57.0, |
| "learning_rate": 6.902612826603324e-07, |
| "logits/chosen": 0.04652204364538193, |
| "logits/rejected": 0.10454034060239792, |
| "logps/chosen": -40.00874328613281, |
| "logps/rejected": -60.814361572265625, |
| "loss": 0.3879, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.38040611147880554, |
| "rewards/margins": 0.8988234400749207, |
| "rewards/rejected": -1.279229760169983, |
| "step": 2608 |
| }, |
| { |
| "epoch": 1.5581947743467932, |
| "grad_norm": 51.25, |
| "learning_rate": 6.883610451306413e-07, |
| "logits/chosen": 0.11851513385772705, |
| "logits/rejected": 0.14123034477233887, |
| "logps/chosen": -41.09252166748047, |
| "logps/rejected": -64.00010681152344, |
| "loss": 0.3637, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3220548927783966, |
| "rewards/margins": 1.0083606243133545, |
| "rewards/rejected": -1.3304154872894287, |
| "step": 2624 |
| }, |
| { |
| "epoch": 1.5676959619952493, |
| "grad_norm": 50.75, |
| "learning_rate": 6.8646080760095e-07, |
| "logits/chosen": 0.016296017915010452, |
| "logits/rejected": 0.11665618419647217, |
| "logps/chosen": -39.934112548828125, |
| "logps/rejected": -61.658660888671875, |
| "loss": 0.3787, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.35257166624069214, |
| "rewards/margins": 0.9432335495948792, |
| "rewards/rejected": -1.2958052158355713, |
| "step": 2640 |
| }, |
| { |
| "epoch": 1.5771971496437054, |
| "grad_norm": 47.75, |
| "learning_rate": 6.845605700712589e-07, |
| "logits/chosen": 0.029370354488492012, |
| "logits/rejected": 0.13856028020381927, |
| "logps/chosen": -40.105552673339844, |
| "logps/rejected": -66.02179718017578, |
| "loss": 0.3523, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3618098795413971, |
| "rewards/margins": 1.0319128036499023, |
| "rewards/rejected": -1.3937227725982666, |
| "step": 2656 |
| }, |
| { |
| "epoch": 1.5866983372921615, |
| "grad_norm": 56.25, |
| "learning_rate": 6.826603325415676e-07, |
| "logits/chosen": 0.020885644480586052, |
| "logits/rejected": 0.17086170613765717, |
| "logps/chosen": -39.169334411621094, |
| "logps/rejected": -64.6575698852539, |
| "loss": 0.3559, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3273469805717468, |
| "rewards/margins": 0.9927965998649597, |
| "rewards/rejected": -1.320143461227417, |
| "step": 2672 |
| }, |
| { |
| "epoch": 1.5961995249406176, |
| "grad_norm": 51.75, |
| "learning_rate": 6.807600950118765e-07, |
| "logits/chosen": 0.03238772973418236, |
| "logits/rejected": 0.07314316928386688, |
| "logps/chosen": -42.21445083618164, |
| "logps/rejected": -64.58654022216797, |
| "loss": 0.378, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4178285002708435, |
| "rewards/margins": 0.9217057228088379, |
| "rewards/rejected": -1.3395342826843262, |
| "step": 2688 |
| }, |
| { |
| "epoch": 1.6057007125890737, |
| "grad_norm": 46.0, |
| "learning_rate": 6.788598574821852e-07, |
| "logits/chosen": 0.13759638369083405, |
| "logits/rejected": 0.21088193356990814, |
| "logps/chosen": -41.520233154296875, |
| "logps/rejected": -66.49211120605469, |
| "loss": 0.364, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.36787232756614685, |
| "rewards/margins": 0.9915460348129272, |
| "rewards/rejected": -1.359418511390686, |
| "step": 2704 |
| }, |
| { |
| "epoch": 1.6152019002375297, |
| "grad_norm": 43.5, |
| "learning_rate": 6.76959619952494e-07, |
| "logits/chosen": 0.03905269503593445, |
| "logits/rejected": 0.13661767542362213, |
| "logps/chosen": -38.46835708618164, |
| "logps/rejected": -60.601619720458984, |
| "loss": 0.3568, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.31080418825149536, |
| "rewards/margins": 0.9947383999824524, |
| "rewards/rejected": -1.3055424690246582, |
| "step": 2720 |
| }, |
| { |
| "epoch": 1.6247030878859858, |
| "grad_norm": 44.0, |
| "learning_rate": 6.750593824228028e-07, |
| "logits/chosen": 0.07454045116901398, |
| "logits/rejected": 0.17212067544460297, |
| "logps/chosen": -40.81611251831055, |
| "logps/rejected": -62.773468017578125, |
| "loss": 0.3971, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4132643938064575, |
| "rewards/margins": 0.8847667574882507, |
| "rewards/rejected": -1.2980310916900635, |
| "step": 2736 |
| }, |
| { |
| "epoch": 1.634204275534442, |
| "grad_norm": 38.75, |
| "learning_rate": 6.731591448931116e-07, |
| "logits/chosen": 0.0922178328037262, |
| "logits/rejected": 0.13588553667068481, |
| "logps/chosen": -39.945396423339844, |
| "logps/rejected": -61.7259407043457, |
| "loss": 0.3703, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.35344234108924866, |
| "rewards/margins": 0.9801003336906433, |
| "rewards/rejected": -1.3335425853729248, |
| "step": 2752 |
| }, |
| { |
| "epoch": 1.643705463182898, |
| "grad_norm": 55.25, |
| "learning_rate": 6.712589073634204e-07, |
| "logits/chosen": 0.09883704781532288, |
| "logits/rejected": 0.133531391620636, |
| "logps/chosen": -40.637210845947266, |
| "logps/rejected": -62.80479431152344, |
| "loss": 0.3582, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.34419822692871094, |
| "rewards/margins": 1.0021624565124512, |
| "rewards/rejected": -1.3463605642318726, |
| "step": 2768 |
| }, |
| { |
| "epoch": 1.6532066508313539, |
| "grad_norm": 35.75, |
| "learning_rate": 6.693586698337292e-07, |
| "logits/chosen": 0.06989182531833649, |
| "logits/rejected": 0.08873751759529114, |
| "logps/chosen": -37.793495178222656, |
| "logps/rejected": -60.03639602661133, |
| "loss": 0.3492, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.29584282636642456, |
| "rewards/margins": 1.0541434288024902, |
| "rewards/rejected": -1.3499860763549805, |
| "step": 2784 |
| }, |
| { |
| "epoch": 1.66270783847981, |
| "grad_norm": 44.5, |
| "learning_rate": 6.67458432304038e-07, |
| "logits/chosen": -0.0033259475603699684, |
| "logits/rejected": 0.14055626094341278, |
| "logps/chosen": -41.03327178955078, |
| "logps/rejected": -61.638668060302734, |
| "loss": 0.4027, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4033549129962921, |
| "rewards/margins": 0.8529766798019409, |
| "rewards/rejected": -1.2563316822052002, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.672209026128266, |
| "grad_norm": 55.25, |
| "learning_rate": 6.655581947743467e-07, |
| "logits/chosen": 0.054215628653764725, |
| "logits/rejected": 0.1540539413690567, |
| "logps/chosen": -42.16969299316406, |
| "logps/rejected": -63.94367218017578, |
| "loss": 0.3909, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4552299976348877, |
| "rewards/margins": 0.8886625170707703, |
| "rewards/rejected": -1.3438924551010132, |
| "step": 2816 |
| }, |
| { |
| "epoch": 1.6817102137767221, |
| "grad_norm": 62.0, |
| "learning_rate": 6.636579572446556e-07, |
| "logits/chosen": 0.054992396384477615, |
| "logits/rejected": 0.13705193996429443, |
| "logps/chosen": -40.24445724487305, |
| "logps/rejected": -60.925933837890625, |
| "loss": 0.3809, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.36651647090911865, |
| "rewards/margins": 0.9117545485496521, |
| "rewards/rejected": -1.278270959854126, |
| "step": 2832 |
| }, |
| { |
| "epoch": 1.691211401425178, |
| "grad_norm": 49.25, |
| "learning_rate": 6.617577197149643e-07, |
| "logits/chosen": 0.11320605874061584, |
| "logits/rejected": 0.17928367853164673, |
| "logps/chosen": -39.65250015258789, |
| "logps/rejected": -62.30231475830078, |
| "loss": 0.3734, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3596932888031006, |
| "rewards/margins": 0.9421689510345459, |
| "rewards/rejected": -1.3018622398376465, |
| "step": 2848 |
| }, |
| { |
| "epoch": 1.700712589073634, |
| "grad_norm": 40.0, |
| "learning_rate": 6.598574821852732e-07, |
| "logits/chosen": 0.025672361254692078, |
| "logits/rejected": 0.07097212225198746, |
| "logps/chosen": -41.138633728027344, |
| "logps/rejected": -63.86958694458008, |
| "loss": 0.3738, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.39380064606666565, |
| "rewards/margins": 0.9426943063735962, |
| "rewards/rejected": -1.3364949226379395, |
| "step": 2864 |
| }, |
| { |
| "epoch": 1.7102137767220902, |
| "grad_norm": 50.75, |
| "learning_rate": 6.579572446555819e-07, |
| "logits/chosen": 0.07025223225355148, |
| "logits/rejected": 0.06127552688121796, |
| "logps/chosen": -41.751319885253906, |
| "logps/rejected": -61.100685119628906, |
| "loss": 0.3783, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3228084146976471, |
| "rewards/margins": 0.9433329105377197, |
| "rewards/rejected": -1.2661415338516235, |
| "step": 2880 |
| }, |
| { |
| "epoch": 1.7197149643705463, |
| "grad_norm": 50.75, |
| "learning_rate": 6.560570071258908e-07, |
| "logits/chosen": -0.0122856879606843, |
| "logits/rejected": 0.08213039487600327, |
| "logps/chosen": -39.67725372314453, |
| "logps/rejected": -61.21652603149414, |
| "loss": 0.3817, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.39572060108184814, |
| "rewards/margins": 0.9105318188667297, |
| "rewards/rejected": -1.306252360343933, |
| "step": 2896 |
| }, |
| { |
| "epoch": 1.7292161520190024, |
| "grad_norm": 64.5, |
| "learning_rate": 6.541567695961995e-07, |
| "logits/chosen": 0.011255351826548576, |
| "logits/rejected": 0.10233234614133835, |
| "logps/chosen": -42.118900299072266, |
| "logps/rejected": -61.916690826416016, |
| "loss": 0.4019, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.41262125968933105, |
| "rewards/margins": 0.8827176094055176, |
| "rewards/rejected": -1.2953388690948486, |
| "step": 2912 |
| }, |
| { |
| "epoch": 1.7387173396674585, |
| "grad_norm": 44.25, |
| "learning_rate": 6.522565320665084e-07, |
| "logits/chosen": 0.07264780253171921, |
| "logits/rejected": 0.10011003911495209, |
| "logps/chosen": -41.43605422973633, |
| "logps/rejected": -59.60805892944336, |
| "loss": 0.406, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.3696948289871216, |
| "rewards/margins": 0.8629406690597534, |
| "rewards/rejected": -1.2326353788375854, |
| "step": 2928 |
| }, |
| { |
| "epoch": 1.7482185273159145, |
| "grad_norm": 41.0, |
| "learning_rate": 6.503562945368171e-07, |
| "logits/chosen": 0.05529964715242386, |
| "logits/rejected": 0.17115283012390137, |
| "logps/chosen": -40.79829788208008, |
| "logps/rejected": -65.05158996582031, |
| "loss": 0.3523, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.336140513420105, |
| "rewards/margins": 1.0045894384384155, |
| "rewards/rejected": -1.340729832649231, |
| "step": 2944 |
| }, |
| { |
| "epoch": 1.7577197149643706, |
| "grad_norm": 45.5, |
| "learning_rate": 6.484560570071259e-07, |
| "logits/chosen": 0.04629014432430267, |
| "logits/rejected": 0.1502920687198639, |
| "logps/chosen": -41.52702713012695, |
| "logps/rejected": -62.9970817565918, |
| "loss": 0.3898, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.4380848705768585, |
| "rewards/margins": 0.9207091331481934, |
| "rewards/rejected": -1.3587939739227295, |
| "step": 2960 |
| }, |
| { |
| "epoch": 1.7672209026128267, |
| "grad_norm": 44.5, |
| "learning_rate": 6.465558194774347e-07, |
| "logits/chosen": -0.011330801993608475, |
| "logits/rejected": 0.10742415487766266, |
| "logps/chosen": -40.159461975097656, |
| "logps/rejected": -64.75343322753906, |
| "loss": 0.3439, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3370579481124878, |
| "rewards/margins": 1.050633192062378, |
| "rewards/rejected": -1.3876910209655762, |
| "step": 2976 |
| }, |
| { |
| "epoch": 1.7767220902612828, |
| "grad_norm": 45.75, |
| "learning_rate": 6.446555819477434e-07, |
| "logits/chosen": 0.06578174233436584, |
| "logits/rejected": 0.1567022204399109, |
| "logps/chosen": -41.37073516845703, |
| "logps/rejected": -63.61731719970703, |
| "loss": 0.4027, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.42729830741882324, |
| "rewards/margins": 0.8533509373664856, |
| "rewards/rejected": -1.280649185180664, |
| "step": 2992 |
| }, |
| { |
| "epoch": 1.7862232779097387, |
| "grad_norm": 45.0, |
| "learning_rate": 6.427553444180523e-07, |
| "logits/chosen": 0.028342464938759804, |
| "logits/rejected": 0.17107811570167542, |
| "logps/chosen": -39.00218963623047, |
| "logps/rejected": -63.736061096191406, |
| "loss": 0.3565, |
| "rewards/accuracies": 0.9765625, |
| "rewards/chosen": -0.35766667127609253, |
| "rewards/margins": 0.9768067598342896, |
| "rewards/rejected": -1.3344734907150269, |
| "step": 3008 |
| }, |
| { |
| "epoch": 1.7957244655581948, |
| "grad_norm": 38.25, |
| "learning_rate": 6.40855106888361e-07, |
| "logits/chosen": 0.011551400646567345, |
| "logits/rejected": 0.13449110090732574, |
| "logps/chosen": -41.99809646606445, |
| "logps/rejected": -65.39181518554688, |
| "loss": 0.393, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4379570782184601, |
| "rewards/margins": 0.8674665689468384, |
| "rewards/rejected": -1.3054237365722656, |
| "step": 3024 |
| }, |
| { |
| "epoch": 1.8052256532066508, |
| "grad_norm": 63.5, |
| "learning_rate": 6.389548693586699e-07, |
| "logits/chosen": 0.05890597403049469, |
| "logits/rejected": 0.1898549646139145, |
| "logps/chosen": -39.68592834472656, |
| "logps/rejected": -65.33939361572266, |
| "loss": 0.3571, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.34970682859420776, |
| "rewards/margins": 1.0283163785934448, |
| "rewards/rejected": -1.3780232667922974, |
| "step": 3040 |
| }, |
| { |
| "epoch": 1.814726840855107, |
| "grad_norm": 58.25, |
| "learning_rate": 6.370546318289785e-07, |
| "logits/chosen": 0.019725359976291656, |
| "logits/rejected": 0.15305927395820618, |
| "logps/chosen": -42.078033447265625, |
| "logps/rejected": -61.74312973022461, |
| "loss": 0.4095, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.4093819856643677, |
| "rewards/margins": 0.836819052696228, |
| "rewards/rejected": -1.2462011575698853, |
| "step": 3056 |
| }, |
| { |
| "epoch": 1.8242280285035628, |
| "grad_norm": 53.0, |
| "learning_rate": 6.351543942992874e-07, |
| "logits/chosen": 0.057205233722925186, |
| "logits/rejected": 0.1036025658249855, |
| "logps/chosen": -42.514862060546875, |
| "logps/rejected": -60.93022155761719, |
| "loss": 0.4002, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.3770271837711334, |
| "rewards/margins": 0.8727684617042542, |
| "rewards/rejected": -1.24979567527771, |
| "step": 3072 |
| }, |
| { |
| "epoch": 1.833729216152019, |
| "grad_norm": 46.0, |
| "learning_rate": 6.332541567695961e-07, |
| "logits/chosen": 0.032310646027326584, |
| "logits/rejected": 0.1559012234210968, |
| "logps/chosen": -43.822120666503906, |
| "logps/rejected": -64.49073791503906, |
| "loss": 0.4014, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.49134525656700134, |
| "rewards/margins": 0.861031174659729, |
| "rewards/rejected": -1.3523763418197632, |
| "step": 3088 |
| }, |
| { |
| "epoch": 1.843230403800475, |
| "grad_norm": 61.75, |
| "learning_rate": 6.31353919239905e-07, |
| "logits/chosen": 0.004301354289054871, |
| "logits/rejected": 0.08244706690311432, |
| "logps/chosen": -38.925811767578125, |
| "logps/rejected": -63.42317199707031, |
| "loss": 0.3441, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.34742915630340576, |
| "rewards/margins": 1.040870189666748, |
| "rewards/rejected": -1.3882992267608643, |
| "step": 3104 |
| }, |
| { |
| "epoch": 1.852731591448931, |
| "grad_norm": 54.25, |
| "learning_rate": 6.294536817102137e-07, |
| "logits/chosen": -0.00210411474108696, |
| "logits/rejected": 0.1236819252371788, |
| "logps/chosen": -38.81071090698242, |
| "logps/rejected": -61.1712646484375, |
| "loss": 0.3845, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.37758293747901917, |
| "rewards/margins": 0.8995540738105774, |
| "rewards/rejected": -1.2771369218826294, |
| "step": 3120 |
| }, |
| { |
| "epoch": 1.8622327790973872, |
| "grad_norm": 50.0, |
| "learning_rate": 6.275534441805226e-07, |
| "logits/chosen": 0.04914906993508339, |
| "logits/rejected": 0.13415980339050293, |
| "logps/chosen": -41.947120666503906, |
| "logps/rejected": -61.855255126953125, |
| "loss": 0.3982, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4452968239784241, |
| "rewards/margins": 0.8472458720207214, |
| "rewards/rejected": -1.2925426959991455, |
| "step": 3136 |
| }, |
| { |
| "epoch": 1.8717339667458432, |
| "grad_norm": 46.25, |
| "learning_rate": 6.256532066508313e-07, |
| "logits/chosen": 0.0254144836217165, |
| "logits/rejected": 0.12364037334918976, |
| "logps/chosen": -41.690284729003906, |
| "logps/rejected": -62.46638870239258, |
| "loss": 0.3497, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3699144124984741, |
| "rewards/margins": 1.013372540473938, |
| "rewards/rejected": -1.383286952972412, |
| "step": 3152 |
| }, |
| { |
| "epoch": 1.8812351543942993, |
| "grad_norm": 46.25, |
| "learning_rate": 6.2375296912114e-07, |
| "logits/chosen": 0.03663626313209534, |
| "logits/rejected": 0.14503881335258484, |
| "logps/chosen": -41.162559509277344, |
| "logps/rejected": -64.2937240600586, |
| "loss": 0.355, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.39480462670326233, |
| "rewards/margins": 1.0223582983016968, |
| "rewards/rejected": -1.4171628952026367, |
| "step": 3168 |
| }, |
| { |
| "epoch": 1.8907363420427554, |
| "grad_norm": 40.0, |
| "learning_rate": 6.218527315914489e-07, |
| "logits/chosen": 0.0167838204652071, |
| "logits/rejected": 0.09799753874540329, |
| "logps/chosen": -39.695152282714844, |
| "logps/rejected": -64.00861358642578, |
| "loss": 0.3409, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3396395742893219, |
| "rewards/margins": 1.0993303060531616, |
| "rewards/rejected": -1.4389699697494507, |
| "step": 3184 |
| }, |
| { |
| "epoch": 1.9002375296912115, |
| "grad_norm": 55.5, |
| "learning_rate": 6.199524940617576e-07, |
| "logits/chosen": 0.04460003226995468, |
| "logits/rejected": 0.14700554311275482, |
| "logps/chosen": -44.2281494140625, |
| "logps/rejected": -61.84652328491211, |
| "loss": 0.4352, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.5021332502365112, |
| "rewards/margins": 0.7495644688606262, |
| "rewards/rejected": -1.2516977787017822, |
| "step": 3200 |
| }, |
| { |
| "epoch": 1.9097387173396676, |
| "grad_norm": 32.5, |
| "learning_rate": 6.180522565320665e-07, |
| "logits/chosen": 0.08530572056770325, |
| "logits/rejected": 0.15016531944274902, |
| "logps/chosen": -41.49757385253906, |
| "logps/rejected": -67.6200942993164, |
| "loss": 0.3317, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.38793495297431946, |
| "rewards/margins": 1.099698543548584, |
| "rewards/rejected": -1.487633466720581, |
| "step": 3216 |
| }, |
| { |
| "epoch": 1.9192399049881235, |
| "grad_norm": 61.0, |
| "learning_rate": 6.161520190023752e-07, |
| "logits/chosen": 0.029300598427653313, |
| "logits/rejected": 0.13747388124465942, |
| "logps/chosen": -41.173736572265625, |
| "logps/rejected": -63.320472717285156, |
| "loss": 0.4127, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.4380740225315094, |
| "rewards/margins": 0.867578387260437, |
| "rewards/rejected": -1.3056524991989136, |
| "step": 3232 |
| }, |
| { |
| "epoch": 1.9287410926365796, |
| "grad_norm": 44.75, |
| "learning_rate": 6.142517814726841e-07, |
| "logits/chosen": 0.08258551359176636, |
| "logits/rejected": 0.16131961345672607, |
| "logps/chosen": -42.77522277832031, |
| "logps/rejected": -64.70687866210938, |
| "loss": 0.395, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4444977045059204, |
| "rewards/margins": 0.8981250524520874, |
| "rewards/rejected": -1.3426228761672974, |
| "step": 3248 |
| }, |
| { |
| "epoch": 1.9382422802850356, |
| "grad_norm": 67.5, |
| "learning_rate": 6.123515439429928e-07, |
| "logits/chosen": 0.044392723590135574, |
| "logits/rejected": 0.17256534099578857, |
| "logps/chosen": -40.32421875, |
| "logps/rejected": -63.82672119140625, |
| "loss": 0.3611, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.40335986018180847, |
| "rewards/margins": 1.0055795907974243, |
| "rewards/rejected": -1.4089393615722656, |
| "step": 3264 |
| }, |
| { |
| "epoch": 1.9477434679334917, |
| "grad_norm": 49.25, |
| "learning_rate": 6.104513064133017e-07, |
| "logits/chosen": 0.1122560128569603, |
| "logits/rejected": 0.1721959263086319, |
| "logps/chosen": -41.03020477294922, |
| "logps/rejected": -62.24818420410156, |
| "loss": 0.4023, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.41969960927963257, |
| "rewards/margins": 0.8544268608093262, |
| "rewards/rejected": -1.2741265296936035, |
| "step": 3280 |
| }, |
| { |
| "epoch": 1.9572446555819476, |
| "grad_norm": 54.5, |
| "learning_rate": 6.085510688836104e-07, |
| "logits/chosen": 0.03646089881658554, |
| "logits/rejected": 0.1658436357975006, |
| "logps/chosen": -41.14552307128906, |
| "logps/rejected": -63.396366119384766, |
| "loss": 0.3697, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4058351218700409, |
| "rewards/margins": 0.9656636714935303, |
| "rewards/rejected": -1.3714988231658936, |
| "step": 3296 |
| }, |
| { |
| "epoch": 1.9667458432304037, |
| "grad_norm": 46.75, |
| "learning_rate": 6.066508313539193e-07, |
| "logits/chosen": 0.03148533031344414, |
| "logits/rejected": 0.14277218282222748, |
| "logps/chosen": -42.05448913574219, |
| "logps/rejected": -65.2642593383789, |
| "loss": 0.3928, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4272368550300598, |
| "rewards/margins": 0.8805699348449707, |
| "rewards/rejected": -1.3078068494796753, |
| "step": 3312 |
| }, |
| { |
| "epoch": 1.9762470308788598, |
| "grad_norm": 41.75, |
| "learning_rate": 6.04750593824228e-07, |
| "logits/chosen": 0.06896740198135376, |
| "logits/rejected": 0.10403262823820114, |
| "logps/chosen": -40.34957504272461, |
| "logps/rejected": -61.979915618896484, |
| "loss": 0.3788, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3596211373806, |
| "rewards/margins": 0.9470376968383789, |
| "rewards/rejected": -1.3066588640213013, |
| "step": 3328 |
| }, |
| { |
| "epoch": 1.9857482185273159, |
| "grad_norm": 55.5, |
| "learning_rate": 6.028503562945369e-07, |
| "logits/chosen": -0.00018313713371753693, |
| "logits/rejected": 0.09493206441402435, |
| "logps/chosen": -40.674354553222656, |
| "logps/rejected": -62.300506591796875, |
| "loss": 0.3676, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.35612305998802185, |
| "rewards/margins": 0.9978376626968384, |
| "rewards/rejected": -1.3539607524871826, |
| "step": 3344 |
| }, |
| { |
| "epoch": 1.995249406175772, |
| "grad_norm": 59.75, |
| "learning_rate": 6.009501187648456e-07, |
| "logits/chosen": -0.004904988221824169, |
| "logits/rejected": 0.09877481311559677, |
| "logps/chosen": -40.2232666015625, |
| "logps/rejected": -61.96391296386719, |
| "loss": 0.3655, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3332652151584625, |
| "rewards/margins": 1.0086623430252075, |
| "rewards/rejected": -1.3419275283813477, |
| "step": 3360 |
| }, |
| { |
| "epoch": 2.004750593824228, |
| "grad_norm": 58.5, |
| "learning_rate": 5.990498812351543e-07, |
| "logits/chosen": 0.06698215007781982, |
| "logits/rejected": 0.17540045082569122, |
| "logps/chosen": -43.14054870605469, |
| "logps/rejected": -63.408565521240234, |
| "loss": 0.4218, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4646109938621521, |
| "rewards/margins": 0.7780359387397766, |
| "rewards/rejected": -1.2426469326019287, |
| "step": 3376 |
| }, |
| { |
| "epoch": 2.014251781472684, |
| "grad_norm": 44.25, |
| "learning_rate": 5.971496437054632e-07, |
| "logits/chosen": 0.09281051158905029, |
| "logits/rejected": 0.1449931114912033, |
| "logps/chosen": -41.34113311767578, |
| "logps/rejected": -64.54020690917969, |
| "loss": 0.3712, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.37270694971084595, |
| "rewards/margins": 0.932529628276825, |
| "rewards/rejected": -1.305236577987671, |
| "step": 3392 |
| }, |
| { |
| "epoch": 2.02375296912114, |
| "grad_norm": 43.75, |
| "learning_rate": 5.952494061757719e-07, |
| "logits/chosen": 0.03906689211726189, |
| "logits/rejected": 0.14161323010921478, |
| "logps/chosen": -39.38663101196289, |
| "logps/rejected": -63.778724670410156, |
| "loss": 0.3588, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3333834707736969, |
| "rewards/margins": 1.012413501739502, |
| "rewards/rejected": -1.3457969427108765, |
| "step": 3408 |
| }, |
| { |
| "epoch": 2.0332541567695963, |
| "grad_norm": 35.5, |
| "learning_rate": 5.933491686460808e-07, |
| "logits/chosen": 0.009185846894979477, |
| "logits/rejected": 0.027248330414295197, |
| "logps/chosen": -40.65418243408203, |
| "logps/rejected": -59.860408782958984, |
| "loss": 0.391, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.4065134525299072, |
| "rewards/margins": 0.879849910736084, |
| "rewards/rejected": -1.2863633632659912, |
| "step": 3424 |
| }, |
| { |
| "epoch": 2.0427553444180524, |
| "grad_norm": 50.25, |
| "learning_rate": 5.914489311163895e-07, |
| "logits/chosen": -0.04159889370203018, |
| "logits/rejected": 0.12180589139461517, |
| "logps/chosen": -40.518218994140625, |
| "logps/rejected": -63.04229736328125, |
| "loss": 0.3689, |
| "rewards/accuracies": 0.9921875, |
| "rewards/chosen": -0.39485669136047363, |
| "rewards/margins": 0.9171349406242371, |
| "rewards/rejected": -1.311991572380066, |
| "step": 3440 |
| }, |
| { |
| "epoch": 2.0522565320665085, |
| "grad_norm": 47.0, |
| "learning_rate": 5.895486935866984e-07, |
| "logits/chosen": 0.01238052174448967, |
| "logits/rejected": 0.09721352905035019, |
| "logps/chosen": -42.85576248168945, |
| "logps/rejected": -65.13483428955078, |
| "loss": 0.39, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.43811485171318054, |
| "rewards/margins": 0.9170427918434143, |
| "rewards/rejected": -1.3551576137542725, |
| "step": 3456 |
| }, |
| { |
| "epoch": 2.0617577197149646, |
| "grad_norm": 43.0, |
| "learning_rate": 5.876484560570071e-07, |
| "logits/chosen": 0.00015027448534965515, |
| "logits/rejected": 0.10659853368997574, |
| "logps/chosen": -40.47695541381836, |
| "logps/rejected": -62.642822265625, |
| "loss": 0.3501, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.37582921981811523, |
| "rewards/margins": 1.0362910032272339, |
| "rewards/rejected": -1.4121201038360596, |
| "step": 3472 |
| }, |
| { |
| "epoch": 2.07125890736342, |
| "grad_norm": 44.0, |
| "learning_rate": 5.85748218527316e-07, |
| "logits/chosen": -0.007269053254276514, |
| "logits/rejected": 0.11244423687458038, |
| "logps/chosen": -40.462440490722656, |
| "logps/rejected": -65.11763000488281, |
| "loss": 0.3694, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3953917920589447, |
| "rewards/margins": 1.003770351409912, |
| "rewards/rejected": -1.3991621732711792, |
| "step": 3488 |
| }, |
| { |
| "epoch": 2.0807600950118763, |
| "grad_norm": 40.25, |
| "learning_rate": 5.838479809976247e-07, |
| "logits/chosen": 0.007935550063848495, |
| "logits/rejected": 0.1055232435464859, |
| "logps/chosen": -39.9696159362793, |
| "logps/rejected": -65.34225463867188, |
| "loss": 0.3638, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.38794174790382385, |
| "rewards/margins": 0.9854438304901123, |
| "rewards/rejected": -1.3733854293823242, |
| "step": 3504 |
| }, |
| { |
| "epoch": 2.0902612826603324, |
| "grad_norm": 49.0, |
| "learning_rate": 5.819477434679335e-07, |
| "logits/chosen": -0.002856359351426363, |
| "logits/rejected": 0.12148334830999374, |
| "logps/chosen": -40.93573760986328, |
| "logps/rejected": -62.00894546508789, |
| "loss": 0.4115, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.42675450444221497, |
| "rewards/margins": 0.8534324765205383, |
| "rewards/rejected": -1.2801870107650757, |
| "step": 3520 |
| }, |
| { |
| "epoch": 2.0997624703087885, |
| "grad_norm": 52.75, |
| "learning_rate": 5.800475059382422e-07, |
| "logits/chosen": 0.019883442670106888, |
| "logits/rejected": 0.12564308941364288, |
| "logps/chosen": -42.050662994384766, |
| "logps/rejected": -61.40731430053711, |
| "loss": 0.3947, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.45309561491012573, |
| "rewards/margins": 0.8529905080795288, |
| "rewards/rejected": -1.3060861825942993, |
| "step": 3536 |
| }, |
| { |
| "epoch": 2.1092636579572446, |
| "grad_norm": 44.25, |
| "learning_rate": 5.78147268408551e-07, |
| "logits/chosen": -0.0019861001055687666, |
| "logits/rejected": 0.13409556448459625, |
| "logps/chosen": -42.92340850830078, |
| "logps/rejected": -66.3511734008789, |
| "loss": 0.3933, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.43070507049560547, |
| "rewards/margins": 0.907360315322876, |
| "rewards/rejected": -1.3380653858184814, |
| "step": 3552 |
| }, |
| { |
| "epoch": 2.1187648456057007, |
| "grad_norm": 48.25, |
| "learning_rate": 5.762470308788598e-07, |
| "logits/chosen": 0.13548046350479126, |
| "logits/rejected": 0.11156810820102692, |
| "logps/chosen": -41.66028594970703, |
| "logps/rejected": -62.116031646728516, |
| "loss": 0.3611, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3860835134983063, |
| "rewards/margins": 0.9740406274795532, |
| "rewards/rejected": -1.360124111175537, |
| "step": 3568 |
| }, |
| { |
| "epoch": 2.1282660332541568, |
| "grad_norm": 49.25, |
| "learning_rate": 5.743467933491685e-07, |
| "logits/chosen": 0.022612586617469788, |
| "logits/rejected": 0.13782542943954468, |
| "logps/chosen": -39.91732406616211, |
| "logps/rejected": -64.74801635742188, |
| "loss": 0.3778, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.38275399804115295, |
| "rewards/margins": 0.9404458999633789, |
| "rewards/rejected": -1.3231998682022095, |
| "step": 3584 |
| }, |
| { |
| "epoch": 2.137767220902613, |
| "grad_norm": 46.5, |
| "learning_rate": 5.724465558194774e-07, |
| "logits/chosen": 0.06513424217700958, |
| "logits/rejected": 0.1354922503232956, |
| "logps/chosen": -40.324771881103516, |
| "logps/rejected": -62.070045471191406, |
| "loss": 0.3937, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4051070213317871, |
| "rewards/margins": 0.8988076448440552, |
| "rewards/rejected": -1.3039146661758423, |
| "step": 3600 |
| }, |
| { |
| "epoch": 2.147268408551069, |
| "grad_norm": 53.75, |
| "learning_rate": 5.705463182897861e-07, |
| "logits/chosen": 0.09106743335723877, |
| "logits/rejected": 0.22678810358047485, |
| "logps/chosen": -44.392295837402344, |
| "logps/rejected": -65.7562255859375, |
| "loss": 0.3996, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.5041301846504211, |
| "rewards/margins": 0.863517701625824, |
| "rewards/rejected": -1.3676478862762451, |
| "step": 3616 |
| }, |
| { |
| "epoch": 2.156769596199525, |
| "grad_norm": 49.5, |
| "learning_rate": 5.68646080760095e-07, |
| "logits/chosen": 0.045438431203365326, |
| "logits/rejected": 0.09103736281394958, |
| "logps/chosen": -38.36294174194336, |
| "logps/rejected": -58.81281661987305, |
| "loss": 0.3819, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.3600079417228699, |
| "rewards/margins": 0.9226837754249573, |
| "rewards/rejected": -1.2826919555664062, |
| "step": 3632 |
| }, |
| { |
| "epoch": 2.166270783847981, |
| "grad_norm": 46.0, |
| "learning_rate": 5.667458432304037e-07, |
| "logits/chosen": 0.13296620547771454, |
| "logits/rejected": 0.18886858224868774, |
| "logps/chosen": -40.72220230102539, |
| "logps/rejected": -64.00652313232422, |
| "loss": 0.3879, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.399859756231308, |
| "rewards/margins": 0.9217446446418762, |
| "rewards/rejected": -1.3216043710708618, |
| "step": 3648 |
| }, |
| { |
| "epoch": 2.175771971496437, |
| "grad_norm": 45.5, |
| "learning_rate": 5.648456057007126e-07, |
| "logits/chosen": 0.02620471641421318, |
| "logits/rejected": 0.11671482771635056, |
| "logps/chosen": -40.500267028808594, |
| "logps/rejected": -62.4256591796875, |
| "loss": 0.3847, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.39798420667648315, |
| "rewards/margins": 0.910751223564148, |
| "rewards/rejected": -1.3087353706359863, |
| "step": 3664 |
| }, |
| { |
| "epoch": 2.1852731591448933, |
| "grad_norm": 38.5, |
| "learning_rate": 5.629453681710213e-07, |
| "logits/chosen": 0.0530809611082077, |
| "logits/rejected": 0.16400957107543945, |
| "logps/chosen": -42.896827697753906, |
| "logps/rejected": -63.195579528808594, |
| "loss": 0.4002, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.394864559173584, |
| "rewards/margins": 0.8955377340316772, |
| "rewards/rejected": -1.2904024124145508, |
| "step": 3680 |
| }, |
| { |
| "epoch": 2.1947743467933494, |
| "grad_norm": 50.5, |
| "learning_rate": 5.610451306413302e-07, |
| "logits/chosen": 0.040826160460710526, |
| "logits/rejected": 0.10787297785282135, |
| "logps/chosen": -40.30386734008789, |
| "logps/rejected": -64.05851745605469, |
| "loss": 0.3583, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3300076425075531, |
| "rewards/margins": 1.0026148557662964, |
| "rewards/rejected": -1.3326225280761719, |
| "step": 3696 |
| }, |
| { |
| "epoch": 2.204275534441805, |
| "grad_norm": 58.0, |
| "learning_rate": 5.591448931116389e-07, |
| "logits/chosen": 0.06701106578111649, |
| "logits/rejected": 0.13853205740451813, |
| "logps/chosen": -42.53071594238281, |
| "logps/rejected": -62.566688537597656, |
| "loss": 0.3845, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.41142404079437256, |
| "rewards/margins": 0.9055157899856567, |
| "rewards/rejected": -1.3169398307800293, |
| "step": 3712 |
| }, |
| { |
| "epoch": 2.213776722090261, |
| "grad_norm": 57.0, |
| "learning_rate": 5.572446555819477e-07, |
| "logits/chosen": 0.01710950955748558, |
| "logits/rejected": 0.095205157995224, |
| "logps/chosen": -40.496944427490234, |
| "logps/rejected": -61.76165771484375, |
| "loss": 0.3853, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3820299506187439, |
| "rewards/margins": 0.9107472896575928, |
| "rewards/rejected": -1.292777180671692, |
| "step": 3728 |
| }, |
| { |
| "epoch": 2.223277909738717, |
| "grad_norm": 56.0, |
| "learning_rate": 5.553444180522565e-07, |
| "logits/chosen": 0.08512625843286514, |
| "logits/rejected": 0.16064119338989258, |
| "logps/chosen": -41.3702278137207, |
| "logps/rejected": -66.45726013183594, |
| "loss": 0.3534, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3733311891555786, |
| "rewards/margins": 1.0442577600479126, |
| "rewards/rejected": -1.4175888299942017, |
| "step": 3744 |
| }, |
| { |
| "epoch": 2.2327790973871733, |
| "grad_norm": 60.75, |
| "learning_rate": 5.534441805225653e-07, |
| "logits/chosen": 0.09103821218013763, |
| "logits/rejected": 0.13160774111747742, |
| "logps/chosen": -43.8742790222168, |
| "logps/rejected": -65.8563003540039, |
| "loss": 0.3973, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.4512379467487335, |
| "rewards/margins": 0.9139763712882996, |
| "rewards/rejected": -1.365214228630066, |
| "step": 3760 |
| }, |
| { |
| "epoch": 2.2422802850356294, |
| "grad_norm": 44.75, |
| "learning_rate": 5.515439429928741e-07, |
| "logits/chosen": -0.0005833394825458527, |
| "logits/rejected": 0.10005127638578415, |
| "logps/chosen": -42.24885559082031, |
| "logps/rejected": -64.85872650146484, |
| "loss": 0.3583, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.42689433693885803, |
| "rewards/margins": 0.9682207703590393, |
| "rewards/rejected": -1.3951151371002197, |
| "step": 3776 |
| }, |
| { |
| "epoch": 2.2517814726840855, |
| "grad_norm": 53.5, |
| "learning_rate": 5.496437054631829e-07, |
| "logits/chosen": 0.05875023454427719, |
| "logits/rejected": 0.13941840827465057, |
| "logps/chosen": -42.663665771484375, |
| "logps/rejected": -63.30080795288086, |
| "loss": 0.3864, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.4756395220756531, |
| "rewards/margins": 0.941820502281189, |
| "rewards/rejected": -1.4174600839614868, |
| "step": 3792 |
| }, |
| { |
| "epoch": 2.2612826603325415, |
| "grad_norm": 39.25, |
| "learning_rate": 5.477434679334917e-07, |
| "logits/chosen": 0.02674500085413456, |
| "logits/rejected": 0.17680123448371887, |
| "logps/chosen": -38.55767822265625, |
| "logps/rejected": -62.6890754699707, |
| "loss": 0.3493, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.2811494767665863, |
| "rewards/margins": 1.024692416191101, |
| "rewards/rejected": -1.3058419227600098, |
| "step": 3808 |
| }, |
| { |
| "epoch": 2.2707838479809976, |
| "grad_norm": 48.25, |
| "learning_rate": 5.458432304038004e-07, |
| "logits/chosen": 0.009942879900336266, |
| "logits/rejected": 0.1574607640504837, |
| "logps/chosen": -41.19435501098633, |
| "logps/rejected": -63.181053161621094, |
| "loss": 0.3878, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.42380547523498535, |
| "rewards/margins": 0.9084890484809875, |
| "rewards/rejected": -1.3322944641113281, |
| "step": 3824 |
| }, |
| { |
| "epoch": 2.2802850356294537, |
| "grad_norm": 49.25, |
| "learning_rate": 5.439429928741093e-07, |
| "logits/chosen": 0.06854081898927689, |
| "logits/rejected": 0.1759743094444275, |
| "logps/chosen": -40.84571075439453, |
| "logps/rejected": -64.14063262939453, |
| "loss": 0.4032, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.4499894678592682, |
| "rewards/margins": 0.8732954263687134, |
| "rewards/rejected": -1.3232848644256592, |
| "step": 3840 |
| }, |
| { |
| "epoch": 2.28978622327791, |
| "grad_norm": 58.5, |
| "learning_rate": 5.42042755344418e-07, |
| "logits/chosen": 0.019341815263032913, |
| "logits/rejected": 0.16274170577526093, |
| "logps/chosen": -41.712154388427734, |
| "logps/rejected": -63.17491149902344, |
| "loss": 0.3988, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4367818534374237, |
| "rewards/margins": 0.8796077370643616, |
| "rewards/rejected": -1.3163896799087524, |
| "step": 3856 |
| }, |
| { |
| "epoch": 2.299287410926366, |
| "grad_norm": 45.5, |
| "learning_rate": 5.401425178147269e-07, |
| "logits/chosen": 0.0484287403523922, |
| "logits/rejected": 0.15023472905158997, |
| "logps/chosen": -40.65409851074219, |
| "logps/rejected": -64.70632934570312, |
| "loss": 0.3934, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.3949016332626343, |
| "rewards/margins": 0.9213113784790039, |
| "rewards/rejected": -1.3162128925323486, |
| "step": 3872 |
| }, |
| { |
| "epoch": 2.308788598574822, |
| "grad_norm": 45.5, |
| "learning_rate": 5.382422802850356e-07, |
| "logits/chosen": 0.0558081790804863, |
| "logits/rejected": 0.14007531106472015, |
| "logps/chosen": -40.67841339111328, |
| "logps/rejected": -63.90584945678711, |
| "loss": 0.3557, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.40134915709495544, |
| "rewards/margins": 0.9840033054351807, |
| "rewards/rejected": -1.385352373123169, |
| "step": 3888 |
| }, |
| { |
| "epoch": 2.318289786223278, |
| "grad_norm": 43.5, |
| "learning_rate": 5.363420427553445e-07, |
| "logits/chosen": 0.06617691367864609, |
| "logits/rejected": 0.13138173520565033, |
| "logps/chosen": -42.65940856933594, |
| "logps/rejected": -63.277469635009766, |
| "loss": 0.3987, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.452262282371521, |
| "rewards/margins": 0.8770920634269714, |
| "rewards/rejected": -1.3293542861938477, |
| "step": 3904 |
| }, |
| { |
| "epoch": 2.3277909738717337, |
| "grad_norm": 49.75, |
| "learning_rate": 5.344418052256532e-07, |
| "logits/chosen": 0.0712217167019844, |
| "logits/rejected": 0.11869163066148758, |
| "logps/chosen": -42.18745803833008, |
| "logps/rejected": -62.14987564086914, |
| "loss": 0.387, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.3666439354419708, |
| "rewards/margins": 0.9148606061935425, |
| "rewards/rejected": -1.281504511833191, |
| "step": 3920 |
| }, |
| { |
| "epoch": 2.33729216152019, |
| "grad_norm": 41.25, |
| "learning_rate": 5.32541567695962e-07, |
| "logits/chosen": 0.052989356219768524, |
| "logits/rejected": 0.1383398026227951, |
| "logps/chosen": -39.75597381591797, |
| "logps/rejected": -61.32164764404297, |
| "loss": 0.3859, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.3883766829967499, |
| "rewards/margins": 0.9118002653121948, |
| "rewards/rejected": -1.3001768589019775, |
| "step": 3936 |
| }, |
| { |
| "epoch": 2.346793349168646, |
| "grad_norm": 45.25, |
| "learning_rate": 5.306413301662708e-07, |
| "logits/chosen": 0.07647877931594849, |
| "logits/rejected": 0.12356515228748322, |
| "logps/chosen": -41.482666015625, |
| "logps/rejected": -64.74012756347656, |
| "loss": 0.4029, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.4555918574333191, |
| "rewards/margins": 0.8827881217002869, |
| "rewards/rejected": -1.338379979133606, |
| "step": 3952 |
| }, |
| { |
| "epoch": 2.356294536817102, |
| "grad_norm": 49.0, |
| "learning_rate": 5.287410926365796e-07, |
| "logits/chosen": 0.025606969371438026, |
| "logits/rejected": 0.11031323671340942, |
| "logps/chosen": -40.05935287475586, |
| "logps/rejected": -61.486900329589844, |
| "loss": 0.367, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.34517067670822144, |
| "rewards/margins": 1.008548378944397, |
| "rewards/rejected": -1.3537191152572632, |
| "step": 3968 |
| }, |
| { |
| "epoch": 2.365795724465558, |
| "grad_norm": 52.75, |
| "learning_rate": 5.268408551068883e-07, |
| "logits/chosen": 0.020188216120004654, |
| "logits/rejected": 0.07814847677946091, |
| "logps/chosen": -41.763946533203125, |
| "logps/rejected": -62.15244674682617, |
| "loss": 0.3841, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4187239706516266, |
| "rewards/margins": 0.9112571477890015, |
| "rewards/rejected": -1.3299810886383057, |
| "step": 3984 |
| }, |
| { |
| "epoch": 2.375296912114014, |
| "grad_norm": 53.5, |
| "learning_rate": 5.24940617577197e-07, |
| "logits/chosen": -0.013901928439736366, |
| "logits/rejected": 0.1482914239168167, |
| "logps/chosen": -40.2778205871582, |
| "logps/rejected": -64.91542053222656, |
| "loss": 0.3821, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.42066463828086853, |
| "rewards/margins": 0.9068177938461304, |
| "rewards/rejected": -1.3274823427200317, |
| "step": 4000 |
| }, |
| { |
| "epoch": 2.3847980997624703, |
| "grad_norm": 60.75, |
| "learning_rate": 5.230403800475059e-07, |
| "logits/chosen": 0.07787059247493744, |
| "logits/rejected": 0.17709147930145264, |
| "logps/chosen": -42.24334716796875, |
| "logps/rejected": -63.37432861328125, |
| "loss": 0.3924, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3965498208999634, |
| "rewards/margins": 0.8994001150131226, |
| "rewards/rejected": -1.295949935913086, |
| "step": 4016 |
| }, |
| { |
| "epoch": 2.3942992874109263, |
| "grad_norm": 47.25, |
| "learning_rate": 5.211401425178146e-07, |
| "logits/chosen": 0.06299006193876266, |
| "logits/rejected": 0.14556781947612762, |
| "logps/chosen": -42.120365142822266, |
| "logps/rejected": -64.56780242919922, |
| "loss": 0.3795, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4348159730434418, |
| "rewards/margins": 0.9128706455230713, |
| "rewards/rejected": -1.3476866483688354, |
| "step": 4032 |
| }, |
| { |
| "epoch": 2.4038004750593824, |
| "grad_norm": 44.25, |
| "learning_rate": 5.192399049881235e-07, |
| "logits/chosen": 0.086119145154953, |
| "logits/rejected": 0.1857333481311798, |
| "logps/chosen": -40.12603759765625, |
| "logps/rejected": -61.00786590576172, |
| "loss": 0.4011, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.41915708780288696, |
| "rewards/margins": 0.8710625767707825, |
| "rewards/rejected": -1.2902196645736694, |
| "step": 4048 |
| }, |
| { |
| "epoch": 2.4133016627078385, |
| "grad_norm": 37.75, |
| "learning_rate": 5.173396674584322e-07, |
| "logits/chosen": 0.027670690789818764, |
| "logits/rejected": 0.1215248703956604, |
| "logps/chosen": -40.98219299316406, |
| "logps/rejected": -62.411865234375, |
| "loss": 0.3782, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.3689427971839905, |
| "rewards/margins": 0.9613568186759949, |
| "rewards/rejected": -1.3302994966506958, |
| "step": 4064 |
| }, |
| { |
| "epoch": 2.4228028503562946, |
| "grad_norm": 45.5, |
| "learning_rate": 5.154394299287411e-07, |
| "logits/chosen": 0.03985697776079178, |
| "logits/rejected": 0.12585824728012085, |
| "logps/chosen": -42.074466705322266, |
| "logps/rejected": -64.21247100830078, |
| "loss": 0.3755, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3887619376182556, |
| "rewards/margins": 0.9904804229736328, |
| "rewards/rejected": -1.3792424201965332, |
| "step": 4080 |
| }, |
| { |
| "epoch": 2.4323040380047507, |
| "grad_norm": 43.75, |
| "learning_rate": 5.135391923990498e-07, |
| "logits/chosen": 0.05401856452226639, |
| "logits/rejected": 0.10051050782203674, |
| "logps/chosen": -41.186561584472656, |
| "logps/rejected": -63.16551971435547, |
| "loss": 0.4086, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.4640354812145233, |
| "rewards/margins": 0.8580038547515869, |
| "rewards/rejected": -1.3220393657684326, |
| "step": 4096 |
| }, |
| { |
| "epoch": 2.441805225653207, |
| "grad_norm": 48.25, |
| "learning_rate": 5.116389548693586e-07, |
| "logits/chosen": 0.06853917986154556, |
| "logits/rejected": 0.1522480845451355, |
| "logps/chosen": -41.95103073120117, |
| "logps/rejected": -64.08912658691406, |
| "loss": 0.4026, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4834258258342743, |
| "rewards/margins": 0.8559648394584656, |
| "rewards/rejected": -1.3393906354904175, |
| "step": 4112 |
| }, |
| { |
| "epoch": 2.451306413301663, |
| "grad_norm": 41.5, |
| "learning_rate": 5.097387173396674e-07, |
| "logits/chosen": 0.07197491824626923, |
| "logits/rejected": 0.16541120409965515, |
| "logps/chosen": -42.12483215332031, |
| "logps/rejected": -66.00977325439453, |
| "loss": 0.3502, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.4519379436969757, |
| "rewards/margins": 1.0152369737625122, |
| "rewards/rejected": -1.467175006866455, |
| "step": 4128 |
| }, |
| { |
| "epoch": 2.460807600950119, |
| "grad_norm": 41.0, |
| "learning_rate": 5.078384798099762e-07, |
| "logits/chosen": 0.048155199736356735, |
| "logits/rejected": 0.14590026438236237, |
| "logps/chosen": -41.200191497802734, |
| "logps/rejected": -63.48173522949219, |
| "loss": 0.3958, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.46383845806121826, |
| "rewards/margins": 0.8997552394866943, |
| "rewards/rejected": -1.3635936975479126, |
| "step": 4144 |
| }, |
| { |
| "epoch": 2.470308788598575, |
| "grad_norm": 44.0, |
| "learning_rate": 5.05938242280285e-07, |
| "logits/chosen": 0.06484143435955048, |
| "logits/rejected": 0.13990077376365662, |
| "logps/chosen": -41.676612854003906, |
| "logps/rejected": -66.45744323730469, |
| "loss": 0.3469, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.404636025428772, |
| "rewards/margins": 1.0517361164093018, |
| "rewards/rejected": -1.4563721418380737, |
| "step": 4160 |
| }, |
| { |
| "epoch": 2.4798099762470307, |
| "grad_norm": 56.0, |
| "learning_rate": 5.040380047505938e-07, |
| "logits/chosen": 0.10340757668018341, |
| "logits/rejected": 0.12481331825256348, |
| "logps/chosen": -41.96209716796875, |
| "logps/rejected": -66.62129211425781, |
| "loss": 0.3694, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.4350067377090454, |
| "rewards/margins": 0.967208981513977, |
| "rewards/rejected": -1.4022157192230225, |
| "step": 4176 |
| }, |
| { |
| "epoch": 2.489311163895487, |
| "grad_norm": 49.5, |
| "learning_rate": 5.021377672209026e-07, |
| "logits/chosen": 0.07601352035999298, |
| "logits/rejected": 0.17555804550647736, |
| "logps/chosen": -41.83412170410156, |
| "logps/rejected": -62.92101287841797, |
| "loss": 0.385, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4230985641479492, |
| "rewards/margins": 0.9050178527832031, |
| "rewards/rejected": -1.3281164169311523, |
| "step": 4192 |
| }, |
| { |
| "epoch": 2.498812351543943, |
| "grad_norm": 44.5, |
| "learning_rate": 5.002375296912114e-07, |
| "logits/chosen": 0.03947605937719345, |
| "logits/rejected": 0.14804242551326752, |
| "logps/chosen": -42.070068359375, |
| "logps/rejected": -67.8903579711914, |
| "loss": 0.3496, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3808472752571106, |
| "rewards/margins": 1.0300160646438599, |
| "rewards/rejected": -1.4108633995056152, |
| "step": 4208 |
| }, |
| { |
| "epoch": 2.508313539192399, |
| "grad_norm": 45.25, |
| "learning_rate": 4.983372921615201e-07, |
| "logits/chosen": 0.05546602979302406, |
| "logits/rejected": 0.106099434196949, |
| "logps/chosen": -39.989471435546875, |
| "logps/rejected": -63.233856201171875, |
| "loss": 0.3484, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.33619987964630127, |
| "rewards/margins": 1.0194822549819946, |
| "rewards/rejected": -1.3556820154190063, |
| "step": 4224 |
| }, |
| { |
| "epoch": 2.517814726840855, |
| "grad_norm": 52.0, |
| "learning_rate": 4.96437054631829e-07, |
| "logits/chosen": 0.026712900027632713, |
| "logits/rejected": 0.09488269686698914, |
| "logps/chosen": -41.7761344909668, |
| "logps/rejected": -60.326847076416016, |
| "loss": 0.3839, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.4112342596054077, |
| "rewards/margins": 0.9102669358253479, |
| "rewards/rejected": -1.3215012550354004, |
| "step": 4240 |
| }, |
| { |
| "epoch": 2.527315914489311, |
| "grad_norm": 47.5, |
| "learning_rate": 4.945368171021377e-07, |
| "logits/chosen": 0.050753381103277206, |
| "logits/rejected": 0.10667050629854202, |
| "logps/chosen": -42.3249626159668, |
| "logps/rejected": -64.19955444335938, |
| "loss": 0.4062, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.46207451820373535, |
| "rewards/margins": 0.864716649055481, |
| "rewards/rejected": -1.3267912864685059, |
| "step": 4256 |
| }, |
| { |
| "epoch": 2.5368171021377672, |
| "grad_norm": 47.25, |
| "learning_rate": 4.926365795724465e-07, |
| "logits/chosen": 0.007675782777369022, |
| "logits/rejected": 0.06300470232963562, |
| "logps/chosen": -42.04328918457031, |
| "logps/rejected": -61.39952850341797, |
| "loss": 0.3991, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.45607635378837585, |
| "rewards/margins": 0.8779357671737671, |
| "rewards/rejected": -1.3340120315551758, |
| "step": 4272 |
| }, |
| { |
| "epoch": 2.5463182897862233, |
| "grad_norm": 52.75, |
| "learning_rate": 4.907363420427553e-07, |
| "logits/chosen": 0.08944439142942429, |
| "logits/rejected": 0.16112002730369568, |
| "logps/chosen": -42.05665588378906, |
| "logps/rejected": -65.32984924316406, |
| "loss": 0.3578, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.43149158358573914, |
| "rewards/margins": 0.9926659464836121, |
| "rewards/rejected": -1.4241576194763184, |
| "step": 4288 |
| }, |
| { |
| "epoch": 2.5558194774346794, |
| "grad_norm": 43.75, |
| "learning_rate": 4.888361045130641e-07, |
| "logits/chosen": 0.048501964658498764, |
| "logits/rejected": 0.11412826180458069, |
| "logps/chosen": -40.358360290527344, |
| "logps/rejected": -62.57841110229492, |
| "loss": 0.3745, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3942060172557831, |
| "rewards/margins": 0.9297899007797241, |
| "rewards/rejected": -1.32399582862854, |
| "step": 4304 |
| }, |
| { |
| "epoch": 2.5653206650831355, |
| "grad_norm": 62.5, |
| "learning_rate": 4.869358669833729e-07, |
| "logits/chosen": 0.048068106174468994, |
| "logits/rejected": 0.11154348403215408, |
| "logps/chosen": -42.39956283569336, |
| "logps/rejected": -64.91427612304688, |
| "loss": 0.3943, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.42655476927757263, |
| "rewards/margins": 0.9072946310043335, |
| "rewards/rejected": -1.3338494300842285, |
| "step": 4320 |
| }, |
| { |
| "epoch": 2.5748218527315916, |
| "grad_norm": 44.25, |
| "learning_rate": 4.850356294536817e-07, |
| "logits/chosen": 0.0894806981086731, |
| "logits/rejected": 0.12811443209648132, |
| "logps/chosen": -42.88471603393555, |
| "logps/rejected": -63.97032165527344, |
| "loss": 0.3795, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4201812446117401, |
| "rewards/margins": 0.9957151412963867, |
| "rewards/rejected": -1.4158962965011597, |
| "step": 4336 |
| }, |
| { |
| "epoch": 2.5843230403800472, |
| "grad_norm": 47.25, |
| "learning_rate": 4.831353919239905e-07, |
| "logits/chosen": 0.11298425495624542, |
| "logits/rejected": 0.12845686078071594, |
| "logps/chosen": -42.59984588623047, |
| "logps/rejected": -62.72551727294922, |
| "loss": 0.4004, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4689531922340393, |
| "rewards/margins": 0.8442877531051636, |
| "rewards/rejected": -1.3132410049438477, |
| "step": 4352 |
| }, |
| { |
| "epoch": 2.5938242280285033, |
| "grad_norm": 41.5, |
| "learning_rate": 4.812351543942993e-07, |
| "logits/chosen": 0.03180324286222458, |
| "logits/rejected": 0.11583473533391953, |
| "logps/chosen": -41.20764923095703, |
| "logps/rejected": -62.94646453857422, |
| "loss": 0.3847, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.42716217041015625, |
| "rewards/margins": 0.8983253240585327, |
| "rewards/rejected": -1.3254876136779785, |
| "step": 4368 |
| }, |
| { |
| "epoch": 2.6033254156769594, |
| "grad_norm": 53.5, |
| "learning_rate": 4.793349168646081e-07, |
| "logits/chosen": 0.09349140524864197, |
| "logits/rejected": 0.1733095794916153, |
| "logps/chosen": -42.4179573059082, |
| "logps/rejected": -64.96019744873047, |
| "loss": 0.3653, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.4492948353290558, |
| "rewards/margins": 0.9919961094856262, |
| "rewards/rejected": -1.4412909746170044, |
| "step": 4384 |
| }, |
| { |
| "epoch": 2.6128266033254155, |
| "grad_norm": 42.5, |
| "learning_rate": 4.774346793349168e-07, |
| "logits/chosen": -0.035068899393081665, |
| "logits/rejected": 0.11121690273284912, |
| "logps/chosen": -38.56016540527344, |
| "logps/rejected": -62.15085220336914, |
| "loss": 0.3403, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.33855387568473816, |
| "rewards/margins": 1.0478910207748413, |
| "rewards/rejected": -1.3864448070526123, |
| "step": 4400 |
| }, |
| { |
| "epoch": 2.6223277909738716, |
| "grad_norm": 38.75, |
| "learning_rate": 4.755344418052256e-07, |
| "logits/chosen": 0.08347325026988983, |
| "logits/rejected": 0.181388258934021, |
| "logps/chosen": -42.476497650146484, |
| "logps/rejected": -65.1061019897461, |
| "loss": 0.3708, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.4493361711502075, |
| "rewards/margins": 0.9572871327400208, |
| "rewards/rejected": -1.4066232442855835, |
| "step": 4416 |
| }, |
| { |
| "epoch": 2.6318289786223277, |
| "grad_norm": 34.25, |
| "learning_rate": 4.736342042755344e-07, |
| "logits/chosen": -0.0012733405455946922, |
| "logits/rejected": 0.10057978332042694, |
| "logps/chosen": -40.456119537353516, |
| "logps/rejected": -63.483036041259766, |
| "loss": 0.3623, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.35637179017066956, |
| "rewards/margins": 0.99317467212677, |
| "rewards/rejected": -1.3495464324951172, |
| "step": 4432 |
| }, |
| { |
| "epoch": 2.6413301662707838, |
| "grad_norm": 60.0, |
| "learning_rate": 4.717339667458432e-07, |
| "logits/chosen": 0.04187817499041557, |
| "logits/rejected": 0.11814681440591812, |
| "logps/chosen": -42.3458251953125, |
| "logps/rejected": -61.30575180053711, |
| "loss": 0.4308, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.46507805585861206, |
| "rewards/margins": 0.7858736515045166, |
| "rewards/rejected": -1.2509517669677734, |
| "step": 4448 |
| }, |
| { |
| "epoch": 2.65083135391924, |
| "grad_norm": 43.25, |
| "learning_rate": 4.69833729216152e-07, |
| "logits/chosen": 0.02010425180196762, |
| "logits/rejected": 0.12688446044921875, |
| "logps/chosen": -41.3795051574707, |
| "logps/rejected": -62.74733352661133, |
| "loss": 0.3521, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.40290382504463196, |
| "rewards/margins": 0.9980502128601074, |
| "rewards/rejected": -1.4009541273117065, |
| "step": 4464 |
| }, |
| { |
| "epoch": 2.660332541567696, |
| "grad_norm": 49.75, |
| "learning_rate": 4.679334916864608e-07, |
| "logits/chosen": 0.04653356224298477, |
| "logits/rejected": 0.1141427606344223, |
| "logps/chosen": -42.21457290649414, |
| "logps/rejected": -63.52908706665039, |
| "loss": 0.3789, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.39218419790267944, |
| "rewards/margins": 0.9429512023925781, |
| "rewards/rejected": -1.3351353406906128, |
| "step": 4480 |
| }, |
| { |
| "epoch": 2.669833729216152, |
| "grad_norm": 49.0, |
| "learning_rate": 4.660332541567696e-07, |
| "logits/chosen": 0.03440314531326294, |
| "logits/rejected": 0.06475966423749924, |
| "logps/chosen": -40.21307373046875, |
| "logps/rejected": -61.209136962890625, |
| "loss": 0.3654, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.36982235312461853, |
| "rewards/margins": 0.9860597252845764, |
| "rewards/rejected": -1.355882167816162, |
| "step": 4496 |
| }, |
| { |
| "epoch": 2.679334916864608, |
| "grad_norm": 40.75, |
| "learning_rate": 4.641330166270784e-07, |
| "logits/chosen": 0.05200592428445816, |
| "logits/rejected": 0.09623152762651443, |
| "logps/chosen": -40.480892181396484, |
| "logps/rejected": -61.244468688964844, |
| "loss": 0.3711, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.32693397998809814, |
| "rewards/margins": 0.9501146078109741, |
| "rewards/rejected": -1.2770487070083618, |
| "step": 4512 |
| }, |
| { |
| "epoch": 2.688836104513064, |
| "grad_norm": 44.0, |
| "learning_rate": 4.622327790973872e-07, |
| "logits/chosen": 0.011258577927947044, |
| "logits/rejected": 0.15000179409980774, |
| "logps/chosen": -42.406429290771484, |
| "logps/rejected": -67.35293579101562, |
| "loss": 0.3564, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.42154744267463684, |
| "rewards/margins": 1.0057048797607422, |
| "rewards/rejected": -1.427252173423767, |
| "step": 4528 |
| }, |
| { |
| "epoch": 2.6983372921615203, |
| "grad_norm": 42.25, |
| "learning_rate": 4.60332541567696e-07, |
| "logits/chosen": 0.01903606206178665, |
| "logits/rejected": 0.128606379032135, |
| "logps/chosen": -38.58045959472656, |
| "logps/rejected": -61.52817916870117, |
| "loss": 0.3749, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3256430923938751, |
| "rewards/margins": 0.9467271566390991, |
| "rewards/rejected": -1.2723702192306519, |
| "step": 4544 |
| }, |
| { |
| "epoch": 2.7078384798099764, |
| "grad_norm": 62.25, |
| "learning_rate": 4.584323040380047e-07, |
| "logits/chosen": 0.03335714340209961, |
| "logits/rejected": 0.16635264456272125, |
| "logps/chosen": -39.833648681640625, |
| "logps/rejected": -61.806251525878906, |
| "loss": 0.3795, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3701300024986267, |
| "rewards/margins": 0.9247270226478577, |
| "rewards/rejected": -1.2948570251464844, |
| "step": 4560 |
| }, |
| { |
| "epoch": 2.7173396674584325, |
| "grad_norm": 34.0, |
| "learning_rate": 4.565320665083135e-07, |
| "logits/chosen": 0.011373243294656277, |
| "logits/rejected": 0.1352601945400238, |
| "logps/chosen": -41.90055465698242, |
| "logps/rejected": -65.95397186279297, |
| "loss": 0.372, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.4336491525173187, |
| "rewards/margins": 0.9594928026199341, |
| "rewards/rejected": -1.3931418657302856, |
| "step": 4576 |
| }, |
| { |
| "epoch": 2.7268408551068886, |
| "grad_norm": 40.75, |
| "learning_rate": 4.5463182897862227e-07, |
| "logits/chosen": 0.03081374615430832, |
| "logits/rejected": 0.1375807374715805, |
| "logps/chosen": -40.30317687988281, |
| "logps/rejected": -62.725467681884766, |
| "loss": 0.3491, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.33172154426574707, |
| "rewards/margins": 1.030013918876648, |
| "rewards/rejected": -1.3617355823516846, |
| "step": 4592 |
| }, |
| { |
| "epoch": 2.7363420427553447, |
| "grad_norm": 53.5, |
| "learning_rate": 4.5273159144893107e-07, |
| "logits/chosen": 0.002102881669998169, |
| "logits/rejected": 0.05711708217859268, |
| "logps/chosen": -39.306236267089844, |
| "logps/rejected": -62.230323791503906, |
| "loss": 0.3496, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3352733254432678, |
| "rewards/margins": 1.0495156049728394, |
| "rewards/rejected": -1.3847888708114624, |
| "step": 4608 |
| }, |
| { |
| "epoch": 2.7458432304038007, |
| "grad_norm": 54.0, |
| "learning_rate": 4.5083135391923986e-07, |
| "logits/chosen": 0.02053050696849823, |
| "logits/rejected": 0.12392740696668625, |
| "logps/chosen": -41.42273712158203, |
| "logps/rejected": -63.603023529052734, |
| "loss": 0.3822, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4146535396575928, |
| "rewards/margins": 0.9132061004638672, |
| "rewards/rejected": -1.32785964012146, |
| "step": 4624 |
| }, |
| { |
| "epoch": 2.7553444180522564, |
| "grad_norm": 38.5, |
| "learning_rate": 4.4893111638954866e-07, |
| "logits/chosen": 0.0008830556180328131, |
| "logits/rejected": 0.11984378099441528, |
| "logps/chosen": -41.754310607910156, |
| "logps/rejected": -67.43799591064453, |
| "loss": 0.3531, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4328157305717468, |
| "rewards/margins": 1.053360939025879, |
| "rewards/rejected": -1.4861767292022705, |
| "step": 4640 |
| }, |
| { |
| "epoch": 2.7648456057007125, |
| "grad_norm": 51.5, |
| "learning_rate": 4.4703087885985746e-07, |
| "logits/chosen": 0.0631004199385643, |
| "logits/rejected": 0.13722942769527435, |
| "logps/chosen": -41.07665252685547, |
| "logps/rejected": -62.44814682006836, |
| "loss": 0.3926, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.39121901988983154, |
| "rewards/margins": 0.9159356355667114, |
| "rewards/rejected": -1.307154655456543, |
| "step": 4656 |
| }, |
| { |
| "epoch": 2.7743467933491686, |
| "grad_norm": 50.25, |
| "learning_rate": 4.4513064133016626e-07, |
| "logits/chosen": 0.034338682889938354, |
| "logits/rejected": 0.18351401388645172, |
| "logps/chosen": -40.274269104003906, |
| "logps/rejected": -65.64212799072266, |
| "loss": 0.3389, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.34793734550476074, |
| "rewards/margins": 1.0671257972717285, |
| "rewards/rejected": -1.4150630235671997, |
| "step": 4672 |
| }, |
| { |
| "epoch": 2.7838479809976246, |
| "grad_norm": 44.25, |
| "learning_rate": 4.4323040380047505e-07, |
| "logits/chosen": 0.0658361166715622, |
| "logits/rejected": 0.22124770283699036, |
| "logps/chosen": -41.521907806396484, |
| "logps/rejected": -66.72534942626953, |
| "loss": 0.337, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.3596152663230896, |
| "rewards/margins": 1.0657168626785278, |
| "rewards/rejected": -1.4253321886062622, |
| "step": 4688 |
| }, |
| { |
| "epoch": 2.7933491686460807, |
| "grad_norm": 53.5, |
| "learning_rate": 4.4133016627078385e-07, |
| "logits/chosen": 0.09054507315158844, |
| "logits/rejected": 0.17531618475914001, |
| "logps/chosen": -39.683006286621094, |
| "logps/rejected": -65.05414581298828, |
| "loss": 0.3834, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.39716941118240356, |
| "rewards/margins": 0.9602290987968445, |
| "rewards/rejected": -1.357398271560669, |
| "step": 4704 |
| }, |
| { |
| "epoch": 2.802850356294537, |
| "grad_norm": 36.25, |
| "learning_rate": 4.3942992874109265e-07, |
| "logits/chosen": 0.03398045897483826, |
| "logits/rejected": 0.1653180718421936, |
| "logps/chosen": -41.86040496826172, |
| "logps/rejected": -66.90827941894531, |
| "loss": 0.3572, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3843865394592285, |
| "rewards/margins": 1.0734546184539795, |
| "rewards/rejected": -1.457841157913208, |
| "step": 4720 |
| }, |
| { |
| "epoch": 2.812351543942993, |
| "grad_norm": 37.5, |
| "learning_rate": 4.3752969121140144e-07, |
| "logits/chosen": 0.039868682622909546, |
| "logits/rejected": 0.1837385594844818, |
| "logps/chosen": -40.565216064453125, |
| "logps/rejected": -63.868896484375, |
| "loss": 0.3527, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.378643661737442, |
| "rewards/margins": 0.9983595013618469, |
| "rewards/rejected": -1.3770031929016113, |
| "step": 4736 |
| }, |
| { |
| "epoch": 2.821852731591449, |
| "grad_norm": 55.25, |
| "learning_rate": 4.356294536817102e-07, |
| "logits/chosen": 0.09310074895620346, |
| "logits/rejected": 0.14162081480026245, |
| "logps/chosen": -41.470603942871094, |
| "logps/rejected": -65.76705169677734, |
| "loss": 0.3606, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4034523069858551, |
| "rewards/margins": 1.0022685527801514, |
| "rewards/rejected": -1.4057209491729736, |
| "step": 4752 |
| }, |
| { |
| "epoch": 2.831353919239905, |
| "grad_norm": 41.75, |
| "learning_rate": 4.33729216152019e-07, |
| "logits/chosen": 0.11650769412517548, |
| "logits/rejected": 0.15587171912193298, |
| "logps/chosen": -41.557228088378906, |
| "logps/rejected": -60.604087829589844, |
| "loss": 0.3735, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.4029754400253296, |
| "rewards/margins": 0.8826481103897095, |
| "rewards/rejected": -1.2856236696243286, |
| "step": 4768 |
| }, |
| { |
| "epoch": 2.840855106888361, |
| "grad_norm": 43.75, |
| "learning_rate": 4.318289786223278e-07, |
| "logits/chosen": 0.02145340107381344, |
| "logits/rejected": 0.14118751883506775, |
| "logps/chosen": -42.241329193115234, |
| "logps/rejected": -64.31900024414062, |
| "loss": 0.3717, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4430907368659973, |
| "rewards/margins": 0.9653340578079224, |
| "rewards/rejected": -1.4084248542785645, |
| "step": 4784 |
| }, |
| { |
| "epoch": 2.850356294536817, |
| "grad_norm": 43.25, |
| "learning_rate": 4.299287410926365e-07, |
| "logits/chosen": 0.026168465614318848, |
| "logits/rejected": 0.06277532130479813, |
| "logps/chosen": -41.47759246826172, |
| "logps/rejected": -60.46055603027344, |
| "loss": 0.4074, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.43215835094451904, |
| "rewards/margins": 0.8435641527175903, |
| "rewards/rejected": -1.2757225036621094, |
| "step": 4800 |
| }, |
| { |
| "epoch": 2.859857482185273, |
| "grad_norm": 42.25, |
| "learning_rate": 4.280285035629453e-07, |
| "logits/chosen": 0.074520543217659, |
| "logits/rejected": 0.16152548789978027, |
| "logps/chosen": -41.94219970703125, |
| "logps/rejected": -64.68109893798828, |
| "loss": 0.3645, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3893333375453949, |
| "rewards/margins": 0.9812647104263306, |
| "rewards/rejected": -1.3705980777740479, |
| "step": 4816 |
| }, |
| { |
| "epoch": 2.869358669833729, |
| "grad_norm": 39.25, |
| "learning_rate": 4.261282660332541e-07, |
| "logits/chosen": 0.02079102396965027, |
| "logits/rejected": 0.15321360528469086, |
| "logps/chosen": -39.940887451171875, |
| "logps/rejected": -64.5802230834961, |
| "loss": 0.3704, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.41797706484794617, |
| "rewards/margins": 0.9673931002616882, |
| "rewards/rejected": -1.385370135307312, |
| "step": 4832 |
| }, |
| { |
| "epoch": 2.878859857482185, |
| "grad_norm": 56.75, |
| "learning_rate": 4.242280285035629e-07, |
| "logits/chosen": 0.020523881539702415, |
| "logits/rejected": 0.18188393115997314, |
| "logps/chosen": -40.60096740722656, |
| "logps/rejected": -64.39540100097656, |
| "loss": 0.3794, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.42755821347236633, |
| "rewards/margins": 0.9751715064048767, |
| "rewards/rejected": -1.4027297496795654, |
| "step": 4848 |
| }, |
| { |
| "epoch": 2.888361045130641, |
| "grad_norm": 41.0, |
| "learning_rate": 4.223277909738717e-07, |
| "logits/chosen": 0.038833338767290115, |
| "logits/rejected": 0.15533462166786194, |
| "logps/chosen": -39.362998962402344, |
| "logps/rejected": -65.98983001708984, |
| "loss": 0.3129, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.29717180132865906, |
| "rewards/margins": 1.1468960046768188, |
| "rewards/rejected": -1.4440677165985107, |
| "step": 4864 |
| }, |
| { |
| "epoch": 2.8978622327790973, |
| "grad_norm": 63.75, |
| "learning_rate": 4.204275534441805e-07, |
| "logits/chosen": 0.05080319941043854, |
| "logits/rejected": 0.11638832837343216, |
| "logps/chosen": -40.73345184326172, |
| "logps/rejected": -62.49778747558594, |
| "loss": 0.387, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.3901386260986328, |
| "rewards/margins": 0.9359588623046875, |
| "rewards/rejected": -1.3260974884033203, |
| "step": 4880 |
| }, |
| { |
| "epoch": 2.9073634204275534, |
| "grad_norm": 41.0, |
| "learning_rate": 4.185273159144893e-07, |
| "logits/chosen": 0.05678907781839371, |
| "logits/rejected": 0.19164641201496124, |
| "logps/chosen": -40.55916213989258, |
| "logps/rejected": -65.99730682373047, |
| "loss": 0.3831, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.3813077211380005, |
| "rewards/margins": 0.9531018733978271, |
| "rewards/rejected": -1.3344097137451172, |
| "step": 4896 |
| }, |
| { |
| "epoch": 2.9168646080760094, |
| "grad_norm": 53.0, |
| "learning_rate": 4.166270783847981e-07, |
| "logits/chosen": -0.0027349982410669327, |
| "logits/rejected": 0.0990891233086586, |
| "logps/chosen": -39.93202209472656, |
| "logps/rejected": -62.99840545654297, |
| "loss": 0.3721, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.36999353766441345, |
| "rewards/margins": 0.9719130992889404, |
| "rewards/rejected": -1.3419066667556763, |
| "step": 4912 |
| }, |
| { |
| "epoch": 2.9263657957244655, |
| "grad_norm": 40.25, |
| "learning_rate": 4.147268408551069e-07, |
| "logits/chosen": 0.05395427346229553, |
| "logits/rejected": 0.13786588609218597, |
| "logps/chosen": -40.425132751464844, |
| "logps/rejected": -61.25279998779297, |
| "loss": 0.3538, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.34139615297317505, |
| "rewards/margins": 1.0007226467132568, |
| "rewards/rejected": -1.3421188592910767, |
| "step": 4928 |
| }, |
| { |
| "epoch": 2.9358669833729216, |
| "grad_norm": 65.0, |
| "learning_rate": 4.1282660332541564e-07, |
| "logits/chosen": 0.09111806005239487, |
| "logits/rejected": 0.13585761189460754, |
| "logps/chosen": -43.16571807861328, |
| "logps/rejected": -64.36392211914062, |
| "loss": 0.3918, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4209873378276825, |
| "rewards/margins": 0.8701137900352478, |
| "rewards/rejected": -1.2911012172698975, |
| "step": 4944 |
| }, |
| { |
| "epoch": 2.9453681710213777, |
| "grad_norm": 53.0, |
| "learning_rate": 4.1092636579572444e-07, |
| "logits/chosen": 0.05853826552629471, |
| "logits/rejected": 0.18161046504974365, |
| "logps/chosen": -40.96638488769531, |
| "logps/rejected": -66.1131591796875, |
| "loss": 0.3467, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3545738160610199, |
| "rewards/margins": 1.0768417119979858, |
| "rewards/rejected": -1.4314155578613281, |
| "step": 4960 |
| }, |
| { |
| "epoch": 2.954869358669834, |
| "grad_norm": 41.5, |
| "learning_rate": 4.0902612826603324e-07, |
| "logits/chosen": 0.06567525863647461, |
| "logits/rejected": 0.18003502488136292, |
| "logps/chosen": -39.08511734008789, |
| "logps/rejected": -61.72608947753906, |
| "loss": 0.3658, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3397156000137329, |
| "rewards/margins": 1.0000014305114746, |
| "rewards/rejected": -1.3397170305252075, |
| "step": 4976 |
| }, |
| { |
| "epoch": 2.96437054631829, |
| "grad_norm": 44.25, |
| "learning_rate": 4.0712589073634203e-07, |
| "logits/chosen": 0.016267111524939537, |
| "logits/rejected": 0.13151559233665466, |
| "logps/chosen": -40.71261978149414, |
| "logps/rejected": -62.006256103515625, |
| "loss": 0.3973, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.4248185455799103, |
| "rewards/margins": 0.8792138695716858, |
| "rewards/rejected": -1.3040324449539185, |
| "step": 4992 |
| }, |
| { |
| "epoch": 2.973871733966746, |
| "grad_norm": 58.5, |
| "learning_rate": 4.0522565320665083e-07, |
| "logits/chosen": 0.03460274264216423, |
| "logits/rejected": 0.13233664631843567, |
| "logps/chosen": -40.716407775878906, |
| "logps/rejected": -62.79671096801758, |
| "loss": 0.3879, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4130927622318268, |
| "rewards/margins": 0.9299353957176208, |
| "rewards/rejected": -1.3430280685424805, |
| "step": 5008 |
| }, |
| { |
| "epoch": 2.983372921615202, |
| "grad_norm": 47.0, |
| "learning_rate": 4.0332541567695963e-07, |
| "logits/chosen": -0.026960894465446472, |
| "logits/rejected": 0.11078447848558426, |
| "logps/chosen": -39.08903121948242, |
| "logps/rejected": -62.35736083984375, |
| "loss": 0.3588, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3670194745063782, |
| "rewards/margins": 1.0107150077819824, |
| "rewards/rejected": -1.3777345418930054, |
| "step": 5024 |
| }, |
| { |
| "epoch": 2.992874109263658, |
| "grad_norm": 54.0, |
| "learning_rate": 4.0142517814726837e-07, |
| "logits/chosen": 0.026894917711615562, |
| "logits/rejected": 0.1446702629327774, |
| "logps/chosen": -41.22724914550781, |
| "logps/rejected": -64.66578674316406, |
| "loss": 0.3601, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3958403468132019, |
| "rewards/margins": 0.9907092452049255, |
| "rewards/rejected": -1.386549472808838, |
| "step": 5040 |
| }, |
| { |
| "epoch": 3.002375296912114, |
| "grad_norm": 38.25, |
| "learning_rate": 3.9952494061757717e-07, |
| "logits/chosen": 0.0721140205860138, |
| "logits/rejected": 0.08904615044593811, |
| "logps/chosen": -42.36117172241211, |
| "logps/rejected": -62.753578186035156, |
| "loss": 0.3669, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.4690694510936737, |
| "rewards/margins": 0.9362343549728394, |
| "rewards/rejected": -1.4053038358688354, |
| "step": 5056 |
| }, |
| { |
| "epoch": 3.01187648456057, |
| "grad_norm": 38.75, |
| "learning_rate": 3.9762470308788596e-07, |
| "logits/chosen": 0.08058139681816101, |
| "logits/rejected": 0.14635083079338074, |
| "logps/chosen": -40.83808135986328, |
| "logps/rejected": -62.58879470825195, |
| "loss": 0.3736, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.40799736976623535, |
| "rewards/margins": 0.9339090585708618, |
| "rewards/rejected": -1.3419064283370972, |
| "step": 5072 |
| }, |
| { |
| "epoch": 3.021377672209026, |
| "grad_norm": 49.25, |
| "learning_rate": 3.9572446555819476e-07, |
| "logits/chosen": 0.07044284790754318, |
| "logits/rejected": 0.14013811945915222, |
| "logps/chosen": -40.04065704345703, |
| "logps/rejected": -60.96607971191406, |
| "loss": 0.3681, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.35846540331840515, |
| "rewards/margins": 0.9638465642929077, |
| "rewards/rejected": -1.3223118782043457, |
| "step": 5088 |
| }, |
| { |
| "epoch": 3.030878859857482, |
| "grad_norm": 52.0, |
| "learning_rate": 3.9382422802850356e-07, |
| "logits/chosen": 0.060851648449897766, |
| "logits/rejected": 0.1331576406955719, |
| "logps/chosen": -41.8791618347168, |
| "logps/rejected": -64.0675048828125, |
| "loss": 0.3568, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.359686017036438, |
| "rewards/margins": 0.9825432300567627, |
| "rewards/rejected": -1.3422291278839111, |
| "step": 5104 |
| }, |
| { |
| "epoch": 3.040380047505938, |
| "grad_norm": 40.5, |
| "learning_rate": 3.919239904988123e-07, |
| "logits/chosen": 0.009696897119283676, |
| "logits/rejected": 0.12189489603042603, |
| "logps/chosen": -39.73085021972656, |
| "logps/rejected": -63.392818450927734, |
| "loss": 0.3708, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3765300512313843, |
| "rewards/margins": 0.9506427049636841, |
| "rewards/rejected": -1.3271726369857788, |
| "step": 5120 |
| }, |
| { |
| "epoch": 3.0498812351543942, |
| "grad_norm": 48.75, |
| "learning_rate": 3.900237529691211e-07, |
| "logits/chosen": 0.10276854038238525, |
| "logits/rejected": 0.13192720711231232, |
| "logps/chosen": -42.13685989379883, |
| "logps/rejected": -63.64410400390625, |
| "loss": 0.3788, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.39574676752090454, |
| "rewards/margins": 0.9484460949897766, |
| "rewards/rejected": -1.3441928625106812, |
| "step": 5136 |
| }, |
| { |
| "epoch": 3.0593824228028503, |
| "grad_norm": 52.25, |
| "learning_rate": 3.881235154394299e-07, |
| "logits/chosen": 0.11582867801189423, |
| "logits/rejected": 0.11568093299865723, |
| "logps/chosen": -43.7314338684082, |
| "logps/rejected": -63.54302215576172, |
| "loss": 0.3977, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.4822978973388672, |
| "rewards/margins": 0.8951501846313477, |
| "rewards/rejected": -1.3774480819702148, |
| "step": 5152 |
| }, |
| { |
| "epoch": 3.0688836104513064, |
| "grad_norm": 49.25, |
| "learning_rate": 3.862232779097387e-07, |
| "logits/chosen": 0.08060070872306824, |
| "logits/rejected": 0.10565990209579468, |
| "logps/chosen": -41.34340286254883, |
| "logps/rejected": -63.646820068359375, |
| "loss": 0.3848, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3923201858997345, |
| "rewards/margins": 0.9066511392593384, |
| "rewards/rejected": -1.29897141456604, |
| "step": 5168 |
| }, |
| { |
| "epoch": 3.0783847980997625, |
| "grad_norm": 46.25, |
| "learning_rate": 3.843230403800475e-07, |
| "logits/chosen": 0.055783893913030624, |
| "logits/rejected": 0.14008364081382751, |
| "logps/chosen": -41.782936096191406, |
| "logps/rejected": -62.20945739746094, |
| "loss": 0.3836, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.39728862047195435, |
| "rewards/margins": 0.8838520050048828, |
| "rewards/rejected": -1.2811405658721924, |
| "step": 5184 |
| }, |
| { |
| "epoch": 3.0878859857482186, |
| "grad_norm": 43.5, |
| "learning_rate": 3.824228028503563e-07, |
| "logits/chosen": 0.025185655802488327, |
| "logits/rejected": 0.12884651124477386, |
| "logps/chosen": -41.46108627319336, |
| "logps/rejected": -65.40599822998047, |
| "loss": 0.3744, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.3796524405479431, |
| "rewards/margins": 0.9922239780426025, |
| "rewards/rejected": -1.3718763589859009, |
| "step": 5200 |
| }, |
| { |
| "epoch": 3.0973871733966747, |
| "grad_norm": 57.25, |
| "learning_rate": 3.805225653206651e-07, |
| "logits/chosen": 0.036128733307123184, |
| "logits/rejected": 0.14582857489585876, |
| "logps/chosen": -40.435264587402344, |
| "logps/rejected": -62.72749328613281, |
| "loss": 0.3616, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.34147587418556213, |
| "rewards/margins": 1.0060913562774658, |
| "rewards/rejected": -1.3475672006607056, |
| "step": 5216 |
| }, |
| { |
| "epoch": 3.1068883610451308, |
| "grad_norm": 54.25, |
| "learning_rate": 3.786223277909739e-07, |
| "logits/chosen": 0.08208269625902176, |
| "logits/rejected": 0.1514563262462616, |
| "logps/chosen": -43.135169982910156, |
| "logps/rejected": -62.251564025878906, |
| "loss": 0.4333, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -0.4657037854194641, |
| "rewards/margins": 0.7968182563781738, |
| "rewards/rejected": -1.2625218629837036, |
| "step": 5232 |
| }, |
| { |
| "epoch": 3.116389548693587, |
| "grad_norm": 47.75, |
| "learning_rate": 3.767220902612827e-07, |
| "logits/chosen": 0.0023325812071561813, |
| "logits/rejected": 0.07401884347200394, |
| "logps/chosen": -42.72369384765625, |
| "logps/rejected": -65.19200134277344, |
| "loss": 0.3696, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4340641498565674, |
| "rewards/margins": 0.9625180959701538, |
| "rewards/rejected": -1.3965823650360107, |
| "step": 5248 |
| }, |
| { |
| "epoch": 3.125890736342043, |
| "grad_norm": 55.5, |
| "learning_rate": 3.748218527315915e-07, |
| "logits/chosen": 0.04036543890833855, |
| "logits/rejected": 0.1542033553123474, |
| "logps/chosen": -40.91764831542969, |
| "logps/rejected": -67.87660217285156, |
| "loss": 0.3403, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.42045873403549194, |
| "rewards/margins": 1.0873808860778809, |
| "rewards/rejected": -1.5078396797180176, |
| "step": 5264 |
| }, |
| { |
| "epoch": 3.1353919239904986, |
| "grad_norm": 55.0, |
| "learning_rate": 3.729216152019002e-07, |
| "logits/chosen": 0.0443246066570282, |
| "logits/rejected": 0.21922534704208374, |
| "logps/chosen": -42.55435562133789, |
| "logps/rejected": -65.33242797851562, |
| "loss": 0.4199, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.4705503284931183, |
| "rewards/margins": 0.8472099304199219, |
| "rewards/rejected": -1.3177603483200073, |
| "step": 5280 |
| }, |
| { |
| "epoch": 3.1448931116389547, |
| "grad_norm": 53.25, |
| "learning_rate": 3.71021377672209e-07, |
| "logits/chosen": 0.009390661492943764, |
| "logits/rejected": 0.12596291303634644, |
| "logps/chosen": -40.291473388671875, |
| "logps/rejected": -62.212562561035156, |
| "loss": 0.3862, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.4117724299430847, |
| "rewards/margins": 0.9006836414337158, |
| "rewards/rejected": -1.3124560117721558, |
| "step": 5296 |
| }, |
| { |
| "epoch": 3.1543942992874108, |
| "grad_norm": 50.0, |
| "learning_rate": 3.6912114014251776e-07, |
| "logits/chosen": -0.022532382979989052, |
| "logits/rejected": 0.11396850645542145, |
| "logps/chosen": -40.82377624511719, |
| "logps/rejected": -63.843467712402344, |
| "loss": 0.3762, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.40509068965911865, |
| "rewards/margins": 0.9473409652709961, |
| "rewards/rejected": -1.3524316549301147, |
| "step": 5312 |
| }, |
| { |
| "epoch": 3.163895486935867, |
| "grad_norm": 51.25, |
| "learning_rate": 3.6722090261282656e-07, |
| "logits/chosen": -0.019587915390729904, |
| "logits/rejected": 0.09787953644990921, |
| "logps/chosen": -38.83479690551758, |
| "logps/rejected": -62.37450408935547, |
| "loss": 0.3457, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.31038060784339905, |
| "rewards/margins": 1.0302282571792603, |
| "rewards/rejected": -1.340608835220337, |
| "step": 5328 |
| }, |
| { |
| "epoch": 3.173396674584323, |
| "grad_norm": 45.0, |
| "learning_rate": 3.6532066508313535e-07, |
| "logits/chosen": 0.0068716611713171005, |
| "logits/rejected": 0.10715761035680771, |
| "logps/chosen": -39.36448287963867, |
| "logps/rejected": -61.61016845703125, |
| "loss": 0.3694, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.35794466733932495, |
| "rewards/margins": 0.9621202349662781, |
| "rewards/rejected": -1.3200650215148926, |
| "step": 5344 |
| }, |
| { |
| "epoch": 3.182897862232779, |
| "grad_norm": 57.5, |
| "learning_rate": 3.6342042755344415e-07, |
| "logits/chosen": 0.08224662393331528, |
| "logits/rejected": 0.10815519094467163, |
| "logps/chosen": -43.51884078979492, |
| "logps/rejected": -65.097900390625, |
| "loss": 0.3834, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.4219568073749542, |
| "rewards/margins": 0.934171736240387, |
| "rewards/rejected": -1.3561286926269531, |
| "step": 5360 |
| }, |
| { |
| "epoch": 3.192399049881235, |
| "grad_norm": 47.5, |
| "learning_rate": 3.6152019002375295e-07, |
| "logits/chosen": 0.011415719985961914, |
| "logits/rejected": 0.10578812658786774, |
| "logps/chosen": -41.20673370361328, |
| "logps/rejected": -63.52978515625, |
| "loss": 0.365, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.41507184505462646, |
| "rewards/margins": 0.9821330308914185, |
| "rewards/rejected": -1.397204875946045, |
| "step": 5376 |
| }, |
| { |
| "epoch": 3.201900237529691, |
| "grad_norm": 42.5, |
| "learning_rate": 3.5961995249406174e-07, |
| "logits/chosen": 0.03810262307524681, |
| "logits/rejected": 0.12873801589012146, |
| "logps/chosen": -41.731388092041016, |
| "logps/rejected": -65.47847747802734, |
| "loss": 0.3798, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.44025665521621704, |
| "rewards/margins": 0.9518187046051025, |
| "rewards/rejected": -1.3920754194259644, |
| "step": 5392 |
| }, |
| { |
| "epoch": 3.2114014251781473, |
| "grad_norm": 49.25, |
| "learning_rate": 3.5771971496437054e-07, |
| "logits/chosen": -0.023601891472935677, |
| "logits/rejected": 0.06935597211122513, |
| "logps/chosen": -38.68426513671875, |
| "logps/rejected": -61.103973388671875, |
| "loss": 0.3634, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.37336137890815735, |
| "rewards/margins": 1.0120704174041748, |
| "rewards/rejected": -1.3854318857192993, |
| "step": 5408 |
| }, |
| { |
| "epoch": 3.2209026128266034, |
| "grad_norm": 52.5, |
| "learning_rate": 3.5581947743467934e-07, |
| "logits/chosen": 0.04542490094900131, |
| "logits/rejected": 0.09217678010463715, |
| "logps/chosen": -43.481048583984375, |
| "logps/rejected": -63.5504264831543, |
| "loss": 0.415, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.49973058700561523, |
| "rewards/margins": 0.8297138214111328, |
| "rewards/rejected": -1.3294442892074585, |
| "step": 5424 |
| }, |
| { |
| "epoch": 3.2304038004750595, |
| "grad_norm": 47.0, |
| "learning_rate": 3.5391923990498813e-07, |
| "logits/chosen": 0.10594628006219864, |
| "logits/rejected": 0.17065931856632233, |
| "logps/chosen": -40.92080307006836, |
| "logps/rejected": -64.85243225097656, |
| "loss": 0.3541, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3807818293571472, |
| "rewards/margins": 1.0301458835601807, |
| "rewards/rejected": -1.4109277725219727, |
| "step": 5440 |
| }, |
| { |
| "epoch": 3.2399049881235156, |
| "grad_norm": 66.0, |
| "learning_rate": 3.5201900237529693e-07, |
| "logits/chosen": 0.04224825277924538, |
| "logits/rejected": 0.11179321259260178, |
| "logps/chosen": -42.496681213378906, |
| "logps/rejected": -64.15071105957031, |
| "loss": 0.3789, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.41750308871269226, |
| "rewards/margins": 0.9374274015426636, |
| "rewards/rejected": -1.3549304008483887, |
| "step": 5456 |
| }, |
| { |
| "epoch": 3.2494061757719717, |
| "grad_norm": 51.75, |
| "learning_rate": 3.5011876484560573e-07, |
| "logits/chosen": 0.06084592267870903, |
| "logits/rejected": 0.13545851409435272, |
| "logps/chosen": -38.884830474853516, |
| "logps/rejected": -61.21929931640625, |
| "loss": 0.3627, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3169533908367157, |
| "rewards/margins": 0.991154670715332, |
| "rewards/rejected": -1.3081080913543701, |
| "step": 5472 |
| }, |
| { |
| "epoch": 3.2589073634204277, |
| "grad_norm": 52.5, |
| "learning_rate": 3.482185273159145e-07, |
| "logits/chosen": 0.03806290030479431, |
| "logits/rejected": 0.11884848773479462, |
| "logps/chosen": -41.503875732421875, |
| "logps/rejected": -63.86762237548828, |
| "loss": 0.3766, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.40684768557548523, |
| "rewards/margins": 0.953045129776001, |
| "rewards/rejected": -1.3598929643630981, |
| "step": 5488 |
| }, |
| { |
| "epoch": 3.268408551068884, |
| "grad_norm": 57.0, |
| "learning_rate": 3.463182897862232e-07, |
| "logits/chosen": -0.0012608803808689117, |
| "logits/rejected": 0.10703101754188538, |
| "logps/chosen": -41.227787017822266, |
| "logps/rejected": -64.53129577636719, |
| "loss": 0.3804, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4662329852581024, |
| "rewards/margins": 0.9123408794403076, |
| "rewards/rejected": -1.3785738945007324, |
| "step": 5504 |
| }, |
| { |
| "epoch": 3.2779097387173395, |
| "grad_norm": 35.75, |
| "learning_rate": 3.44418052256532e-07, |
| "logits/chosen": 0.09872997552156448, |
| "logits/rejected": 0.16530072689056396, |
| "logps/chosen": -42.996360778808594, |
| "logps/rejected": -64.78094482421875, |
| "loss": 0.3656, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.43684783577919006, |
| "rewards/margins": 0.9832981824874878, |
| "rewards/rejected": -1.420146107673645, |
| "step": 5520 |
| }, |
| { |
| "epoch": 3.2874109263657956, |
| "grad_norm": 50.25, |
| "learning_rate": 3.425178147268408e-07, |
| "logits/chosen": 0.03449930623173714, |
| "logits/rejected": 0.17792941629886627, |
| "logps/chosen": -40.4708366394043, |
| "logps/rejected": -65.352294921875, |
| "loss": 0.3781, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.43737536668777466, |
| "rewards/margins": 0.9423834681510925, |
| "rewards/rejected": -1.3797588348388672, |
| "step": 5536 |
| }, |
| { |
| "epoch": 3.2969121140142517, |
| "grad_norm": 47.0, |
| "learning_rate": 3.406175771971496e-07, |
| "logits/chosen": 0.09654662013053894, |
| "logits/rejected": 0.17820453643798828, |
| "logps/chosen": -41.23036193847656, |
| "logps/rejected": -63.637550354003906, |
| "loss": 0.3973, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.36412250995635986, |
| "rewards/margins": 0.9050154685974121, |
| "rewards/rejected": -1.269137978553772, |
| "step": 5552 |
| }, |
| { |
| "epoch": 3.3064133016627077, |
| "grad_norm": 62.5, |
| "learning_rate": 3.387173396674584e-07, |
| "logits/chosen": 0.03694698214530945, |
| "logits/rejected": 0.13538768887519836, |
| "logps/chosen": -42.770050048828125, |
| "logps/rejected": -63.76899719238281, |
| "loss": 0.4137, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.49573254585266113, |
| "rewards/margins": 0.8381365537643433, |
| "rewards/rejected": -1.3338689804077148, |
| "step": 5568 |
| }, |
| { |
| "epoch": 3.315914489311164, |
| "grad_norm": 45.0, |
| "learning_rate": 3.368171021377672e-07, |
| "logits/chosen": 0.02915555238723755, |
| "logits/rejected": 0.1136770024895668, |
| "logps/chosen": -42.1306266784668, |
| "logps/rejected": -64.29971313476562, |
| "loss": 0.3729, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.4326779246330261, |
| "rewards/margins": 0.928207516670227, |
| "rewards/rejected": -1.360885500907898, |
| "step": 5584 |
| }, |
| { |
| "epoch": 3.32541567695962, |
| "grad_norm": 45.75, |
| "learning_rate": 3.34916864608076e-07, |
| "logits/chosen": 0.07809650897979736, |
| "logits/rejected": 0.12652210891246796, |
| "logps/chosen": -41.568660736083984, |
| "logps/rejected": -63.57560348510742, |
| "loss": 0.3613, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3894152045249939, |
| "rewards/margins": 1.0010182857513428, |
| "rewards/rejected": -1.3904335498809814, |
| "step": 5600 |
| }, |
| { |
| "epoch": 3.334916864608076, |
| "grad_norm": 58.5, |
| "learning_rate": 3.330166270783848e-07, |
| "logits/chosen": 0.040801357477903366, |
| "logits/rejected": 0.1780150681734085, |
| "logps/chosen": -41.86552810668945, |
| "logps/rejected": -65.90018463134766, |
| "loss": 0.3516, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.38096481561660767, |
| "rewards/margins": 1.0388493537902832, |
| "rewards/rejected": -1.4198143482208252, |
| "step": 5616 |
| }, |
| { |
| "epoch": 3.344418052256532, |
| "grad_norm": 53.5, |
| "learning_rate": 3.311163895486936e-07, |
| "logits/chosen": 0.01211823895573616, |
| "logits/rejected": 0.09781680256128311, |
| "logps/chosen": -41.53929138183594, |
| "logps/rejected": -63.851688385009766, |
| "loss": 0.3701, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4125053286552429, |
| "rewards/margins": 0.9792938828468323, |
| "rewards/rejected": -1.3917990922927856, |
| "step": 5632 |
| }, |
| { |
| "epoch": 3.353919239904988, |
| "grad_norm": 50.75, |
| "learning_rate": 3.292161520190024e-07, |
| "logits/chosen": 0.09114633500576019, |
| "logits/rejected": 0.15826475620269775, |
| "logps/chosen": -41.09789276123047, |
| "logps/rejected": -65.02250671386719, |
| "loss": 0.3713, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.41848012804985046, |
| "rewards/margins": 0.9512667059898376, |
| "rewards/rejected": -1.3697468042373657, |
| "step": 5648 |
| }, |
| { |
| "epoch": 3.3634204275534443, |
| "grad_norm": 51.25, |
| "learning_rate": 3.273159144893112e-07, |
| "logits/chosen": 0.021810825914144516, |
| "logits/rejected": 0.11112373322248459, |
| "logps/chosen": -43.460018157958984, |
| "logps/rejected": -64.75310516357422, |
| "loss": 0.3962, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.4909437894821167, |
| "rewards/margins": 0.8337816596031189, |
| "rewards/rejected": -1.3247253894805908, |
| "step": 5664 |
| }, |
| { |
| "epoch": 3.3729216152019004, |
| "grad_norm": 71.0, |
| "learning_rate": 3.2541567695961993e-07, |
| "logits/chosen": 0.06712229549884796, |
| "logits/rejected": 0.15457472205162048, |
| "logps/chosen": -41.23008346557617, |
| "logps/rejected": -63.24055480957031, |
| "loss": 0.3614, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3526281416416168, |
| "rewards/margins": 0.9992504119873047, |
| "rewards/rejected": -1.3518785238265991, |
| "step": 5680 |
| }, |
| { |
| "epoch": 3.3824228028503565, |
| "grad_norm": 37.5, |
| "learning_rate": 3.235154394299287e-07, |
| "logits/chosen": 0.008397895842790604, |
| "logits/rejected": 0.11099565029144287, |
| "logps/chosen": -42.79941940307617, |
| "logps/rejected": -62.79872131347656, |
| "loss": 0.3956, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.432941198348999, |
| "rewards/margins": 0.8876244425773621, |
| "rewards/rejected": -1.3205657005310059, |
| "step": 5696 |
| }, |
| { |
| "epoch": 3.391923990498812, |
| "grad_norm": 46.75, |
| "learning_rate": 3.216152019002375e-07, |
| "logits/chosen": 0.03148173540830612, |
| "logits/rejected": 0.09412042796611786, |
| "logps/chosen": -40.27482223510742, |
| "logps/rejected": -64.9065170288086, |
| "loss": 0.3578, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.36367034912109375, |
| "rewards/margins": 1.0289617776870728, |
| "rewards/rejected": -1.392632246017456, |
| "step": 5712 |
| }, |
| { |
| "epoch": 3.401425178147268, |
| "grad_norm": 61.25, |
| "learning_rate": 3.1971496437054627e-07, |
| "logits/chosen": 0.03281719982624054, |
| "logits/rejected": 0.10020038485527039, |
| "logps/chosen": -41.20103073120117, |
| "logps/rejected": -62.454811096191406, |
| "loss": 0.3892, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.4000158905982971, |
| "rewards/margins": 0.9230848550796509, |
| "rewards/rejected": -1.3231008052825928, |
| "step": 5728 |
| }, |
| { |
| "epoch": 3.4109263657957243, |
| "grad_norm": 42.5, |
| "learning_rate": 3.1781472684085506e-07, |
| "logits/chosen": 0.03375418484210968, |
| "logits/rejected": 0.07718026638031006, |
| "logps/chosen": -42.45466613769531, |
| "logps/rejected": -62.32984924316406, |
| "loss": 0.4054, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -0.4012918472290039, |
| "rewards/margins": 0.8690704107284546, |
| "rewards/rejected": -1.2703622579574585, |
| "step": 5744 |
| }, |
| { |
| "epoch": 3.4204275534441804, |
| "grad_norm": 50.25, |
| "learning_rate": 3.1591448931116386e-07, |
| "logits/chosen": 0.04832587391138077, |
| "logits/rejected": 0.14456292986869812, |
| "logps/chosen": -38.7043571472168, |
| "logps/rejected": -61.933982849121094, |
| "loss": 0.368, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.33057889342308044, |
| "rewards/margins": 0.9730508923530579, |
| "rewards/rejected": -1.3036296367645264, |
| "step": 5760 |
| }, |
| { |
| "epoch": 3.4299287410926365, |
| "grad_norm": 41.5, |
| "learning_rate": 3.1401425178147266e-07, |
| "logits/chosen": 0.07459306716918945, |
| "logits/rejected": 0.08237803727388382, |
| "logps/chosen": -41.34931945800781, |
| "logps/rejected": -62.95779037475586, |
| "loss": 0.3687, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.37355130910873413, |
| "rewards/margins": 1.0145928859710693, |
| "rewards/rejected": -1.3881440162658691, |
| "step": 5776 |
| }, |
| { |
| "epoch": 3.4394299287410925, |
| "grad_norm": 57.5, |
| "learning_rate": 3.1211401425178145e-07, |
| "logits/chosen": 0.03901844099164009, |
| "logits/rejected": 0.11285356432199478, |
| "logps/chosen": -40.3481559753418, |
| "logps/rejected": -63.19662094116211, |
| "loss": 0.3758, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.3881966769695282, |
| "rewards/margins": 0.9612338542938232, |
| "rewards/rejected": -1.3494304418563843, |
| "step": 5792 |
| }, |
| { |
| "epoch": 3.4489311163895486, |
| "grad_norm": 35.75, |
| "learning_rate": 3.1021377672209025e-07, |
| "logits/chosen": -0.005038658622652292, |
| "logits/rejected": 0.06839510053396225, |
| "logps/chosen": -39.993309020996094, |
| "logps/rejected": -60.90965270996094, |
| "loss": 0.3949, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.37709131836891174, |
| "rewards/margins": 0.8873628973960876, |
| "rewards/rejected": -1.2644541263580322, |
| "step": 5808 |
| }, |
| { |
| "epoch": 3.4584323040380047, |
| "grad_norm": 56.0, |
| "learning_rate": 3.0831353919239905e-07, |
| "logits/chosen": 0.09682201594114304, |
| "logits/rejected": 0.11965522170066833, |
| "logps/chosen": -41.68549346923828, |
| "logps/rejected": -64.51807403564453, |
| "loss": 0.3576, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.36759233474731445, |
| "rewards/margins": 1.0216158628463745, |
| "rewards/rejected": -1.389208197593689, |
| "step": 5824 |
| }, |
| { |
| "epoch": 3.467933491686461, |
| "grad_norm": 47.25, |
| "learning_rate": 3.0641330166270784e-07, |
| "logits/chosen": 0.04335436224937439, |
| "logits/rejected": 0.09136922657489777, |
| "logps/chosen": -41.07777404785156, |
| "logps/rejected": -62.33134460449219, |
| "loss": 0.3905, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.41379624605178833, |
| "rewards/margins": 0.8802096843719482, |
| "rewards/rejected": -1.2940058708190918, |
| "step": 5840 |
| }, |
| { |
| "epoch": 3.477434679334917, |
| "grad_norm": 46.75, |
| "learning_rate": 3.0451306413301664e-07, |
| "logits/chosen": 0.07700560986995697, |
| "logits/rejected": 0.11378947645425797, |
| "logps/chosen": -41.164493560791016, |
| "logps/rejected": -62.910099029541016, |
| "loss": 0.3575, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.35974451899528503, |
| "rewards/margins": 0.9829455614089966, |
| "rewards/rejected": -1.3426902294158936, |
| "step": 5856 |
| }, |
| { |
| "epoch": 3.486935866983373, |
| "grad_norm": 37.5, |
| "learning_rate": 3.026128266033254e-07, |
| "logits/chosen": -0.009967565536499023, |
| "logits/rejected": 0.13184106349945068, |
| "logps/chosen": -41.89753723144531, |
| "logps/rejected": -65.06842041015625, |
| "loss": 0.373, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.43801864981651306, |
| "rewards/margins": 0.9392563104629517, |
| "rewards/rejected": -1.377274990081787, |
| "step": 5872 |
| }, |
| { |
| "epoch": 3.496437054631829, |
| "grad_norm": 47.25, |
| "learning_rate": 3.007125890736342e-07, |
| "logits/chosen": 0.0052648792043328285, |
| "logits/rejected": 0.1504189670085907, |
| "logps/chosen": -40.17364501953125, |
| "logps/rejected": -62.997554779052734, |
| "loss": 0.3612, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3528364896774292, |
| "rewards/margins": 0.9979708194732666, |
| "rewards/rejected": -1.3508071899414062, |
| "step": 5888 |
| }, |
| { |
| "epoch": 3.505938242280285, |
| "grad_norm": 54.75, |
| "learning_rate": 2.98812351543943e-07, |
| "logits/chosen": 0.017038684338331223, |
| "logits/rejected": 0.06112390756607056, |
| "logps/chosen": -41.39101791381836, |
| "logps/rejected": -64.11455535888672, |
| "loss": 0.386, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4848855137825012, |
| "rewards/margins": 0.9375608563423157, |
| "rewards/rejected": -1.4224462509155273, |
| "step": 5904 |
| }, |
| { |
| "epoch": 3.5154394299287413, |
| "grad_norm": 49.75, |
| "learning_rate": 2.969121140142518e-07, |
| "logits/chosen": 0.020917030051350594, |
| "logits/rejected": 0.09810462594032288, |
| "logps/chosen": -39.80662155151367, |
| "logps/rejected": -60.70354080200195, |
| "loss": 0.3528, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3079432249069214, |
| "rewards/margins": 1.035301685333252, |
| "rewards/rejected": -1.3432449102401733, |
| "step": 5920 |
| }, |
| { |
| "epoch": 3.5249406175771973, |
| "grad_norm": 48.25, |
| "learning_rate": 2.9501187648456057e-07, |
| "logits/chosen": 0.07180722057819366, |
| "logits/rejected": 0.20187009871006012, |
| "logps/chosen": -42.24882507324219, |
| "logps/rejected": -66.23828887939453, |
| "loss": 0.3684, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.36295828223228455, |
| "rewards/margins": 0.9907328486442566, |
| "rewards/rejected": -1.3536912202835083, |
| "step": 5936 |
| }, |
| { |
| "epoch": 3.5344418052256534, |
| "grad_norm": 55.75, |
| "learning_rate": 2.9311163895486937e-07, |
| "logits/chosen": 0.04826946556568146, |
| "logits/rejected": 0.18203888833522797, |
| "logps/chosen": -44.12788772583008, |
| "logps/rejected": -65.50685119628906, |
| "loss": 0.4068, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.45171403884887695, |
| "rewards/margins": 0.8484476208686829, |
| "rewards/rejected": -1.300161600112915, |
| "step": 5952 |
| }, |
| { |
| "epoch": 3.5439429928741095, |
| "grad_norm": 67.5, |
| "learning_rate": 2.912114014251781e-07, |
| "logits/chosen": 0.03645118325948715, |
| "logits/rejected": 0.10001954436302185, |
| "logps/chosen": -41.21052551269531, |
| "logps/rejected": -59.9468994140625, |
| "loss": 0.4013, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.432812362909317, |
| "rewards/margins": 0.8651708364486694, |
| "rewards/rejected": -1.297983169555664, |
| "step": 5968 |
| }, |
| { |
| "epoch": 3.553444180522565, |
| "grad_norm": 59.0, |
| "learning_rate": 2.893111638954869e-07, |
| "logits/chosen": 0.014936832711100578, |
| "logits/rejected": 0.13327737152576447, |
| "logps/chosen": -39.648658752441406, |
| "logps/rejected": -64.09784698486328, |
| "loss": 0.3456, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.34701600670814514, |
| "rewards/margins": 1.0596061944961548, |
| "rewards/rejected": -1.4066221714019775, |
| "step": 5984 |
| }, |
| { |
| "epoch": 3.5629453681710213, |
| "grad_norm": 67.0, |
| "learning_rate": 2.874109263657957e-07, |
| "logits/chosen": -0.038970720022916794, |
| "logits/rejected": 0.1254507154226303, |
| "logps/chosen": -39.5800666809082, |
| "logps/rejected": -63.40620422363281, |
| "loss": 0.3871, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3893817365169525, |
| "rewards/margins": 0.9221255779266357, |
| "rewards/rejected": -1.3115073442459106, |
| "step": 6000 |
| }, |
| { |
| "epoch": 3.5724465558194773, |
| "grad_norm": 46.25, |
| "learning_rate": 2.855106888361045e-07, |
| "logits/chosen": 0.027300620451569557, |
| "logits/rejected": 0.14734551310539246, |
| "logps/chosen": -39.858192443847656, |
| "logps/rejected": -64.0389633178711, |
| "loss": 0.365, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.40028518438339233, |
| "rewards/margins": 0.9700754284858704, |
| "rewards/rejected": -1.3703604936599731, |
| "step": 6016 |
| }, |
| { |
| "epoch": 3.5819477434679334, |
| "grad_norm": 50.0, |
| "learning_rate": 2.836104513064133e-07, |
| "logits/chosen": 0.056467145681381226, |
| "logits/rejected": 0.12261079996824265, |
| "logps/chosen": -40.67634963989258, |
| "logps/rejected": -62.402809143066406, |
| "loss": 0.3536, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.3845675587654114, |
| "rewards/margins": 1.0051778554916382, |
| "rewards/rejected": -1.3897454738616943, |
| "step": 6032 |
| }, |
| { |
| "epoch": 3.5914489311163895, |
| "grad_norm": 55.25, |
| "learning_rate": 2.8171021377672204e-07, |
| "logits/chosen": 0.10420281440019608, |
| "logits/rejected": 0.13047069311141968, |
| "logps/chosen": -40.67375946044922, |
| "logps/rejected": -61.41259765625, |
| "loss": 0.3815, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3753702640533447, |
| "rewards/margins": 0.9167128205299377, |
| "rewards/rejected": -1.2920830249786377, |
| "step": 6048 |
| }, |
| { |
| "epoch": 3.6009501187648456, |
| "grad_norm": 68.0, |
| "learning_rate": 2.7980997624703084e-07, |
| "logits/chosen": 0.05103810504078865, |
| "logits/rejected": 0.10168743878602982, |
| "logps/chosen": -41.25544738769531, |
| "logps/rejected": -60.72285461425781, |
| "loss": 0.3892, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.3910224437713623, |
| "rewards/margins": 0.9060119390487671, |
| "rewards/rejected": -1.2970343828201294, |
| "step": 6064 |
| }, |
| { |
| "epoch": 3.6104513064133017, |
| "grad_norm": 54.25, |
| "learning_rate": 2.7790973871733964e-07, |
| "logits/chosen": 0.013073207810521126, |
| "logits/rejected": 0.12633521854877472, |
| "logps/chosen": -41.78628921508789, |
| "logps/rejected": -66.28352355957031, |
| "loss": 0.353, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.40143895149230957, |
| "rewards/margins": 1.0364677906036377, |
| "rewards/rejected": -1.4379067420959473, |
| "step": 6080 |
| }, |
| { |
| "epoch": 3.619952494061758, |
| "grad_norm": 50.75, |
| "learning_rate": 2.7600950118764843e-07, |
| "logits/chosen": 0.08037324994802475, |
| "logits/rejected": 0.17064939439296722, |
| "logps/chosen": -42.0279655456543, |
| "logps/rejected": -64.91426086425781, |
| "loss": 0.3877, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.43144869804382324, |
| "rewards/margins": 0.8975583910942078, |
| "rewards/rejected": -1.3290070295333862, |
| "step": 6096 |
| }, |
| { |
| "epoch": 3.629453681710214, |
| "grad_norm": 56.5, |
| "learning_rate": 2.7410926365795723e-07, |
| "logits/chosen": 0.055456578731536865, |
| "logits/rejected": 0.22028645873069763, |
| "logps/chosen": -39.780517578125, |
| "logps/rejected": -63.519813537597656, |
| "loss": 0.3566, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3360990881919861, |
| "rewards/margins": 0.9839707612991333, |
| "rewards/rejected": -1.3200697898864746, |
| "step": 6112 |
| }, |
| { |
| "epoch": 3.63895486935867, |
| "grad_norm": 39.5, |
| "learning_rate": 2.7220902612826603e-07, |
| "logits/chosen": 0.060147836804389954, |
| "logits/rejected": 0.1302787959575653, |
| "logps/chosen": -41.454872131347656, |
| "logps/rejected": -64.60635375976562, |
| "loss": 0.3871, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.4015429615974426, |
| "rewards/margins": 0.9040000438690186, |
| "rewards/rejected": -1.3055431842803955, |
| "step": 6128 |
| }, |
| { |
| "epoch": 3.648456057007126, |
| "grad_norm": 47.25, |
| "learning_rate": 2.703087885985748e-07, |
| "logits/chosen": -0.013463707640767097, |
| "logits/rejected": 0.07376192510128021, |
| "logps/chosen": -39.62220764160156, |
| "logps/rejected": -62.20975875854492, |
| "loss": 0.363, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.35915249586105347, |
| "rewards/margins": 0.9867640733718872, |
| "rewards/rejected": -1.3459166288375854, |
| "step": 6144 |
| }, |
| { |
| "epoch": 3.6579572446555817, |
| "grad_norm": 49.75, |
| "learning_rate": 2.684085510688836e-07, |
| "logits/chosen": 0.010295089334249496, |
| "logits/rejected": 0.17823557555675507, |
| "logps/chosen": -41.6696662902832, |
| "logps/rejected": -68.72691345214844, |
| "loss": 0.3555, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.4070267975330353, |
| "rewards/margins": 1.028420329093933, |
| "rewards/rejected": -1.4354472160339355, |
| "step": 6160 |
| }, |
| { |
| "epoch": 3.667458432304038, |
| "grad_norm": 37.0, |
| "learning_rate": 2.665083135391924e-07, |
| "logits/chosen": 0.10623430460691452, |
| "logits/rejected": 0.16167762875556946, |
| "logps/chosen": -42.180198669433594, |
| "logps/rejected": -66.37389373779297, |
| "loss": 0.3498, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4009030759334564, |
| "rewards/margins": 1.0136104822158813, |
| "rewards/rejected": -1.4145135879516602, |
| "step": 6176 |
| }, |
| { |
| "epoch": 3.676959619952494, |
| "grad_norm": 51.5, |
| "learning_rate": 2.646080760095012e-07, |
| "logits/chosen": 0.048040006309747696, |
| "logits/rejected": 0.14534232020378113, |
| "logps/chosen": -43.282859802246094, |
| "logps/rejected": -66.5873794555664, |
| "loss": 0.3543, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4304080605506897, |
| "rewards/margins": 1.0149825811386108, |
| "rewards/rejected": -1.4453905820846558, |
| "step": 6192 |
| }, |
| { |
| "epoch": 3.68646080760095, |
| "grad_norm": 47.25, |
| "learning_rate": 2.6270783847980996e-07, |
| "logits/chosen": 0.08240100741386414, |
| "logits/rejected": 0.16390424966812134, |
| "logps/chosen": -40.04043960571289, |
| "logps/rejected": -62.7987060546875, |
| "loss": 0.3618, |
| "rewards/accuracies": 0.953125, |
| "rewards/chosen": -0.33928748965263367, |
| "rewards/margins": 0.9862152934074402, |
| "rewards/rejected": -1.325502634048462, |
| "step": 6208 |
| }, |
| { |
| "epoch": 3.695961995249406, |
| "grad_norm": 80.5, |
| "learning_rate": 2.6080760095011876e-07, |
| "logits/chosen": 0.021123535931110382, |
| "logits/rejected": 0.1238473653793335, |
| "logps/chosen": -40.08449935913086, |
| "logps/rejected": -64.84993743896484, |
| "loss": 0.3652, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.3657268285751343, |
| "rewards/margins": 1.0157949924468994, |
| "rewards/rejected": -1.3815219402313232, |
| "step": 6224 |
| }, |
| { |
| "epoch": 3.705463182897862, |
| "grad_norm": 56.25, |
| "learning_rate": 2.589073634204275e-07, |
| "logits/chosen": 0.09684562683105469, |
| "logits/rejected": 0.18495050072669983, |
| "logps/chosen": -41.702301025390625, |
| "logps/rejected": -63.34584426879883, |
| "loss": 0.3913, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.4372140169143677, |
| "rewards/margins": 0.9003235101699829, |
| "rewards/rejected": -1.3375376462936401, |
| "step": 6240 |
| }, |
| { |
| "epoch": 3.7149643705463182, |
| "grad_norm": 40.25, |
| "learning_rate": 2.570071258907363e-07, |
| "logits/chosen": 0.028134455904364586, |
| "logits/rejected": 0.16499164700508118, |
| "logps/chosen": -41.335540771484375, |
| "logps/rejected": -63.30027770996094, |
| "loss": 0.3824, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.4398137331008911, |
| "rewards/margins": 0.9075998067855835, |
| "rewards/rejected": -1.347413420677185, |
| "step": 6256 |
| }, |
| { |
| "epoch": 3.7244655581947743, |
| "grad_norm": 41.75, |
| "learning_rate": 2.551068883610451e-07, |
| "logits/chosen": 0.016426438465714455, |
| "logits/rejected": 0.13184240460395813, |
| "logps/chosen": -40.241729736328125, |
| "logps/rejected": -61.8122673034668, |
| "loss": 0.3795, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.35463353991508484, |
| "rewards/margins": 0.9521807432174683, |
| "rewards/rejected": -1.3068143129348755, |
| "step": 6272 |
| }, |
| { |
| "epoch": 3.7339667458432304, |
| "grad_norm": 50.5, |
| "learning_rate": 2.532066508313539e-07, |
| "logits/chosen": 0.028402097523212433, |
| "logits/rejected": 0.19204524159431458, |
| "logps/chosen": -41.228050231933594, |
| "logps/rejected": -63.15939712524414, |
| "loss": 0.3886, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -0.4359303116798401, |
| "rewards/margins": 0.902795135974884, |
| "rewards/rejected": -1.3387255668640137, |
| "step": 6288 |
| }, |
| { |
| "epoch": 3.7434679334916865, |
| "grad_norm": 49.75, |
| "learning_rate": 2.513064133016627e-07, |
| "logits/chosen": 0.03932081162929535, |
| "logits/rejected": 0.220359206199646, |
| "logps/chosen": -39.12273025512695, |
| "logps/rejected": -63.374549865722656, |
| "loss": 0.384, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.32400816679000854, |
| "rewards/margins": 0.9151921272277832, |
| "rewards/rejected": -1.239200234413147, |
| "step": 6304 |
| }, |
| { |
| "epoch": 3.7529691211401426, |
| "grad_norm": 35.25, |
| "learning_rate": 2.494061757719715e-07, |
| "logits/chosen": 0.0723482072353363, |
| "logits/rejected": 0.16996870934963226, |
| "logps/chosen": -42.35596466064453, |
| "logps/rejected": -63.216426849365234, |
| "loss": 0.3757, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.47743964195251465, |
| "rewards/margins": 0.9289887547492981, |
| "rewards/rejected": -1.4064284563064575, |
| "step": 6320 |
| }, |
| { |
| "epoch": 3.7624703087885987, |
| "grad_norm": 51.25, |
| "learning_rate": 2.475059382422803e-07, |
| "logits/chosen": 0.08452095091342926, |
| "logits/rejected": 0.16239051520824432, |
| "logps/chosen": -42.04669189453125, |
| "logps/rejected": -65.5211181640625, |
| "loss": 0.3721, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.42908474802970886, |
| "rewards/margins": 0.9674755930900574, |
| "rewards/rejected": -1.3965604305267334, |
| "step": 6336 |
| }, |
| { |
| "epoch": 3.7719714964370548, |
| "grad_norm": 39.5, |
| "learning_rate": 2.456057007125891e-07, |
| "logits/chosen": 0.028654370456933975, |
| "logits/rejected": 0.0740758553147316, |
| "logps/chosen": -41.6036262512207, |
| "logps/rejected": -61.02503967285156, |
| "loss": 0.409, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.42152711749076843, |
| "rewards/margins": 0.8335566520690918, |
| "rewards/rejected": -1.2550837993621826, |
| "step": 6352 |
| }, |
| { |
| "epoch": 3.781472684085511, |
| "grad_norm": 34.75, |
| "learning_rate": 2.437054631828979e-07, |
| "logits/chosen": 0.05283776670694351, |
| "logits/rejected": 0.15333330631256104, |
| "logps/chosen": -40.32767868041992, |
| "logps/rejected": -63.946720123291016, |
| "loss": 0.3505, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3486955463886261, |
| "rewards/margins": 1.0453296899795532, |
| "rewards/rejected": -1.394025444984436, |
| "step": 6368 |
| }, |
| { |
| "epoch": 3.790973871733967, |
| "grad_norm": 44.25, |
| "learning_rate": 2.418052256532066e-07, |
| "logits/chosen": 0.030136309564113617, |
| "logits/rejected": 0.09111961722373962, |
| "logps/chosen": -43.44806671142578, |
| "logps/rejected": -64.65709686279297, |
| "loss": 0.3773, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.48328760266304016, |
| "rewards/margins": 0.9284934997558594, |
| "rewards/rejected": -1.4117810726165771, |
| "step": 6384 |
| }, |
| { |
| "epoch": 3.800475059382423, |
| "grad_norm": 69.5, |
| "learning_rate": 2.399049881235154e-07, |
| "logits/chosen": -0.005254952237010002, |
| "logits/rejected": 0.14121603965759277, |
| "logps/chosen": -39.49591827392578, |
| "logps/rejected": -61.99911880493164, |
| "loss": 0.3777, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.3417325019836426, |
| "rewards/margins": 0.9448993802070618, |
| "rewards/rejected": -1.2866318225860596, |
| "step": 6400 |
| }, |
| { |
| "epoch": 3.809976247030879, |
| "grad_norm": 59.75, |
| "learning_rate": 2.380047505938242e-07, |
| "logits/chosen": 0.11867545545101166, |
| "logits/rejected": 0.2170010209083557, |
| "logps/chosen": -40.522804260253906, |
| "logps/rejected": -64.37654876708984, |
| "loss": 0.3459, |
| "rewards/accuracies": 0.9609375, |
| "rewards/chosen": -0.36430996656417847, |
| "rewards/margins": 1.050180435180664, |
| "rewards/rejected": -1.4144904613494873, |
| "step": 6416 |
| }, |
| { |
| "epoch": 3.8194774346793348, |
| "grad_norm": 46.75, |
| "learning_rate": 2.36104513064133e-07, |
| "logits/chosen": 0.09830942749977112, |
| "logits/rejected": 0.1840059906244278, |
| "logps/chosen": -40.83268356323242, |
| "logps/rejected": -63.659610748291016, |
| "loss": 0.3769, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.4339376389980316, |
| "rewards/margins": 0.9776801466941833, |
| "rewards/rejected": -1.4116177558898926, |
| "step": 6432 |
| }, |
| { |
| "epoch": 3.828978622327791, |
| "grad_norm": 54.0, |
| "learning_rate": 2.342042755344418e-07, |
| "logits/chosen": 0.03068104013800621, |
| "logits/rejected": 0.1621265709400177, |
| "logps/chosen": -43.22722625732422, |
| "logps/rejected": -66.03999328613281, |
| "loss": 0.4005, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.4799574315547943, |
| "rewards/margins": 0.8937349915504456, |
| "rewards/rejected": -1.373692512512207, |
| "step": 6448 |
| }, |
| { |
| "epoch": 3.838479809976247, |
| "grad_norm": 50.0, |
| "learning_rate": 2.323040380047506e-07, |
| "logits/chosen": 0.058121610432863235, |
| "logits/rejected": 0.12576644122600555, |
| "logps/chosen": -41.369140625, |
| "logps/rejected": -59.46518325805664, |
| "loss": 0.3966, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.40906572341918945, |
| "rewards/margins": 0.8850200176239014, |
| "rewards/rejected": -1.2940857410430908, |
| "step": 6464 |
| }, |
| { |
| "epoch": 3.847980997624703, |
| "grad_norm": 52.75, |
| "learning_rate": 2.3040380047505937e-07, |
| "logits/chosen": 0.06827502697706223, |
| "logits/rejected": 0.14090177416801453, |
| "logps/chosen": -42.05507278442383, |
| "logps/rejected": -63.853248596191406, |
| "loss": 0.3718, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.40036866068840027, |
| "rewards/margins": 0.9455254077911377, |
| "rewards/rejected": -1.3458940982818604, |
| "step": 6480 |
| }, |
| { |
| "epoch": 3.857482185273159, |
| "grad_norm": 43.75, |
| "learning_rate": 2.2850356294536814e-07, |
| "logits/chosen": 0.029515203088521957, |
| "logits/rejected": 0.14883099496364594, |
| "logps/chosen": -41.896156311035156, |
| "logps/rejected": -64.60520935058594, |
| "loss": 0.3798, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.389555424451828, |
| "rewards/margins": 0.9199389815330505, |
| "rewards/rejected": -1.3094943761825562, |
| "step": 6496 |
| }, |
| { |
| "epoch": 3.866983372921615, |
| "grad_norm": 39.75, |
| "learning_rate": 2.2660332541567694e-07, |
| "logits/chosen": 0.005287522915750742, |
| "logits/rejected": 0.15816916525363922, |
| "logps/chosen": -41.17705535888672, |
| "logps/rejected": -63.35408401489258, |
| "loss": 0.4071, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -0.4392343759536743, |
| "rewards/margins": 0.8662595748901367, |
| "rewards/rejected": -1.305493950843811, |
| "step": 6512 |
| }, |
| { |
| "epoch": 3.8764845605700713, |
| "grad_norm": 46.0, |
| "learning_rate": 2.2470308788598574e-07, |
| "logits/chosen": 0.07539123296737671, |
| "logits/rejected": 0.12418274581432343, |
| "logps/chosen": -41.912052154541016, |
| "logps/rejected": -64.26847076416016, |
| "loss": 0.3873, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.4488585293292999, |
| "rewards/margins": 0.9296143054962158, |
| "rewards/rejected": -1.3784728050231934, |
| "step": 6528 |
| }, |
| { |
| "epoch": 3.8859857482185274, |
| "grad_norm": 47.5, |
| "learning_rate": 2.2280285035629453e-07, |
| "logits/chosen": 0.04668428376317024, |
| "logits/rejected": 0.13457974791526794, |
| "logps/chosen": -40.552486419677734, |
| "logps/rejected": -63.59270477294922, |
| "loss": 0.371, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.3902091681957245, |
| "rewards/margins": 0.9766399264335632, |
| "rewards/rejected": -1.3668489456176758, |
| "step": 6544 |
| }, |
| { |
| "epoch": 3.8954869358669835, |
| "grad_norm": 51.75, |
| "learning_rate": 2.209026128266033e-07, |
| "logits/chosen": 0.019167862832546234, |
| "logits/rejected": 0.10863950103521347, |
| "logps/chosen": -40.27218246459961, |
| "logps/rejected": -63.5718994140625, |
| "loss": 0.355, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.400563508272171, |
| "rewards/margins": 1.0269160270690918, |
| "rewards/rejected": -1.4274795055389404, |
| "step": 6560 |
| }, |
| { |
| "epoch": 3.9049881235154396, |
| "grad_norm": 40.5, |
| "learning_rate": 2.190023752969121e-07, |
| "logits/chosen": 0.032968372106552124, |
| "logits/rejected": 0.1309332251548767, |
| "logps/chosen": -39.79338073730469, |
| "logps/rejected": -63.8279914855957, |
| "loss": 0.3562, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.40201592445373535, |
| "rewards/margins": 1.0143156051635742, |
| "rewards/rejected": -1.4163316488265991, |
| "step": 6576 |
| }, |
| { |
| "epoch": 3.9144893111638956, |
| "grad_norm": 42.75, |
| "learning_rate": 2.171021377672209e-07, |
| "logits/chosen": 0.06808724999427795, |
| "logits/rejected": 0.1723027527332306, |
| "logps/chosen": -39.6837272644043, |
| "logps/rejected": -63.00306701660156, |
| "loss": 0.3484, |
| "rewards/accuracies": 0.9453125, |
| "rewards/chosen": -0.3555365800857544, |
| "rewards/margins": 1.008614420890808, |
| "rewards/rejected": -1.3641510009765625, |
| "step": 6592 |
| }, |
| { |
| "epoch": 3.9239904988123513, |
| "grad_norm": 49.0, |
| "learning_rate": 2.1520190023752967e-07, |
| "logits/chosen": 0.08254537731409073, |
| "logits/rejected": 0.11683779954910278, |
| "logps/chosen": -40.13639450073242, |
| "logps/rejected": -59.88142776489258, |
| "loss": 0.3912, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.36244481801986694, |
| "rewards/margins": 0.8833273649215698, |
| "rewards/rejected": -1.245772123336792, |
| "step": 6608 |
| }, |
| { |
| "epoch": 3.9334916864608074, |
| "grad_norm": 55.25, |
| "learning_rate": 2.1330166270783847e-07, |
| "logits/chosen": 0.017773086205124855, |
| "logits/rejected": 0.12275815010070801, |
| "logps/chosen": -41.630889892578125, |
| "logps/rejected": -64.5182876586914, |
| "loss": 0.3651, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -0.39863353967666626, |
| "rewards/margins": 0.9960266351699829, |
| "rewards/rejected": -1.3946601152420044, |
| "step": 6624 |
| }, |
| { |
| "epoch": 3.9429928741092635, |
| "grad_norm": 44.25, |
| "learning_rate": 2.1140142517814726e-07, |
| "logits/chosen": 0.027539458125829697, |
| "logits/rejected": 0.1340925693511963, |
| "logps/chosen": -40.22187042236328, |
| "logps/rejected": -62.83479690551758, |
| "loss": 0.3761, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.42398306727409363, |
| "rewards/margins": 0.9288855195045471, |
| "rewards/rejected": -1.352868676185608, |
| "step": 6640 |
| }, |
| { |
| "epoch": 3.9524940617577196, |
| "grad_norm": 61.75, |
| "learning_rate": 2.0950118764845603e-07, |
| "logits/chosen": 0.031753845512866974, |
| "logits/rejected": 0.13772860169410706, |
| "logps/chosen": -41.15782928466797, |
| "logps/rejected": -64.47724914550781, |
| "loss": 0.3647, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.38045597076416016, |
| "rewards/margins": 1.0232834815979004, |
| "rewards/rejected": -1.4037394523620605, |
| "step": 6656 |
| }, |
| { |
| "epoch": 3.9619952494061756, |
| "grad_norm": 57.25, |
| "learning_rate": 2.0760095011876483e-07, |
| "logits/chosen": 0.0039275167509913445, |
| "logits/rejected": 0.13760821521282196, |
| "logps/chosen": -39.61132049560547, |
| "logps/rejected": -63.1502685546875, |
| "loss": 0.3637, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.3538341522216797, |
| "rewards/margins": 0.979717493057251, |
| "rewards/rejected": -1.3335516452789307, |
| "step": 6672 |
| }, |
| { |
| "epoch": 3.9714964370546317, |
| "grad_norm": 58.5, |
| "learning_rate": 2.0570071258907363e-07, |
| "logits/chosen": -0.005867550149559975, |
| "logits/rejected": 0.10856153070926666, |
| "logps/chosen": -40.953468322753906, |
| "logps/rejected": -64.8403091430664, |
| "loss": 0.3771, |
| "rewards/accuracies": 0.921875, |
| "rewards/chosen": -0.37854593992233276, |
| "rewards/margins": 0.982690155506134, |
| "rewards/rejected": -1.3612359762191772, |
| "step": 6688 |
| }, |
| { |
| "epoch": 3.980997624703088, |
| "grad_norm": 46.0, |
| "learning_rate": 2.0380047505938242e-07, |
| "logits/chosen": 0.07198520749807358, |
| "logits/rejected": 0.0809374749660492, |
| "logps/chosen": -40.85484313964844, |
| "logps/rejected": -63.39386749267578, |
| "loss": 0.379, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.4362155795097351, |
| "rewards/margins": 0.933577835559845, |
| "rewards/rejected": -1.3697935342788696, |
| "step": 6704 |
| }, |
| { |
| "epoch": 3.990498812351544, |
| "grad_norm": 57.5, |
| "learning_rate": 2.0190023752969122e-07, |
| "logits/chosen": 0.05437842011451721, |
| "logits/rejected": 0.1582392305135727, |
| "logps/chosen": -41.71647262573242, |
| "logps/rejected": -66.48292541503906, |
| "loss": 0.3764, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.4557511806488037, |
| "rewards/margins": 0.9482683539390564, |
| "rewards/rejected": -1.4040195941925049, |
| "step": 6720 |
| }, |
| { |
| "epoch": 4.0, |
| "grad_norm": 65.5, |
| "learning_rate": 2e-07, |
| "logits/chosen": 0.005993685685098171, |
| "logits/rejected": 0.11152346432209015, |
| "logps/chosen": -41.31070327758789, |
| "logps/rejected": -63.83424758911133, |
| "loss": 0.3716, |
| "rewards/accuracies": 0.9296875, |
| "rewards/chosen": -0.42573827505111694, |
| "rewards/margins": 0.946398138999939, |
| "rewards/rejected": -1.3721364736557007, |
| "step": 6736 |
| } |
| ], |
| "logging_steps": 16, |
| "max_steps": 8420, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 5, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
