diff --git "a/run_20251219_040313/checkpoint-500/trainer_state.json" "b/run_20251219_040313/checkpoint-500/trainer_state.json" new file mode 100644--- /dev/null +++ "b/run_20251219_040313/checkpoint-500/trainer_state.json" @@ -0,0 +1,9034 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.3783224024853298, + "eval_steps": 500, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.002761477390403866, + "grad_norm": 0.5198726654052734, + "learning_rate": 0.0, + "log_odds_chosen": 0.4317269027233124, + "log_odds_ratio": -0.5042418837547302, + "logits/chosen": -0.5456271767616272, + "logits/rejected": -0.10779725015163422, + "logps/chosen": -1.998489260673523, + "logps/rejected": -2.3799774646759033, + "loss": 2.1834, + "nll_loss": 2.1329703330993652, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.199848935008049, + "rewards/margins": 0.03814878687262535, + "rewards/rejected": -0.23799772560596466, + "step": 1 + }, + { + "epoch": 0.005522954780807732, + "grad_norm": 0.7381364107131958, + "learning_rate": 4.587155963302753e-08, + "log_odds_chosen": 0.4411306381225586, + "log_odds_ratio": -0.4988963305950165, + "logits/chosen": -0.6594648361206055, + "logits/rejected": -0.07251911610364914, + "logps/chosen": -2.0794589519500732, + "logps/rejected": -2.473759889602661, + "loss": 2.2463, + "nll_loss": 2.196385383605957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20794589817523956, + "rewards/margins": 0.039430104196071625, + "rewards/rejected": -0.24737600982189178, + "step": 2 + }, + { + "epoch": 0.008284432171211598, + "grad_norm": 0.6377604603767395, + "learning_rate": 9.174311926605506e-08, + "log_odds_chosen": 0.5233978033065796, + "log_odds_ratio": -0.4696895480155945, + "logits/chosen": -0.7436237931251526, + "logits/rejected": -0.04083387181162834, + "logps/chosen": -2.0129284858703613, + "logps/rejected": -2.4808387756347656, + "loss": 2.1678, + "nll_loss": 2.120811700820923, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20129284262657166, + "rewards/margins": 0.04679100960493088, + "rewards/rejected": -0.24808385968208313, + "step": 3 + }, + { + "epoch": 0.011045909561615464, + "grad_norm": 0.5183601379394531, + "learning_rate": 1.376146788990826e-07, + "log_odds_chosen": 0.4738181531429291, + "log_odds_ratio": -0.4919201731681824, + "logits/chosen": -0.5707842111587524, + "logits/rejected": -0.06899966299533844, + "logps/chosen": -2.006629467010498, + "logps/rejected": -2.429466724395752, + "loss": 2.1541, + "nll_loss": 2.104935646057129, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20066295564174652, + "rewards/margins": 0.04228372871875763, + "rewards/rejected": -0.24294668436050415, + "step": 4 + }, + { + "epoch": 0.013807386952019331, + "grad_norm": 0.5362528562545776, + "learning_rate": 1.8348623853211012e-07, + "log_odds_chosen": 0.47211968898773193, + "log_odds_ratio": -0.49660325050354004, + "logits/chosen": -0.5831207633018494, + "logits/rejected": -0.13220791518688202, + "logps/chosen": -1.9650582075119019, + "logps/rejected": -2.3817272186279297, + "loss": 2.1187, + "nll_loss": 2.0690417289733887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1965058296918869, + "rewards/margins": 0.04166688770055771, + "rewards/rejected": -0.23817269504070282, + "step": 5 + }, + { + "epoch": 0.016568864342423197, + "grad_norm": 0.5392005443572998, + "learning_rate": 2.2935779816513764e-07, + "log_odds_chosen": 0.3942825496196747, + "log_odds_ratio": -0.518440306186676, + "logits/chosen": -0.5348646640777588, + "logits/rejected": -0.23154743015766144, + "logps/chosen": -2.012246608734131, + "logps/rejected": -2.361220359802246, + "loss": 2.178, + "nll_loss": 2.126155376434326, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2012246698141098, + "rewards/margins": 0.03489736467599869, + "rewards/rejected": -0.2361220419406891, + "step": 6 + }, + { + "epoch": 0.019330341732827064, + "grad_norm": 0.5474534630775452, + "learning_rate": 2.752293577981652e-07, + "log_odds_chosen": 0.2622841000556946, + "log_odds_ratio": -0.576733410358429, + "logits/chosen": -0.5426779985427856, + "logits/rejected": -0.2341059297323227, + "logps/chosen": -2.074805736541748, + "logps/rejected": -2.310166120529175, + "loss": 2.2374, + "nll_loss": 2.179717779159546, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2074805647134781, + "rewards/margins": 0.023536043241620064, + "rewards/rejected": -0.231016606092453, + "step": 7 + }, + { + "epoch": 0.022091819123230928, + "grad_norm": 0.5940824747085571, + "learning_rate": 3.211009174311927e-07, + "log_odds_chosen": 0.41454389691352844, + "log_odds_ratio": -0.5088675618171692, + "logits/chosen": -0.45266416668891907, + "logits/rejected": -0.03741999715566635, + "logps/chosen": -2.081907033920288, + "logps/rejected": -2.452606439590454, + "loss": 2.2474, + "nll_loss": 2.19650936126709, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2081906944513321, + "rewards/margins": 0.037069931626319885, + "rewards/rejected": -0.24526062607765198, + "step": 8 + }, + { + "epoch": 0.024853296513634795, + "grad_norm": 0.5987945795059204, + "learning_rate": 3.6697247706422023e-07, + "log_odds_chosen": 0.666429340839386, + "log_odds_ratio": -0.42349112033843994, + "logits/chosen": -0.7504861950874329, + "logits/rejected": -0.16368302702903748, + "logps/chosen": -1.9702867269515991, + "logps/rejected": -2.566361427307129, + "loss": 2.1456, + "nll_loss": 2.10321307182312, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19702866673469543, + "rewards/margins": 0.05960746854543686, + "rewards/rejected": -0.2566361427307129, + "step": 9 + }, + { + "epoch": 0.027614773904038662, + "grad_norm": 0.7403894662857056, + "learning_rate": 4.128440366972478e-07, + "log_odds_chosen": 0.44253280758857727, + "log_odds_ratio": -0.502151370048523, + "logits/chosen": -0.7380926012992859, + "logits/rejected": -0.23479697108268738, + "logps/chosen": -2.0925681591033936, + "logps/rejected": -2.4903366565704346, + "loss": 2.2572, + "nll_loss": 2.206953287124634, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20925681293010712, + "rewards/margins": 0.03977686166763306, + "rewards/rejected": -0.24903367459774017, + "step": 10 + }, + { + "epoch": 0.030376251294442526, + "grad_norm": 0.6613215208053589, + "learning_rate": 4.587155963302753e-07, + "log_odds_chosen": 0.5335031151771545, + "log_odds_ratio": -0.4685918688774109, + "logits/chosen": -0.6707795262336731, + "logits/rejected": -0.13829460740089417, + "logps/chosen": -2.0549967288970947, + "logps/rejected": -2.5354185104370117, + "loss": 2.2247, + "nll_loss": 2.1778128147125244, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20549967885017395, + "rewards/margins": 0.04804220795631409, + "rewards/rejected": -0.25354188680648804, + "step": 11 + }, + { + "epoch": 0.03313772868484639, + "grad_norm": 0.5580596327781677, + "learning_rate": 5.045871559633028e-07, + "log_odds_chosen": 0.4023962616920471, + "log_odds_ratio": -0.5220973491668701, + "logits/chosen": -0.5904384851455688, + "logits/rejected": 0.0007063774392008781, + "logps/chosen": -1.958735704421997, + "logps/rejected": -2.3157620429992676, + "loss": 2.1455, + "nll_loss": 2.09328293800354, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19587357342243195, + "rewards/margins": 0.03570263832807541, + "rewards/rejected": -0.23157618939876556, + "step": 12 + }, + { + "epoch": 0.03589920607525026, + "grad_norm": 0.4629175662994385, + "learning_rate": 5.504587155963304e-07, + "log_odds_chosen": 0.4922281503677368, + "log_odds_ratio": -0.4793952405452728, + "logits/chosen": -0.489984929561615, + "logits/rejected": -0.08446945250034332, + "logps/chosen": -1.8884625434875488, + "logps/rejected": -2.318817377090454, + "loss": 2.0661, + "nll_loss": 2.0181963443756104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18884626030921936, + "rewards/margins": 0.043035492300987244, + "rewards/rejected": -0.2318817526102066, + "step": 13 + }, + { + "epoch": 0.03866068346565413, + "grad_norm": 0.6970926523208618, + "learning_rate": 5.963302752293579e-07, + "log_odds_chosen": 0.46570533514022827, + "log_odds_ratio": -0.49952778220176697, + "logits/chosen": -0.6894131898880005, + "logits/rejected": -0.010050175711512566, + "logps/chosen": -2.0864779949188232, + "logps/rejected": -2.503685235977173, + "loss": 2.2329, + "nll_loss": 2.182929277420044, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20864778757095337, + "rewards/margins": 0.041720740497112274, + "rewards/rejected": -0.25036853551864624, + "step": 14 + }, + { + "epoch": 0.04142216085605799, + "grad_norm": 0.7035249471664429, + "learning_rate": 6.422018348623854e-07, + "log_odds_chosen": 0.5717816948890686, + "log_odds_ratio": -0.44831815361976624, + "logits/chosen": -0.7732019424438477, + "logits/rejected": 0.13750173151493073, + "logps/chosen": -2.0055058002471924, + "logps/rejected": -2.513883590698242, + "loss": 2.1667, + "nll_loss": 2.1218552589416504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20055057108402252, + "rewards/margins": 0.05083777755498886, + "rewards/rejected": -0.2513883411884308, + "step": 15 + }, + { + "epoch": 0.044183638246461855, + "grad_norm": 0.489059180021286, + "learning_rate": 6.880733944954129e-07, + "log_odds_chosen": 0.45178472995758057, + "log_odds_ratio": -0.501604437828064, + "logits/chosen": -0.6046928763389587, + "logits/rejected": 0.05104057490825653, + "logps/chosen": -1.8943061828613281, + "logps/rejected": -2.2864151000976562, + "loss": 2.0731, + "nll_loss": 2.022980213165283, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18943063914775848, + "rewards/margins": 0.03921087831258774, + "rewards/rejected": -0.22864152491092682, + "step": 16 + }, + { + "epoch": 0.04694511563686572, + "grad_norm": 0.5798972249031067, + "learning_rate": 7.339449541284405e-07, + "log_odds_chosen": 0.521186113357544, + "log_odds_ratio": -0.4737667441368103, + "logits/chosen": -0.5957604646682739, + "logits/rejected": 0.05874314904212952, + "logps/chosen": -1.9864083528518677, + "logps/rejected": -2.4512085914611816, + "loss": 2.1331, + "nll_loss": 2.085712194442749, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1986408233642578, + "rewards/margins": 0.04648003727197647, + "rewards/rejected": -0.24512089788913727, + "step": 17 + }, + { + "epoch": 0.04970659302726959, + "grad_norm": 0.602294921875, + "learning_rate": 7.79816513761468e-07, + "log_odds_chosen": 0.6343832612037659, + "log_odds_ratio": -0.42983177304267883, + "logits/chosen": -0.6467751264572144, + "logits/rejected": -0.0914345234632492, + "logps/chosen": -2.020120143890381, + "logps/rejected": -2.5913381576538086, + "loss": 2.1654, + "nll_loss": 2.1224629878997803, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2020120471715927, + "rewards/margins": 0.05712177976965904, + "rewards/rejected": -0.25913381576538086, + "step": 18 + }, + { + "epoch": 0.05246807041767346, + "grad_norm": 0.6167095303535461, + "learning_rate": 8.256880733944956e-07, + "log_odds_chosen": 0.23837895691394806, + "log_odds_ratio": -0.5851647853851318, + "logits/chosen": -0.6367707848548889, + "logits/rejected": -0.06072646751999855, + "logps/chosen": -2.055112838745117, + "logps/rejected": -2.2656140327453613, + "loss": 2.2217, + "nll_loss": 2.163205146789551, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20551128685474396, + "rewards/margins": 0.021050114184617996, + "rewards/rejected": -0.22656141221523285, + "step": 19 + }, + { + "epoch": 0.055229547808077324, + "grad_norm": 0.5270997285842896, + "learning_rate": 8.71559633027523e-07, + "log_odds_chosen": 0.4399060010910034, + "log_odds_ratio": -0.5041620135307312, + "logits/chosen": -0.5500829815864563, + "logits/rejected": 0.014720816165208817, + "logps/chosen": -1.9720858335494995, + "logps/rejected": -2.3600573539733887, + "loss": 2.1571, + "nll_loss": 2.1066458225250244, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19720861315727234, + "rewards/margins": 0.038797151297330856, + "rewards/rejected": -0.2360057532787323, + "step": 20 + }, + { + "epoch": 0.057991025198481184, + "grad_norm": 0.639258086681366, + "learning_rate": 9.174311926605506e-07, + "log_odds_chosen": 0.35250768065452576, + "log_odds_ratio": -0.5405789017677307, + "logits/chosen": -0.45625555515289307, + "logits/rejected": -0.03890611231327057, + "logps/chosen": -2.1359739303588867, + "logps/rejected": -2.454291820526123, + "loss": 2.2807, + "nll_loss": 2.2266433238983154, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.21359741687774658, + "rewards/margins": 0.0318317785859108, + "rewards/rejected": -0.2454291731119156, + "step": 21 + }, + { + "epoch": 0.06075250258888505, + "grad_norm": 0.5805932283401489, + "learning_rate": 9.633027522935782e-07, + "log_odds_chosen": 0.5011261105537415, + "log_odds_ratio": -0.4887810945510864, + "logits/chosen": -0.6073933839797974, + "logits/rejected": -0.18206848204135895, + "logps/chosen": -1.9807653427124023, + "logps/rejected": -2.4255237579345703, + "loss": 2.1351, + "nll_loss": 2.0862059593200684, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1980765163898468, + "rewards/margins": 0.044475845992565155, + "rewards/rejected": -0.24255238473415375, + "step": 22 + }, + { + "epoch": 0.06351397997928893, + "grad_norm": 0.5500138998031616, + "learning_rate": 1.0091743119266057e-06, + "log_odds_chosen": 0.3974594175815582, + "log_odds_ratio": -0.5229506492614746, + "logits/chosen": -0.5682663321495056, + "logits/rejected": -0.3609941005706787, + "logps/chosen": -1.963416576385498, + "logps/rejected": -2.318310499191284, + "loss": 2.1453, + "nll_loss": 2.0929956436157227, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19634166359901428, + "rewards/margins": 0.03548937663435936, + "rewards/rejected": -0.23183104395866394, + "step": 23 + }, + { + "epoch": 0.06627545736969279, + "grad_norm": 0.5702619552612305, + "learning_rate": 1.055045871559633e-06, + "log_odds_chosen": 0.522091805934906, + "log_odds_ratio": -0.4669547379016876, + "logits/chosen": -0.5776705741882324, + "logits/rejected": 0.03815801814198494, + "logps/chosen": -1.9660505056381226, + "logps/rejected": -2.429518461227417, + "loss": 2.1353, + "nll_loss": 2.0886194705963135, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19660505652427673, + "rewards/margins": 0.04634679853916168, + "rewards/rejected": -0.24295185506343842, + "step": 24 + }, + { + "epoch": 0.06903693476009665, + "grad_norm": 0.6908774375915527, + "learning_rate": 1.1009174311926608e-06, + "log_odds_chosen": 0.6555817723274231, + "log_odds_ratio": -0.42638978362083435, + "logits/chosen": -0.6369448304176331, + "logits/rejected": -0.0680353045463562, + "logps/chosen": -1.996623158454895, + "logps/rejected": -2.585740327835083, + "loss": 2.157, + "nll_loss": 2.1143739223480225, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19966234266757965, + "rewards/margins": 0.058911725878715515, + "rewards/rejected": -0.25857406854629517, + "step": 25 + }, + { + "epoch": 0.07179841215050052, + "grad_norm": 0.6247107982635498, + "learning_rate": 1.1467889908256882e-06, + "log_odds_chosen": 0.37967437505722046, + "log_odds_ratio": -0.528645396232605, + "logits/chosen": -0.5880488157272339, + "logits/rejected": -0.11361770331859589, + "logps/chosen": -2.0194289684295654, + "logps/rejected": -2.3577256202697754, + "loss": 2.1956, + "nll_loss": 2.142758369445801, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20194292068481445, + "rewards/margins": 0.03382965177297592, + "rewards/rejected": -0.23577255010604858, + "step": 26 + }, + { + "epoch": 0.07455988954090438, + "grad_norm": 0.44772788882255554, + "learning_rate": 1.1926605504587159e-06, + "log_odds_chosen": 0.4675738513469696, + "log_odds_ratio": -0.4964170753955841, + "logits/chosen": -0.434501588344574, + "logits/rejected": -0.17547550797462463, + "logps/chosen": -1.865759253501892, + "logps/rejected": -2.272789239883423, + "loss": 2.0462, + "nll_loss": 1.996600866317749, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18657593429088593, + "rewards/margins": 0.04070296883583069, + "rewards/rejected": -0.2272789031267166, + "step": 27 + }, + { + "epoch": 0.07732136693130826, + "grad_norm": 0.5303645730018616, + "learning_rate": 1.2385321100917433e-06, + "log_odds_chosen": 0.5267918109893799, + "log_odds_ratio": -0.46872708201408386, + "logits/chosen": -0.5293477773666382, + "logits/rejected": -0.20712637901306152, + "logps/chosen": -1.9557033777236938, + "logps/rejected": -2.422588586807251, + "loss": 2.123, + "nll_loss": 2.0761377811431885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19557033479213715, + "rewards/margins": 0.04668852686882019, + "rewards/rejected": -0.24225887656211853, + "step": 28 + }, + { + "epoch": 0.08008284432171212, + "grad_norm": 0.558870792388916, + "learning_rate": 1.2844036697247707e-06, + "log_odds_chosen": 0.4761933982372284, + "log_odds_ratio": -0.49172335863113403, + "logits/chosen": -0.5614610314369202, + "logits/rejected": -0.03958575427532196, + "logps/chosen": -2.044667959213257, + "logps/rejected": -2.4699392318725586, + "loss": 2.2024, + "nll_loss": 2.153231143951416, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2044668048620224, + "rewards/margins": 0.042527101933956146, + "rewards/rejected": -0.24699391424655914, + "step": 29 + }, + { + "epoch": 0.08284432171211598, + "grad_norm": 0.6042240262031555, + "learning_rate": 1.3302752293577984e-06, + "log_odds_chosen": 0.6456740498542786, + "log_odds_ratio": -0.4289007782936096, + "logits/chosen": -0.5478062629699707, + "logits/rejected": -0.0002663079649209976, + "logps/chosen": -2.02502703666687, + "logps/rejected": -2.60587477684021, + "loss": 2.1737, + "nll_loss": 2.1308515071868896, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20250271260738373, + "rewards/margins": 0.058084748685359955, + "rewards/rejected": -0.2605874538421631, + "step": 30 + }, + { + "epoch": 0.08560579910251985, + "grad_norm": 0.6082971692085266, + "learning_rate": 1.3761467889908258e-06, + "log_odds_chosen": 0.5572243928909302, + "log_odds_ratio": -0.45846420526504517, + "logits/chosen": -0.5786283016204834, + "logits/rejected": -0.06976839900016785, + "logps/chosen": -2.0147552490234375, + "logps/rejected": -2.5146052837371826, + "loss": 2.1834, + "nll_loss": 2.1375184059143066, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20147554576396942, + "rewards/margins": 0.049984999001026154, + "rewards/rejected": -0.2514605224132538, + "step": 31 + }, + { + "epoch": 0.08836727649292371, + "grad_norm": 0.4792933762073517, + "learning_rate": 1.4220183486238535e-06, + "log_odds_chosen": 0.43500036001205444, + "log_odds_ratio": -0.5046517252922058, + "logits/chosen": -0.4258999824523926, + "logits/rejected": 0.003258749842643738, + "logps/chosen": -2.011561870574951, + "logps/rejected": -2.398580312728882, + "loss": 2.1831, + "nll_loss": 2.132627010345459, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20115619897842407, + "rewards/margins": 0.03870181739330292, + "rewards/rejected": -0.239858016371727, + "step": 32 + }, + { + "epoch": 0.09112875388332758, + "grad_norm": 0.5385621786117554, + "learning_rate": 1.467889908256881e-06, + "log_odds_chosen": 0.573083758354187, + "log_odds_ratio": -0.4524250328540802, + "logits/chosen": -0.4552776515483856, + "logits/rejected": -0.06403280049562454, + "logps/chosen": -1.9241302013397217, + "logps/rejected": -2.4318630695343018, + "loss": 2.0919, + "nll_loss": 2.046658754348755, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19241301715373993, + "rewards/margins": 0.05077328532934189, + "rewards/rejected": -0.24318630993366241, + "step": 33 + }, + { + "epoch": 0.09389023127373144, + "grad_norm": 0.4870143234729767, + "learning_rate": 1.5137614678899084e-06, + "log_odds_chosen": 0.6046110391616821, + "log_odds_ratio": -0.4510130286216736, + "logits/chosen": -0.40763065218925476, + "logits/rejected": -0.08770006895065308, + "logps/chosen": -1.922775387763977, + "logps/rejected": -2.462465524673462, + "loss": 2.0741, + "nll_loss": 2.028975009918213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19227753579616547, + "rewards/margins": 0.053969040513038635, + "rewards/rejected": -0.2462465912103653, + "step": 34 + }, + { + "epoch": 0.09665170866413532, + "grad_norm": 0.6609561443328857, + "learning_rate": 1.559633027522936e-06, + "log_odds_chosen": 0.40986764430999756, + "log_odds_ratio": -0.5126968026161194, + "logits/chosen": -0.8081121444702148, + "logits/rejected": -0.20409950613975525, + "logps/chosen": -1.9795390367507935, + "logps/rejected": -2.3391332626342773, + "loss": 2.152, + "nll_loss": 2.1007797718048096, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19795390963554382, + "rewards/margins": 0.03595941513776779, + "rewards/rejected": -0.23391331732273102, + "step": 35 + }, + { + "epoch": 0.09941318605453918, + "grad_norm": 0.6118403673171997, + "learning_rate": 1.6055045871559635e-06, + "log_odds_chosen": 0.603810727596283, + "log_odds_ratio": -0.4452260434627533, + "logits/chosen": -0.6869708299636841, + "logits/rejected": 0.01393081247806549, + "logps/chosen": -1.9751290082931519, + "logps/rejected": -2.515267848968506, + "loss": 2.1348, + "nll_loss": 2.0902533531188965, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1975128948688507, + "rewards/margins": 0.05401391163468361, + "rewards/rejected": -0.251526802778244, + "step": 36 + }, + { + "epoch": 0.10217466344494304, + "grad_norm": 0.6636885404586792, + "learning_rate": 1.6513761467889911e-06, + "log_odds_chosen": 0.5960386991500854, + "log_odds_ratio": -0.44556429982185364, + "logits/chosen": -0.5324239134788513, + "logits/rejected": -0.1962709128856659, + "logps/chosen": -2.1164088249206543, + "logps/rejected": -2.6588618755340576, + "loss": 2.2904, + "nll_loss": 2.2458741664886475, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2116408795118332, + "rewards/margins": 0.054245319217443466, + "rewards/rejected": -0.26588618755340576, + "step": 37 + }, + { + "epoch": 0.10493614083534691, + "grad_norm": 0.5414693355560303, + "learning_rate": 1.6972477064220186e-06, + "log_odds_chosen": 0.4305243492126465, + "log_odds_ratio": -0.5040389895439148, + "logits/chosen": -0.5475431680679321, + "logits/rejected": -0.15733303129673004, + "logps/chosen": -1.9466073513031006, + "logps/rejected": -2.3275275230407715, + "loss": 2.1303, + "nll_loss": 2.079880475997925, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1946607381105423, + "rewards/margins": 0.03809202462434769, + "rewards/rejected": -0.2327527403831482, + "step": 38 + }, + { + "epoch": 0.10769761822575077, + "grad_norm": 0.7092136740684509, + "learning_rate": 1.743119266055046e-06, + "log_odds_chosen": 0.49458950757980347, + "log_odds_ratio": -0.499477356672287, + "logits/chosen": -0.47480347752571106, + "logits/rejected": 0.009076721966266632, + "logps/chosen": -2.077540159225464, + "logps/rejected": -2.527543067932129, + "loss": 2.2296, + "nll_loss": 2.1796536445617676, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20775403082370758, + "rewards/margins": 0.04500027373433113, + "rewards/rejected": -0.2527543008327484, + "step": 39 + }, + { + "epoch": 0.11045909561615465, + "grad_norm": 0.7840990424156189, + "learning_rate": 1.7889908256880737e-06, + "log_odds_chosen": 0.27720558643341064, + "log_odds_ratio": -0.5738192796707153, + "logits/chosen": -0.6754797101020813, + "logits/rejected": -0.1666824221611023, + "logps/chosen": -2.097449779510498, + "logps/rejected": -2.346257448196411, + "loss": 2.2706, + "nll_loss": 2.2131741046905518, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20974498987197876, + "rewards/margins": 0.02488076500594616, + "rewards/rejected": -0.23462577164173126, + "step": 40 + }, + { + "epoch": 0.11322057300655851, + "grad_norm": 0.49594300985336304, + "learning_rate": 1.8348623853211011e-06, + "log_odds_chosen": 0.4892931878566742, + "log_odds_ratio": -0.4875307083129883, + "logits/chosen": -0.5313754081726074, + "logits/rejected": -0.11813461780548096, + "logps/chosen": -1.9190640449523926, + "logps/rejected": -2.3516781330108643, + "loss": 2.0847, + "nll_loss": 2.035940647125244, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1919064074754715, + "rewards/margins": 0.043261416256427765, + "rewards/rejected": -0.23516784608364105, + "step": 41 + }, + { + "epoch": 0.11598205039696237, + "grad_norm": 0.6706519722938538, + "learning_rate": 1.8807339449541288e-06, + "log_odds_chosen": 0.3553638756275177, + "log_odds_ratio": -0.5406981706619263, + "logits/chosen": -0.6044270992279053, + "logits/rejected": -0.1877760887145996, + "logps/chosen": -2.110844850540161, + "logps/rejected": -2.4292314052581787, + "loss": 2.2811, + "nll_loss": 2.2269935607910156, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2110844999551773, + "rewards/margins": 0.031838610768318176, + "rewards/rejected": -0.24292311072349548, + "step": 42 + }, + { + "epoch": 0.11874352778736624, + "grad_norm": 0.569320023059845, + "learning_rate": 1.9266055045871564e-06, + "log_odds_chosen": 0.4219287633895874, + "log_odds_ratio": -0.5136047005653381, + "logits/chosen": -0.564371645450592, + "logits/rejected": -0.1906774342060089, + "logps/chosen": -1.9711859226226807, + "logps/rejected": -2.341031312942505, + "loss": 2.1533, + "nll_loss": 2.101966381072998, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1971185803413391, + "rewards/margins": 0.036984533071517944, + "rewards/rejected": -0.23410312831401825, + "step": 43 + }, + { + "epoch": 0.1215050051777701, + "grad_norm": 0.6019275188446045, + "learning_rate": 1.9724770642201837e-06, + "log_odds_chosen": 0.3504191040992737, + "log_odds_ratio": -0.5448323488235474, + "logits/chosen": -0.5221514105796814, + "logits/rejected": -0.2111213058233261, + "logps/chosen": -2.0630786418914795, + "logps/rejected": -2.3758115768432617, + "loss": 2.226, + "nll_loss": 2.1714861392974854, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20630787312984467, + "rewards/margins": 0.03127329424023628, + "rewards/rejected": -0.23758116364479065, + "step": 44 + }, + { + "epoch": 0.12426648256817398, + "grad_norm": 0.5997087955474854, + "learning_rate": 2.0183486238532113e-06, + "log_odds_chosen": 0.6440756916999817, + "log_odds_ratio": -0.42918679118156433, + "logits/chosen": -0.6537622213363647, + "logits/rejected": -0.3867274224758148, + "logps/chosen": -1.964981198310852, + "logps/rejected": -2.541144371032715, + "loss": 2.1164, + "nll_loss": 2.073434352874756, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1964981108903885, + "rewards/margins": 0.057616353034973145, + "rewards/rejected": -0.25411444902420044, + "step": 45 + }, + { + "epoch": 0.12702795995857785, + "grad_norm": 0.5181168913841248, + "learning_rate": 2.064220183486239e-06, + "log_odds_chosen": 0.564142107963562, + "log_odds_ratio": -0.4582359790802002, + "logits/chosen": -0.5351958870887756, + "logits/rejected": -0.1501241773366928, + "logps/chosen": -1.84321928024292, + "logps/rejected": -2.331892967224121, + "loss": 2.0016, + "nll_loss": 1.9557558298110962, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18432192504405975, + "rewards/margins": 0.04886738583445549, + "rewards/rejected": -0.23318932950496674, + "step": 46 + }, + { + "epoch": 0.1297894373489817, + "grad_norm": 0.5089595913887024, + "learning_rate": 2.110091743119266e-06, + "log_odds_chosen": 0.4004448652267456, + "log_odds_ratio": -0.5262866616249084, + "logits/chosen": -0.4425760507583618, + "logits/rejected": -0.17353124916553497, + "logps/chosen": -2.045093059539795, + "logps/rejected": -2.4056942462921143, + "loss": 2.2153, + "nll_loss": 2.162646770477295, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20450931787490845, + "rewards/margins": 0.03606009483337402, + "rewards/rejected": -0.24056941270828247, + "step": 47 + }, + { + "epoch": 0.13255091473938557, + "grad_norm": 0.6208946108818054, + "learning_rate": 2.155963302752294e-06, + "log_odds_chosen": 0.18348652124404907, + "log_odds_ratio": -0.6307563781738281, + "logits/chosen": -0.45777827501296997, + "logits/rejected": -0.11396709084510803, + "logps/chosen": -2.1183199882507324, + "logps/rejected": -2.2930657863616943, + "loss": 2.2819, + "nll_loss": 2.21886944770813, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.21183200180530548, + "rewards/margins": 0.01747458055615425, + "rewards/rejected": -0.22930656373500824, + "step": 48 + }, + { + "epoch": 0.13531239212978943, + "grad_norm": 0.6294072866439819, + "learning_rate": 2.2018348623853215e-06, + "log_odds_chosen": 0.6346040964126587, + "log_odds_ratio": -0.43739554286003113, + "logits/chosen": -0.542172908782959, + "logits/rejected": -0.039900042116642, + "logps/chosen": -2.052166223526001, + "logps/rejected": -2.628195285797119, + "loss": 2.2026, + "nll_loss": 2.1588120460510254, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.205216646194458, + "rewards/margins": 0.05760287865996361, + "rewards/rejected": -0.2628195285797119, + "step": 49 + }, + { + "epoch": 0.1380738695201933, + "grad_norm": 0.6679416298866272, + "learning_rate": 2.2477064220183487e-06, + "log_odds_chosen": 0.5639493465423584, + "log_odds_ratio": -0.47183454036712646, + "logits/chosen": -0.6221768856048584, + "logits/rejected": -0.0222525242716074, + "logps/chosen": -2.010540008544922, + "logps/rejected": -2.51875901222229, + "loss": 2.1569, + "nll_loss": 2.1097309589385986, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20105400681495667, + "rewards/margins": 0.05082192271947861, + "rewards/rejected": -0.25187593698501587, + "step": 50 + }, + { + "epoch": 0.14083534691059718, + "grad_norm": 0.6739494800567627, + "learning_rate": 2.2935779816513764e-06, + "log_odds_chosen": 0.4806385040283203, + "log_odds_ratio": -0.486289381980896, + "logits/chosen": -0.5340446829795837, + "logits/rejected": -0.3175644278526306, + "logps/chosen": -2.032139301300049, + "logps/rejected": -2.4611167907714844, + "loss": 2.1976, + "nll_loss": 2.1489548683166504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2032139152288437, + "rewards/margins": 0.04289776086807251, + "rewards/rejected": -0.2461116760969162, + "step": 51 + }, + { + "epoch": 0.14359682430100104, + "grad_norm": 0.5641347765922546, + "learning_rate": 2.339449541284404e-06, + "log_odds_chosen": 0.4302893579006195, + "log_odds_ratio": -0.5059407353401184, + "logits/chosen": -0.4652354121208191, + "logits/rejected": -0.14794279634952545, + "logps/chosen": -1.98146390914917, + "logps/rejected": -2.362124443054199, + "loss": 2.1395, + "nll_loss": 2.0888633728027344, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19814638793468475, + "rewards/margins": 0.0380660817027092, + "rewards/rejected": -0.23621246218681335, + "step": 52 + }, + { + "epoch": 0.1463583016914049, + "grad_norm": 0.6226781606674194, + "learning_rate": 2.3853211009174317e-06, + "log_odds_chosen": 0.38262999057769775, + "log_odds_ratio": -0.5264566540718079, + "logits/chosen": -0.601564884185791, + "logits/rejected": -0.23214676976203918, + "logps/chosen": -1.9446452856063843, + "logps/rejected": -2.2811543941497803, + "loss": 2.1236, + "nll_loss": 2.070988178253174, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1944645196199417, + "rewards/margins": 0.03365091234445572, + "rewards/rejected": -0.22811545431613922, + "step": 53 + }, + { + "epoch": 0.14911977908180876, + "grad_norm": 0.576812207698822, + "learning_rate": 2.431192660550459e-06, + "log_odds_chosen": 0.3935718238353729, + "log_odds_ratio": -0.5239484310150146, + "logits/chosen": -0.5574474930763245, + "logits/rejected": -0.1828337013721466, + "logps/chosen": -1.9322319030761719, + "logps/rejected": -2.280305862426758, + "loss": 2.1135, + "nll_loss": 2.061114549636841, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19322317838668823, + "rewards/margins": 0.03480737283825874, + "rewards/rejected": -0.22803056240081787, + "step": 54 + }, + { + "epoch": 0.15188125647221262, + "grad_norm": 0.5661758184432983, + "learning_rate": 2.4770642201834866e-06, + "log_odds_chosen": 0.39002981781959534, + "log_odds_ratio": -0.5207180380821228, + "logits/chosen": -0.6281000971794128, + "logits/rejected": -0.12741385400295258, + "logps/chosen": -1.9887436628341675, + "logps/rejected": -2.3331456184387207, + "loss": 2.1681, + "nll_loss": 2.115997314453125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.198874369263649, + "rewards/margins": 0.03444019705057144, + "rewards/rejected": -0.23331457376480103, + "step": 55 + }, + { + "epoch": 0.1546427338626165, + "grad_norm": 0.6655378937721252, + "learning_rate": 2.522935779816514e-06, + "log_odds_chosen": 0.6584038138389587, + "log_odds_ratio": -0.427642822265625, + "logits/chosen": -0.6279682517051697, + "logits/rejected": -0.12986338138580322, + "logps/chosen": -1.9852056503295898, + "logps/rejected": -2.5761494636535645, + "loss": 2.1584, + "nll_loss": 2.115626335144043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19852055609226227, + "rewards/margins": 0.059094399213790894, + "rewards/rejected": -0.25761497020721436, + "step": 56 + }, + { + "epoch": 0.15740421125302037, + "grad_norm": 0.48872971534729004, + "learning_rate": 2.5688073394495415e-06, + "log_odds_chosen": 0.42161181569099426, + "log_odds_ratio": -0.514995276927948, + "logits/chosen": -0.49338769912719727, + "logits/rejected": -0.07346588373184204, + "logps/chosen": -1.8429884910583496, + "logps/rejected": -2.2116591930389404, + "loss": 2.014, + "nll_loss": 1.9625122547149658, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18429884314537048, + "rewards/margins": 0.036867089569568634, + "rewards/rejected": -0.22116592526435852, + "step": 57 + }, + { + "epoch": 0.16016568864342423, + "grad_norm": 0.6582737565040588, + "learning_rate": 2.6146788990825687e-06, + "log_odds_chosen": 0.5138990879058838, + "log_odds_ratio": -0.4847075939178467, + "logits/chosen": -0.5234618186950684, + "logits/rejected": -0.12736433744430542, + "logps/chosen": -1.9835774898529053, + "logps/rejected": -2.4381330013275146, + "loss": 2.1515, + "nll_loss": 2.1030287742614746, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19835779070854187, + "rewards/margins": 0.045455530285835266, + "rewards/rejected": -0.24381330609321594, + "step": 58 + }, + { + "epoch": 0.1629271660338281, + "grad_norm": 0.5795738697052002, + "learning_rate": 2.6605504587155968e-06, + "log_odds_chosen": 0.2905232906341553, + "log_odds_ratio": -0.5645147562026978, + "logits/chosen": -0.47340482473373413, + "logits/rejected": -0.21826684474945068, + "logps/chosen": -1.9716663360595703, + "logps/rejected": -2.227067470550537, + "loss": 2.1514, + "nll_loss": 2.0949792861938477, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19716663658618927, + "rewards/margins": 0.025540094822645187, + "rewards/rejected": -0.22270673513412476, + "step": 59 + }, + { + "epoch": 0.16568864342423195, + "grad_norm": 0.5678736567497253, + "learning_rate": 2.706422018348624e-06, + "log_odds_chosen": 0.3714035749435425, + "log_odds_ratio": -0.5299574136734009, + "logits/chosen": -0.45862269401550293, + "logits/rejected": 0.08986371755599976, + "logps/chosen": -2.004216432571411, + "logps/rejected": -2.333292245864868, + "loss": 2.1781, + "nll_loss": 2.125133514404297, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20042164623737335, + "rewards/margins": 0.032907579094171524, + "rewards/rejected": -0.23332923650741577, + "step": 60 + }, + { + "epoch": 0.16845012081463584, + "grad_norm": 0.5007438063621521, + "learning_rate": 2.7522935779816517e-06, + "log_odds_chosen": 0.667113721370697, + "log_odds_ratio": -0.4193596839904785, + "logits/chosen": -0.5191382765769958, + "logits/rejected": -0.19058826565742493, + "logps/chosen": -1.917590856552124, + "logps/rejected": -2.5098061561584473, + "loss": 2.0632, + "nll_loss": 2.021275281906128, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1917591094970703, + "rewards/margins": 0.0592215433716774, + "rewards/rejected": -0.2509806454181671, + "step": 61 + }, + { + "epoch": 0.1712115982050397, + "grad_norm": 0.6429389119148254, + "learning_rate": 2.798165137614679e-06, + "log_odds_chosen": 0.41257914900779724, + "log_odds_ratio": -0.5154306888580322, + "logits/chosen": -0.45402857661247253, + "logits/rejected": 0.11148527264595032, + "logps/chosen": -2.081867218017578, + "logps/rejected": -2.45068359375, + "loss": 2.2481, + "nll_loss": 2.1965107917785645, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20818671584129333, + "rewards/margins": 0.03688164800405502, + "rewards/rejected": -0.24506837129592896, + "step": 62 + }, + { + "epoch": 0.17397307559544356, + "grad_norm": 0.5388195514678955, + "learning_rate": 2.844036697247707e-06, + "log_odds_chosen": 0.5105969905853271, + "log_odds_ratio": -0.4755265414714813, + "logits/chosen": -0.46547558903694153, + "logits/rejected": -0.1431473195552826, + "logps/chosen": -1.9948811531066895, + "logps/rejected": -2.448573589324951, + "loss": 2.1454, + "nll_loss": 2.0978972911834717, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19948811829090118, + "rewards/margins": 0.0453692302107811, + "rewards/rejected": -0.24485734105110168, + "step": 63 + }, + { + "epoch": 0.17673455298584742, + "grad_norm": 0.4911787211894989, + "learning_rate": 2.8899082568807342e-06, + "log_odds_chosen": 0.5747222304344177, + "log_odds_ratio": -0.4552631676197052, + "logits/chosen": -0.5521761178970337, + "logits/rejected": -0.294252872467041, + "logps/chosen": -1.9650574922561646, + "logps/rejected": -2.4793307781219482, + "loss": 2.1135, + "nll_loss": 2.067988634109497, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19650575518608093, + "rewards/margins": 0.051427341997623444, + "rewards/rejected": -0.24793308973312378, + "step": 64 + }, + { + "epoch": 0.17949603037625128, + "grad_norm": 0.533399760723114, + "learning_rate": 2.935779816513762e-06, + "log_odds_chosen": 0.3648831844329834, + "log_odds_ratio": -0.5328130722045898, + "logits/chosen": -0.4136759638786316, + "logits/rejected": -0.046070147305727005, + "logps/chosen": -1.8867706060409546, + "logps/rejected": -2.202749729156494, + "loss": 2.0708, + "nll_loss": 2.0175259113311768, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18867707252502441, + "rewards/margins": 0.031597916036844254, + "rewards/rejected": -0.22027498483657837, + "step": 65 + }, + { + "epoch": 0.18225750776665517, + "grad_norm": 0.5182326436042786, + "learning_rate": 2.981651376146789e-06, + "log_odds_chosen": 0.5400457382202148, + "log_odds_ratio": -0.48501351475715637, + "logits/chosen": -0.6035399436950684, + "logits/rejected": -0.1548141986131668, + "logps/chosen": -1.8529176712036133, + "logps/rejected": -2.325399398803711, + "loss": 2.022, + "nll_loss": 1.973459243774414, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18529179692268372, + "rewards/margins": 0.04724816977977753, + "rewards/rejected": -0.23253995180130005, + "step": 66 + }, + { + "epoch": 0.18501898515705903, + "grad_norm": 0.5337976813316345, + "learning_rate": 3.0275229357798168e-06, + "log_odds_chosen": 0.33741873502731323, + "log_odds_ratio": -0.5426305532455444, + "logits/chosen": -0.39523714780807495, + "logits/rejected": 0.0008284337818622589, + "logps/chosen": -1.997262716293335, + "logps/rejected": -2.2943520545959473, + "loss": 2.1433, + "nll_loss": 2.0890326499938965, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19972628355026245, + "rewards/margins": 0.029708925634622574, + "rewards/rejected": -0.22943520545959473, + "step": 67 + }, + { + "epoch": 0.1877804625474629, + "grad_norm": 0.5520625114440918, + "learning_rate": 3.073394495412844e-06, + "log_odds_chosen": 0.4403454065322876, + "log_odds_ratio": -0.5091642141342163, + "logits/chosen": -0.24377387762069702, + "logits/rejected": 0.08138424903154373, + "logps/chosen": -2.0106263160705566, + "logps/rejected": -2.402209758758545, + "loss": 2.2053, + "nll_loss": 2.1543631553649902, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2010626494884491, + "rewards/margins": 0.039158351719379425, + "rewards/rejected": -0.24022099375724792, + "step": 68 + }, + { + "epoch": 0.19054193993786675, + "grad_norm": 0.650439441204071, + "learning_rate": 3.119266055045872e-06, + "log_odds_chosen": 0.5253904461860657, + "log_odds_ratio": -0.4741411507129669, + "logits/chosen": -0.4677438735961914, + "logits/rejected": -0.013416798785328865, + "logps/chosen": -2.077601909637451, + "logps/rejected": -2.5524415969848633, + "loss": 2.2423, + "nll_loss": 2.194887161254883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20776019990444183, + "rewards/margins": 0.04748394340276718, + "rewards/rejected": -0.2552441358566284, + "step": 69 + }, + { + "epoch": 0.19330341732827064, + "grad_norm": 0.48958319425582886, + "learning_rate": 3.1651376146788993e-06, + "log_odds_chosen": 0.5917057991027832, + "log_odds_ratio": -0.45470088720321655, + "logits/chosen": -0.36797553300857544, + "logits/rejected": -0.3181126117706299, + "logps/chosen": -2.022463321685791, + "logps/rejected": -2.5544497966766357, + "loss": 2.1697, + "nll_loss": 2.124249219894409, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20224635303020477, + "rewards/margins": 0.05319864675402641, + "rewards/rejected": -0.2554450035095215, + "step": 70 + }, + { + "epoch": 0.1960648947186745, + "grad_norm": 0.6293954253196716, + "learning_rate": 3.211009174311927e-06, + "log_odds_chosen": 0.44158488512039185, + "log_odds_ratio": -0.5029429793357849, + "logits/chosen": -0.4253910481929779, + "logits/rejected": -0.14389574527740479, + "logps/chosen": -1.9857234954833984, + "logps/rejected": -2.379268169403076, + "loss": 2.17, + "nll_loss": 2.119678497314453, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19857235252857208, + "rewards/margins": 0.03935447335243225, + "rewards/rejected": -0.23792681097984314, + "step": 71 + }, + { + "epoch": 0.19882637210907836, + "grad_norm": 0.6024268269538879, + "learning_rate": 3.256880733944954e-06, + "log_odds_chosen": 0.5992636680603027, + "log_odds_ratio": -0.4583205580711365, + "logits/chosen": -0.3624473810195923, + "logits/rejected": -0.16807770729064941, + "logps/chosen": -1.8936195373535156, + "logps/rejected": -2.403137683868408, + "loss": 2.0566, + "nll_loss": 2.0107438564300537, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18936194479465485, + "rewards/margins": 0.05095181241631508, + "rewards/rejected": -0.24031376838684082, + "step": 72 + }, + { + "epoch": 0.20158784949948222, + "grad_norm": 0.537467360496521, + "learning_rate": 3.3027522935779823e-06, + "log_odds_chosen": 0.38986673951148987, + "log_odds_ratio": -0.518619179725647, + "logits/chosen": -0.28752538561820984, + "logits/rejected": -0.08893117308616638, + "logps/chosen": -2.056885242462158, + "logps/rejected": -2.4047675132751465, + "loss": 2.2167, + "nll_loss": 2.1648688316345215, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2056885063648224, + "rewards/margins": 0.03478822112083435, + "rewards/rejected": -0.24047674238681793, + "step": 73 + }, + { + "epoch": 0.20434932688988608, + "grad_norm": 0.48359447717666626, + "learning_rate": 3.3486238532110095e-06, + "log_odds_chosen": 0.39413708448410034, + "log_odds_ratio": -0.5191097259521484, + "logits/chosen": -0.28818562626838684, + "logits/rejected": -0.17117023468017578, + "logps/chosen": -1.9585490226745605, + "logps/rejected": -2.3046271800994873, + "loss": 2.1453, + "nll_loss": 2.093356132507324, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19585487246513367, + "rewards/margins": 0.03460782766342163, + "rewards/rejected": -0.2304627150297165, + "step": 74 + }, + { + "epoch": 0.20711080428028997, + "grad_norm": 0.6197869777679443, + "learning_rate": 3.394495412844037e-06, + "log_odds_chosen": 0.4853059649467468, + "log_odds_ratio": -0.487282931804657, + "logits/chosen": -0.4553055763244629, + "logits/rejected": -0.07368504256010056, + "logps/chosen": -1.9564940929412842, + "logps/rejected": -2.38771653175354, + "loss": 2.1426, + "nll_loss": 2.093902826309204, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1956494152545929, + "rewards/margins": 0.04312223941087723, + "rewards/rejected": -0.23877164721488953, + "step": 75 + }, + { + "epoch": 0.20987228167069383, + "grad_norm": 0.5478652119636536, + "learning_rate": 3.4403669724770644e-06, + "log_odds_chosen": 0.4639507532119751, + "log_odds_ratio": -0.4932023882865906, + "logits/chosen": -0.30065810680389404, + "logits/rejected": -0.25181248784065247, + "logps/chosen": -2.044691562652588, + "logps/rejected": -2.460970401763916, + "loss": 2.1933, + "nll_loss": 2.1440277099609375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20446917414665222, + "rewards/margins": 0.04162788391113281, + "rewards/rejected": -0.24609704315662384, + "step": 76 + }, + { + "epoch": 0.2126337590610977, + "grad_norm": 0.5865225195884705, + "learning_rate": 3.486238532110092e-06, + "log_odds_chosen": 0.40864938497543335, + "log_odds_ratio": -0.5256964564323425, + "logits/chosen": -0.414636492729187, + "logits/rejected": -0.1841173619031906, + "logps/chosen": -2.0234270095825195, + "logps/rejected": -2.3900880813598633, + "loss": 2.2079, + "nll_loss": 2.155355215072632, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2023427039384842, + "rewards/margins": 0.03666612505912781, + "rewards/rejected": -0.239008828997612, + "step": 77 + }, + { + "epoch": 0.21539523645150155, + "grad_norm": 0.5245475769042969, + "learning_rate": 3.5321100917431193e-06, + "log_odds_chosen": 0.3867124617099762, + "log_odds_ratio": -0.5356163382530212, + "logits/chosen": -0.22849449515342712, + "logits/rejected": 0.04786547273397446, + "logps/chosen": -2.047503709793091, + "logps/rejected": -2.391859531402588, + "loss": 2.2124, + "nll_loss": 2.1587889194488525, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20475035905838013, + "rewards/margins": 0.03443560749292374, + "rewards/rejected": -0.23918597400188446, + "step": 78 + }, + { + "epoch": 0.2181567138419054, + "grad_norm": 0.4416508972644806, + "learning_rate": 3.5779816513761473e-06, + "log_odds_chosen": 0.457964152097702, + "log_odds_ratio": -0.49229007959365845, + "logits/chosen": -0.24249830842018127, + "logits/rejected": -0.1156221255660057, + "logps/chosen": -1.9371097087860107, + "logps/rejected": -2.3405327796936035, + "loss": 2.1264, + "nll_loss": 2.077136516571045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19371098279953003, + "rewards/margins": 0.0403422936797142, + "rewards/rejected": -0.23405326902866364, + "step": 79 + }, + { + "epoch": 0.2209181912323093, + "grad_norm": 0.511163055896759, + "learning_rate": 3.6238532110091746e-06, + "log_odds_chosen": 0.3918084502220154, + "log_odds_ratio": -0.527028501033783, + "logits/chosen": -0.3217644989490509, + "logits/rejected": -0.12379680573940277, + "logps/chosen": -1.9086521863937378, + "logps/rejected": -2.250434160232544, + "loss": 2.0875, + "nll_loss": 2.034804105758667, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19086521863937378, + "rewards/margins": 0.03417817875742912, + "rewards/rejected": -0.2250434011220932, + "step": 80 + }, + { + "epoch": 0.22367966862271316, + "grad_norm": 0.48532167077064514, + "learning_rate": 3.6697247706422022e-06, + "log_odds_chosen": 0.38070449233055115, + "log_odds_ratio": -0.5242205858230591, + "logits/chosen": -0.24639271199703217, + "logits/rejected": -0.02295789122581482, + "logps/chosen": -1.9235130548477173, + "logps/rejected": -2.256152868270874, + "loss": 2.0966, + "nll_loss": 2.044215440750122, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.192351296544075, + "rewards/margins": 0.03326397389173508, + "rewards/rejected": -0.22561527788639069, + "step": 81 + }, + { + "epoch": 0.22644114601311702, + "grad_norm": 0.4871878921985626, + "learning_rate": 3.7155963302752295e-06, + "log_odds_chosen": 0.3706399202346802, + "log_odds_ratio": -0.5302938222885132, + "logits/chosen": -0.32461822032928467, + "logits/rejected": -0.07178197801113129, + "logps/chosen": -2.002980947494507, + "logps/rejected": -2.3308701515197754, + "loss": 2.1902, + "nll_loss": 2.1371781826019287, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20029808580875397, + "rewards/margins": 0.03278890624642372, + "rewards/rejected": -0.2330869883298874, + "step": 82 + }, + { + "epoch": 0.22920262340352088, + "grad_norm": 0.5049198865890503, + "learning_rate": 3.7614678899082575e-06, + "log_odds_chosen": 0.6436396241188049, + "log_odds_ratio": -0.4284180700778961, + "logits/chosen": -0.3081129789352417, + "logits/rejected": -0.1997162252664566, + "logps/chosen": -1.9448068141937256, + "logps/rejected": -2.5192015171051025, + "loss": 2.1169, + "nll_loss": 2.0740115642547607, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19448068737983704, + "rewards/margins": 0.057439468801021576, + "rewards/rejected": -0.2519201636314392, + "step": 83 + }, + { + "epoch": 0.23196410079392474, + "grad_norm": 0.5246726870536804, + "learning_rate": 3.8073394495412848e-06, + "log_odds_chosen": 0.43793147802352905, + "log_odds_ratio": -0.5015705823898315, + "logits/chosen": -0.29296931624412537, + "logits/rejected": -0.13174469769001007, + "logps/chosen": -1.941349744796753, + "logps/rejected": -2.3253884315490723, + "loss": 2.1333, + "nll_loss": 2.08317232131958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19413498044013977, + "rewards/margins": 0.03840385749936104, + "rewards/rejected": -0.2325388491153717, + "step": 84 + }, + { + "epoch": 0.23472557818432863, + "grad_norm": 0.41432732343673706, + "learning_rate": 3.853211009174313e-06, + "log_odds_chosen": 0.5745496153831482, + "log_odds_ratio": -0.4579155147075653, + "logits/chosen": -0.3449418544769287, + "logits/rejected": -0.19380588829517365, + "logps/chosen": -1.8625164031982422, + "logps/rejected": -2.3706510066986084, + "loss": 2.0375, + "nll_loss": 1.9917008876800537, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1862516552209854, + "rewards/margins": 0.050813477486371994, + "rewards/rejected": -0.2370651215314865, + "step": 85 + }, + { + "epoch": 0.2374870555747325, + "grad_norm": 0.438418984413147, + "learning_rate": 3.89908256880734e-06, + "log_odds_chosen": 0.580481231212616, + "log_odds_ratio": -0.4620014429092407, + "logits/chosen": -0.14796528220176697, + "logits/rejected": -0.32091909646987915, + "logps/chosen": -1.9756391048431396, + "logps/rejected": -2.4986729621887207, + "loss": 2.1484, + "nll_loss": 2.102167844772339, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19756391644477844, + "rewards/margins": 0.052303388714790344, + "rewards/rejected": -0.2498673051595688, + "step": 86 + }, + { + "epoch": 0.24024853296513635, + "grad_norm": 0.4355674684047699, + "learning_rate": 3.944954128440367e-06, + "log_odds_chosen": 0.5157158970832825, + "log_odds_ratio": -0.4764195382595062, + "logits/chosen": -0.16625456511974335, + "logits/rejected": -0.22959333658218384, + "logps/chosen": -1.8700374364852905, + "logps/rejected": -2.3208043575286865, + "loss": 2.0401, + "nll_loss": 1.9924601316452026, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18700376152992249, + "rewards/margins": 0.04507668316364288, + "rewards/rejected": -0.23208042979240417, + "step": 87 + }, + { + "epoch": 0.2430100103555402, + "grad_norm": 0.4138505756855011, + "learning_rate": 3.9908256880733945e-06, + "log_odds_chosen": 0.5811448693275452, + "log_odds_ratio": -0.48126012086868286, + "logits/chosen": -0.3125801980495453, + "logits/rejected": -0.16787581145763397, + "logps/chosen": -1.79574453830719, + "logps/rejected": -2.3057985305786133, + "loss": 1.9834, + "nll_loss": 1.9353020191192627, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17957444489002228, + "rewards/margins": 0.051005423069000244, + "rewards/rejected": -0.23057986795902252, + "step": 88 + }, + { + "epoch": 0.24577148774594407, + "grad_norm": 0.3883982002735138, + "learning_rate": 4.036697247706423e-06, + "log_odds_chosen": 0.6218655705451965, + "log_odds_ratio": -0.43868428468704224, + "logits/chosen": -0.16164137423038483, + "logits/rejected": -0.2850674092769623, + "logps/chosen": -1.8971762657165527, + "logps/rejected": -2.4469919204711914, + "loss": 2.0667, + "nll_loss": 2.0228123664855957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.189717635512352, + "rewards/margins": 0.05498155951499939, + "rewards/rejected": -0.24469918012619019, + "step": 89 + }, + { + "epoch": 0.24853296513634796, + "grad_norm": 0.441839337348938, + "learning_rate": 4.08256880733945e-06, + "log_odds_chosen": 0.5872446894645691, + "log_odds_ratio": -0.44401073455810547, + "logits/chosen": -0.2670595347881317, + "logits/rejected": -0.07647659629583359, + "logps/chosen": -1.8952282667160034, + "logps/rejected": -2.412400245666504, + "loss": 2.066, + "nll_loss": 2.021595001220703, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18952281773090363, + "rewards/margins": 0.051717206835746765, + "rewards/rejected": -0.24124003946781158, + "step": 90 + }, + { + "epoch": 0.2512944425267518, + "grad_norm": 0.4077044427394867, + "learning_rate": 4.128440366972478e-06, + "log_odds_chosen": 0.4384881556034088, + "log_odds_ratio": -0.5076763033866882, + "logits/chosen": -0.15786704421043396, + "logits/rejected": -0.090408094227314, + "logps/chosen": -1.878911018371582, + "logps/rejected": -2.262026786804199, + "loss": 2.047, + "nll_loss": 1.9962477684020996, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18789111077785492, + "rewards/margins": 0.03831159323453903, + "rewards/rejected": -0.22620268166065216, + "step": 91 + }, + { + "epoch": 0.2540559199171557, + "grad_norm": 0.3839576244354248, + "learning_rate": 4.174311926605505e-06, + "log_odds_chosen": 0.42869681119918823, + "log_odds_ratio": -0.5070521235466003, + "logits/chosen": -0.08416657149791718, + "logits/rejected": -0.4555465579032898, + "logps/chosen": -1.9147799015045166, + "logps/rejected": -2.292423725128174, + "loss": 2.0814, + "nll_loss": 2.0306570529937744, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19147798418998718, + "rewards/margins": 0.03776439279317856, + "rewards/rejected": -0.22924238443374634, + "step": 92 + }, + { + "epoch": 0.25681739730755954, + "grad_norm": 0.42368394136428833, + "learning_rate": 4.220183486238532e-06, + "log_odds_chosen": 0.5149534344673157, + "log_odds_ratio": -0.47629106044769287, + "logits/chosen": -0.13718552887439728, + "logits/rejected": -0.20033138990402222, + "logps/chosen": -1.9013878107070923, + "logps/rejected": -2.3537254333496094, + "loss": 2.0789, + "nll_loss": 2.0312626361846924, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1901387870311737, + "rewards/margins": 0.04523376002907753, + "rewards/rejected": -0.23537255823612213, + "step": 93 + }, + { + "epoch": 0.2595788746979634, + "grad_norm": 0.38014668226242065, + "learning_rate": 4.26605504587156e-06, + "log_odds_chosen": 0.4552799165248871, + "log_odds_ratio": -0.49941790103912354, + "logits/chosen": -0.2649000883102417, + "logits/rejected": -0.20182102918624878, + "logps/chosen": -1.8007447719573975, + "logps/rejected": -2.1948485374450684, + "loss": 1.9637, + "nll_loss": 1.9138000011444092, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18007448315620422, + "rewards/margins": 0.03941037505865097, + "rewards/rejected": -0.2194848358631134, + "step": 94 + }, + { + "epoch": 0.26234035208836726, + "grad_norm": 0.3550948202610016, + "learning_rate": 4.311926605504588e-06, + "log_odds_chosen": 0.36030256748199463, + "log_odds_ratio": -0.533312976360321, + "logits/chosen": -0.0772688165307045, + "logits/rejected": -0.16052654385566711, + "logps/chosen": -1.8254213333129883, + "logps/rejected": -2.136554002761841, + "loss": 2.0005, + "nll_loss": 1.9472124576568604, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1825421154499054, + "rewards/margins": 0.03111325576901436, + "rewards/rejected": -0.21365538239479065, + "step": 95 + }, + { + "epoch": 0.26510182947877114, + "grad_norm": 0.40300413966178894, + "learning_rate": 4.357798165137615e-06, + "log_odds_chosen": 0.2818768620491028, + "log_odds_ratio": -0.5676061511039734, + "logits/chosen": -0.10504347085952759, + "logits/rejected": -0.05843639373779297, + "logps/chosen": -1.9888579845428467, + "logps/rejected": -2.2374351024627686, + "loss": 2.1718, + "nll_loss": 2.11505126953125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19888579845428467, + "rewards/margins": 0.02485768496990204, + "rewards/rejected": -0.2237434834241867, + "step": 96 + }, + { + "epoch": 0.26786330686917503, + "grad_norm": 0.398783802986145, + "learning_rate": 4.403669724770643e-06, + "log_odds_chosen": 0.5546217560768127, + "log_odds_ratio": -0.46683651208877563, + "logits/chosen": -0.008073419332504272, + "logits/rejected": -0.08634719252586365, + "logps/chosen": -1.8501688241958618, + "logps/rejected": -2.342288017272949, + "loss": 2.0318, + "nll_loss": 1.9851207733154297, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18501687049865723, + "rewards/margins": 0.04921191930770874, + "rewards/rejected": -0.23422878980636597, + "step": 97 + }, + { + "epoch": 0.27062478425957887, + "grad_norm": 0.39449602365493774, + "learning_rate": 4.44954128440367e-06, + "log_odds_chosen": 0.4133303165435791, + "log_odds_ratio": -0.5122652649879456, + "logits/chosen": -0.05717798322439194, + "logits/rejected": -0.1295921504497528, + "logps/chosen": -1.9054274559020996, + "logps/rejected": -2.266583204269409, + "loss": 2.0813, + "nll_loss": 2.030045509338379, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19054275751113892, + "rewards/margins": 0.03611557558178902, + "rewards/rejected": -0.22665831446647644, + "step": 98 + }, + { + "epoch": 0.27338626164998275, + "grad_norm": 0.39879634976387024, + "learning_rate": 4.4954128440366975e-06, + "log_odds_chosen": 0.547784149646759, + "log_odds_ratio": -0.46036428213119507, + "logits/chosen": -0.09109216928482056, + "logits/rejected": -0.3699495494365692, + "logps/chosen": -1.9557167291641235, + "logps/rejected": -2.4426751136779785, + "loss": 2.1161, + "nll_loss": 2.070013999938965, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1955716609954834, + "rewards/margins": 0.04869585484266281, + "rewards/rejected": -0.2442675083875656, + "step": 99 + }, + { + "epoch": 0.2761477390403866, + "grad_norm": 0.3873852491378784, + "learning_rate": 4.541284403669725e-06, + "log_odds_chosen": 0.3448026478290558, + "log_odds_ratio": -0.5393580198287964, + "logits/chosen": -0.1656271517276764, + "logits/rejected": -0.07856383174657822, + "logps/chosen": -1.8394955396652222, + "logps/rejected": -2.1358819007873535, + "loss": 2.0365, + "nll_loss": 1.9825859069824219, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1839495450258255, + "rewards/margins": 0.029638633131980896, + "rewards/rejected": -0.2135881781578064, + "step": 100 + }, + { + "epoch": 0.2789092164307905, + "grad_norm": 0.3506630063056946, + "learning_rate": 4.587155963302753e-06, + "log_odds_chosen": 0.33486518263816833, + "log_odds_ratio": -0.5475890636444092, + "logits/chosen": 0.009370687417685986, + "logits/rejected": -0.18966984748840332, + "logps/chosen": -1.9532427787780762, + "logps/rejected": -2.2507667541503906, + "loss": 2.132, + "nll_loss": 2.077195405960083, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19532427191734314, + "rewards/margins": 0.02975238859653473, + "rewards/rejected": -0.22507666051387787, + "step": 101 + }, + { + "epoch": 0.28167069382119436, + "grad_norm": 0.33624136447906494, + "learning_rate": 4.63302752293578e-06, + "log_odds_chosen": 0.5504237413406372, + "log_odds_ratio": -0.4613422155380249, + "logits/chosen": -0.06238182261586189, + "logits/rejected": -0.22859227657318115, + "logps/chosen": -1.791447401046753, + "logps/rejected": -2.2658450603485107, + "loss": 1.9877, + "nll_loss": 1.9415662288665771, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1791447550058365, + "rewards/margins": 0.04743976891040802, + "rewards/rejected": -0.2265845388174057, + "step": 102 + }, + { + "epoch": 0.2844321712115982, + "grad_norm": 0.3664388656616211, + "learning_rate": 4.678899082568808e-06, + "log_odds_chosen": 0.3437744081020355, + "log_odds_ratio": -0.5439410209655762, + "logits/chosen": -0.1386101245880127, + "logits/rejected": -0.17609171569347382, + "logps/chosen": -1.8379578590393066, + "logps/rejected": -2.1358203887939453, + "loss": 2.022, + "nll_loss": 1.9676285982131958, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1837957799434662, + "rewards/margins": 0.0297862458974123, + "rewards/rejected": -0.21358203887939453, + "step": 103 + }, + { + "epoch": 0.2871936486020021, + "grad_norm": 0.3293968439102173, + "learning_rate": 4.724770642201835e-06, + "log_odds_chosen": 0.3870212137699127, + "log_odds_ratio": -0.5231497883796692, + "logits/chosen": -0.07520203292369843, + "logits/rejected": -0.15402644872665405, + "logps/chosen": -1.756553053855896, + "logps/rejected": -2.086921453475952, + "loss": 1.9488, + "nll_loss": 1.8965202569961548, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1756553202867508, + "rewards/margins": 0.033036813139915466, + "rewards/rejected": -0.20869213342666626, + "step": 104 + }, + { + "epoch": 0.2899551259924059, + "grad_norm": 0.33918339014053345, + "learning_rate": 4.770642201834863e-06, + "log_odds_chosen": 0.4852727949619293, + "log_odds_ratio": -0.4833363890647888, + "logits/chosen": -0.004435461014509201, + "logits/rejected": -0.1534399837255478, + "logps/chosen": -1.7944328784942627, + "logps/rejected": -2.2146499156951904, + "loss": 1.9627, + "nll_loss": 1.9144006967544556, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17944329977035522, + "rewards/margins": 0.042021699249744415, + "rewards/rejected": -0.22146499156951904, + "step": 105 + }, + { + "epoch": 0.2927166033828098, + "grad_norm": 0.3346327841281891, + "learning_rate": 4.816513761467891e-06, + "log_odds_chosen": 0.5177164673805237, + "log_odds_ratio": -0.47983041405677795, + "logits/chosen": -0.00399226788431406, + "logits/rejected": -0.10182341933250427, + "logps/chosen": -1.7794350385665894, + "logps/rejected": -2.226297616958618, + "loss": 1.9511, + "nll_loss": 1.9030946493148804, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17794349789619446, + "rewards/margins": 0.044686250388622284, + "rewards/rejected": -0.22262977063655853, + "step": 106 + }, + { + "epoch": 0.2954780807732137, + "grad_norm": 0.3454085886478424, + "learning_rate": 4.862385321100918e-06, + "log_odds_chosen": 0.3980882465839386, + "log_odds_ratio": -0.5184234380722046, + "logits/chosen": 0.01898978091776371, + "logits/rejected": -0.21249954402446747, + "logps/chosen": -1.8531229496002197, + "logps/rejected": -2.197577953338623, + "loss": 2.0242, + "nll_loss": 1.9724009037017822, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18531231582164764, + "rewards/margins": 0.03444547951221466, + "rewards/rejected": -0.2197577804327011, + "step": 107 + }, + { + "epoch": 0.2982395581636175, + "grad_norm": 0.3301767408847809, + "learning_rate": 4.908256880733945e-06, + "log_odds_chosen": 0.44768860936164856, + "log_odds_ratio": -0.5000944137573242, + "logits/chosen": 0.06930336356163025, + "logits/rejected": -0.24058011174201965, + "logps/chosen": -1.8849480152130127, + "logps/rejected": -2.278297185897827, + "loss": 2.0446, + "nll_loss": 1.9945893287658691, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18849481642246246, + "rewards/margins": 0.039334915578365326, + "rewards/rejected": -0.2278297394514084, + "step": 108 + }, + { + "epoch": 0.3010010355540214, + "grad_norm": 0.3244481086730957, + "learning_rate": 4.954128440366973e-06, + "log_odds_chosen": 0.6234428286552429, + "log_odds_ratio": -0.44249945878982544, + "logits/chosen": 0.045590970665216446, + "logits/rejected": -0.3307313919067383, + "logps/chosen": -1.8896514177322388, + "logps/rejected": -2.4449257850646973, + "loss": 2.0431, + "nll_loss": 1.9988850355148315, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18896515667438507, + "rewards/margins": 0.0555274523794651, + "rewards/rejected": -0.24449260532855988, + "step": 109 + }, + { + "epoch": 0.30376251294442524, + "grad_norm": 0.31317761540412903, + "learning_rate": 5e-06, + "log_odds_chosen": 0.2916356325149536, + "log_odds_ratio": -0.5743635892868042, + "logits/chosen": 0.144297257065773, + "logits/rejected": -0.26908552646636963, + "logps/chosen": -1.8418867588043213, + "logps/rejected": -2.0925679206848145, + "loss": 2.0441, + "nll_loss": 1.9866299629211426, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18418867886066437, + "rewards/margins": 0.02506813406944275, + "rewards/rejected": -0.20925679802894592, + "step": 110 + }, + { + "epoch": 0.30652399033482913, + "grad_norm": 0.29466620087623596, + "learning_rate": 4.999987154315977e-06, + "log_odds_chosen": 0.42353084683418274, + "log_odds_ratio": -0.5111951231956482, + "logits/chosen": 0.05306434631347656, + "logits/rejected": -0.15503177046775818, + "logps/chosen": -1.7407968044281006, + "logps/rejected": -2.0989816188812256, + "loss": 1.924, + "nll_loss": 1.87285315990448, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17407968640327454, + "rewards/margins": 0.03581848740577698, + "rewards/rejected": -0.2098981738090515, + "step": 111 + }, + { + "epoch": 0.309285467725233, + "grad_norm": 0.3188883364200592, + "learning_rate": 4.999948617395916e-06, + "log_odds_chosen": 0.4043401777744293, + "log_odds_ratio": -0.5218645930290222, + "logits/chosen": 0.16884373128414154, + "logits/rejected": -0.07719510793685913, + "logps/chosen": -1.8007270097732544, + "logps/rejected": -2.151782274246216, + "loss": 1.9777, + "nll_loss": 1.9254791736602783, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18007270991802216, + "rewards/margins": 0.035105518996715546, + "rewards/rejected": -0.2151782363653183, + "step": 112 + }, + { + "epoch": 0.31204694511563685, + "grad_norm": 0.3535216450691223, + "learning_rate": 4.999884389635843e-06, + "log_odds_chosen": 0.41415858268737793, + "log_odds_ratio": -0.5124688744544983, + "logits/chosen": 0.0558352991938591, + "logits/rejected": -0.09803693741559982, + "logps/chosen": -1.8450539112091064, + "logps/rejected": -2.2039334774017334, + "loss": 2.0336, + "nll_loss": 1.9823946952819824, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1845053881406784, + "rewards/margins": 0.035887960344552994, + "rewards/rejected": -0.2203933596611023, + "step": 113 + }, + { + "epoch": 0.31480842250604074, + "grad_norm": 0.2991940379142761, + "learning_rate": 4.9997944716957985e-06, + "log_odds_chosen": 0.514029324054718, + "log_odds_ratio": -0.47918182611465454, + "logits/chosen": 0.16882193088531494, + "logits/rejected": -0.24593651294708252, + "logps/chosen": -1.8493305444717407, + "logps/rejected": -2.298590898513794, + "loss": 2.0272, + "nll_loss": 1.9792673587799072, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18493306636810303, + "rewards/margins": 0.044926032423973083, + "rewards/rejected": -0.22985908389091492, + "step": 114 + }, + { + "epoch": 0.3175698998964446, + "grad_norm": 0.28921976685523987, + "learning_rate": 4.999678864499828e-06, + "log_odds_chosen": 0.38468483090400696, + "log_odds_ratio": -0.5222321152687073, + "logits/chosen": 0.2794942557811737, + "logits/rejected": -0.21004724502563477, + "logps/chosen": -1.8734655380249023, + "logps/rejected": -2.2063117027282715, + "loss": 2.0344, + "nll_loss": 1.9822089672088623, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18734657764434814, + "rewards/margins": 0.03328459709882736, + "rewards/rejected": -0.2206311821937561, + "step": 115 + }, + { + "epoch": 0.32033137728684846, + "grad_norm": 0.30199047923088074, + "learning_rate": 4.999537569235975e-06, + "log_odds_chosen": 0.35607457160949707, + "log_odds_ratio": -0.5390717387199402, + "logits/chosen": 0.17685924470424652, + "logits/rejected": -0.16482022404670715, + "logps/chosen": -1.8530224561691284, + "logps/rejected": -2.1583948135375977, + "loss": 2.0285, + "nll_loss": 1.9746413230895996, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1853022426366806, + "rewards/margins": 0.030537229031324387, + "rewards/rejected": -0.21583949029445648, + "step": 116 + }, + { + "epoch": 0.32309285467725235, + "grad_norm": 0.305818647146225, + "learning_rate": 4.999370587356267e-06, + "log_odds_chosen": 0.3153924345970154, + "log_odds_ratio": -0.5565903782844543, + "logits/chosen": 0.17289672791957855, + "logits/rejected": -0.3006405234336853, + "logps/chosen": -1.9371519088745117, + "logps/rejected": -2.214411497116089, + "loss": 2.1096, + "nll_loss": 2.0539159774780273, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1937151849269867, + "rewards/margins": 0.027725949883461, + "rewards/rejected": -0.2214411348104477, + "step": 117 + }, + { + "epoch": 0.3258543320676562, + "grad_norm": 0.324216365814209, + "learning_rate": 4.9991779205767e-06, + "log_odds_chosen": 0.29256871342658997, + "log_odds_ratio": -0.5621715784072876, + "logits/chosen": 0.055469777435064316, + "logits/rejected": -0.5593094229698181, + "logps/chosen": -1.808518886566162, + "logps/rejected": -2.060049295425415, + "loss": 1.9916, + "nll_loss": 1.9353588819503784, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18085187673568726, + "rewards/margins": 0.02515305206179619, + "rewards/rejected": -0.20600494742393494, + "step": 118 + }, + { + "epoch": 0.32861580945806007, + "grad_norm": 0.316292941570282, + "learning_rate": 4.998959570877224e-06, + "log_odds_chosen": 0.391570121049881, + "log_odds_ratio": -0.519210159778595, + "logits/chosen": 0.29701903462409973, + "logits/rejected": -0.248467817902565, + "logps/chosen": -1.950477123260498, + "logps/rejected": -2.2915000915527344, + "loss": 2.1349, + "nll_loss": 2.0829989910125732, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19504770636558533, + "rewards/margins": 0.034102290868759155, + "rewards/rejected": -0.22914999723434448, + "step": 119 + }, + { + "epoch": 0.3313772868484639, + "grad_norm": 0.29707539081573486, + "learning_rate": 4.99871554050172e-06, + "log_odds_chosen": 0.49879857897758484, + "log_odds_ratio": -0.4770265221595764, + "logits/chosen": 0.1394173949956894, + "logits/rejected": -0.11445163935422897, + "logps/chosen": -1.7406611442565918, + "logps/rejected": -2.167613983154297, + "loss": 1.9065, + "nll_loss": 1.8588043451309204, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17406611144542694, + "rewards/margins": 0.042695268988609314, + "rewards/rejected": -0.21676141023635864, + "step": 120 + }, + { + "epoch": 0.3341387642388678, + "grad_norm": 0.3011818528175354, + "learning_rate": 4.9984458319579775e-06, + "log_odds_chosen": 0.4734145998954773, + "log_odds_ratio": -0.48969942331314087, + "logits/chosen": 0.10353265702724457, + "logits/rejected": -0.22807854413986206, + "logps/chosen": -1.7799421548843384, + "logps/rejected": -2.1871492862701416, + "loss": 1.9363, + "nll_loss": 1.8872833251953125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17799422144889832, + "rewards/margins": 0.04072071984410286, + "rewards/rejected": -0.21871493756771088, + "step": 121 + }, + { + "epoch": 0.3369002416292717, + "grad_norm": 0.28915688395500183, + "learning_rate": 4.99815044801767e-06, + "log_odds_chosen": 0.3440546989440918, + "log_odds_ratio": -0.5443906784057617, + "logits/chosen": 0.22890803217887878, + "logits/rejected": -0.20038765668869019, + "logps/chosen": -1.8269233703613281, + "logps/rejected": -2.1230876445770264, + "loss": 2.0144, + "nll_loss": 1.959984540939331, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18269234895706177, + "rewards/margins": 0.02961641550064087, + "rewards/rejected": -0.21230876445770264, + "step": 122 + }, + { + "epoch": 0.3396617190196755, + "grad_norm": 0.31248003244400024, + "learning_rate": 4.9978293917163225e-06, + "log_odds_chosen": 0.21560871601104736, + "log_odds_ratio": -0.5958309769630432, + "logits/chosen": 0.2199111431837082, + "logits/rejected": -0.15359394252300262, + "logps/chosen": -1.9724359512329102, + "logps/rejected": -2.162384033203125, + "loss": 2.1306, + "nll_loss": 2.0710067749023438, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1972435861825943, + "rewards/margins": 0.01899481564760208, + "rewards/rejected": -0.21623840928077698, + "step": 123 + }, + { + "epoch": 0.3424231964100794, + "grad_norm": 0.3086913824081421, + "learning_rate": 4.997482666353287e-06, + "log_odds_chosen": 0.36193954944610596, + "log_odds_ratio": -0.5371627807617188, + "logits/chosen": 0.07905742526054382, + "logits/rejected": -0.047993652522563934, + "logps/chosen": -1.860518217086792, + "logps/rejected": -2.175654649734497, + "loss": 2.0362, + "nll_loss": 1.982520341873169, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18605183064937592, + "rewards/margins": 0.03151364251971245, + "rewards/rejected": -0.21756546199321747, + "step": 124 + }, + { + "epoch": 0.34518467380048323, + "grad_norm": 0.2956501543521881, + "learning_rate": 4.997110275491702e-06, + "log_odds_chosen": 0.5367758274078369, + "log_odds_ratio": -0.4794245958328247, + "logits/chosen": 0.17313790321350098, + "logits/rejected": -0.5377508401870728, + "logps/chosen": -1.7679404020309448, + "logps/rejected": -2.2323243618011475, + "loss": 1.9515, + "nll_loss": 1.9035319089889526, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17679405212402344, + "rewards/margins": 0.04643838852643967, + "rewards/rejected": -0.2232324331998825, + "step": 125 + }, + { + "epoch": 0.3479461511908871, + "grad_norm": 0.2930620014667511, + "learning_rate": 4.9967122229584614e-06, + "log_odds_chosen": 0.5086734890937805, + "log_odds_ratio": -0.47367364168167114, + "logits/chosen": 0.20121866464614868, + "logits/rejected": -0.3580709397792816, + "logps/chosen": -1.772174596786499, + "logps/rejected": -2.211575746536255, + "loss": 1.9502, + "nll_loss": 1.9028148651123047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17721746861934662, + "rewards/margins": 0.04394011199474335, + "rewards/rejected": -0.22115758061408997, + "step": 126 + }, + { + "epoch": 0.350707628581291, + "grad_norm": 0.313006728887558, + "learning_rate": 4.996288512844169e-06, + "log_odds_chosen": 0.26984095573425293, + "log_odds_ratio": -0.5770066380500793, + "logits/chosen": 0.30613642930984497, + "logits/rejected": -0.0801864042878151, + "logps/chosen": -1.9204916954040527, + "logps/rejected": -2.153423547744751, + "loss": 2.1089, + "nll_loss": 2.051222324371338, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19204919040203094, + "rewards/margins": 0.023293154314160347, + "rewards/rejected": -0.21534234285354614, + "step": 127 + }, + { + "epoch": 0.35346910597169484, + "grad_norm": 0.26883664727211, + "learning_rate": 4.995839149503103e-06, + "log_odds_chosen": 0.4242061972618103, + "log_odds_ratio": -0.5166581273078918, + "logits/chosen": 0.287309467792511, + "logits/rejected": -0.28880101442337036, + "logps/chosen": -1.7962003946304321, + "logps/rejected": -2.1639606952667236, + "loss": 1.9745, + "nll_loss": 1.9228155612945557, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17962004244327545, + "rewards/margins": 0.036776017397642136, + "rewards/rejected": -0.2163960486650467, + "step": 128 + }, + { + "epoch": 0.35623058336209873, + "grad_norm": 0.3000660240650177, + "learning_rate": 4.995364137553166e-06, + "log_odds_chosen": 0.2731889486312866, + "log_odds_ratio": -0.5769234299659729, + "logits/chosen": 0.17702779173851013, + "logits/rejected": -0.16427305340766907, + "logps/chosen": -1.8508391380310059, + "logps/rejected": -2.0855119228363037, + "loss": 2.0272, + "nll_loss": 1.9694753885269165, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18508392572402954, + "rewards/margins": 0.023467278108000755, + "rewards/rejected": -0.20855121314525604, + "step": 129 + }, + { + "epoch": 0.35899206075250256, + "grad_norm": 0.27979782223701477, + "learning_rate": 4.994863481875842e-06, + "log_odds_chosen": 0.539387047290802, + "log_odds_ratio": -0.4670449495315552, + "logits/chosen": 0.19368596374988556, + "logits/rejected": -0.4078698754310608, + "logps/chosen": -1.7874817848205566, + "logps/rejected": -2.2576088905334473, + "loss": 1.9686, + "nll_loss": 1.9218828678131104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17874819040298462, + "rewards/margins": 0.04701270908117294, + "rewards/rejected": -0.22576089203357697, + "step": 130 + }, + { + "epoch": 0.36175353814290645, + "grad_norm": 0.2961781322956085, + "learning_rate": 4.99433718761614e-06, + "log_odds_chosen": 0.482613205909729, + "log_odds_ratio": -0.48607659339904785, + "logits/chosen": 0.3117329776287079, + "logits/rejected": -0.19324824213981628, + "logps/chosen": -1.8348233699798584, + "logps/rejected": -2.2560348510742188, + "loss": 1.9992, + "nll_loss": 1.9505534172058105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1834823340177536, + "rewards/margins": 0.042121127247810364, + "rewards/rejected": -0.22560347616672516, + "step": 131 + }, + { + "epoch": 0.36451501553331034, + "grad_norm": 0.27247706055641174, + "learning_rate": 4.993785260182552e-06, + "log_odds_chosen": 0.5040154457092285, + "log_odds_ratio": -0.47964078187942505, + "logits/chosen": 0.26086652278900146, + "logits/rejected": -0.39971601963043213, + "logps/chosen": -1.7693819999694824, + "logps/rejected": -2.206263780593872, + "loss": 1.9461, + "nll_loss": 1.898113489151001, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17693820595741272, + "rewards/margins": 0.043688178062438965, + "rewards/rejected": -0.2206263691186905, + "step": 132 + }, + { + "epoch": 0.36727649292371417, + "grad_norm": 0.2559458613395691, + "learning_rate": 4.993207705246983e-06, + "log_odds_chosen": 0.37582507729530334, + "log_odds_ratio": -0.529410719871521, + "logits/chosen": 0.24356283247470856, + "logits/rejected": -0.23510105907917023, + "logps/chosen": -1.8250073194503784, + "logps/rejected": -2.1527535915374756, + "loss": 1.9998, + "nll_loss": 1.9468413591384888, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18250073492527008, + "rewards/margins": 0.03277461230754852, + "rewards/rejected": -0.2152753621339798, + "step": 133 + }, + { + "epoch": 0.37003797031411806, + "grad_norm": 0.2661672532558441, + "learning_rate": 4.992604528744705e-06, + "log_odds_chosen": 0.4147174656391144, + "log_odds_ratio": -0.5111517906188965, + "logits/chosen": 0.23473379015922546, + "logits/rejected": -0.5313011407852173, + "logps/chosen": -1.8088786602020264, + "logps/rejected": -2.1657345294952393, + "loss": 1.985, + "nll_loss": 1.9338966608047485, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1808878630399704, + "rewards/margins": 0.035685598850250244, + "rewards/rejected": -0.21657346189022064, + "step": 134 + }, + { + "epoch": 0.37279944770452195, + "grad_norm": 0.26244473457336426, + "learning_rate": 4.9919757368742895e-06, + "log_odds_chosen": 0.4870528280735016, + "log_odds_ratio": -0.4892481565475464, + "logits/chosen": 0.2686365842819214, + "logits/rejected": -0.3376864194869995, + "logps/chosen": -1.7298060655593872, + "logps/rejected": -2.145728349685669, + "loss": 1.9046, + "nll_loss": 1.8556550741195679, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17298059165477753, + "rewards/margins": 0.04159224033355713, + "rewards/rejected": -0.21457283198833466, + "step": 135 + }, + { + "epoch": 0.3755609250949258, + "grad_norm": 0.26679527759552, + "learning_rate": 4.991321336097546e-06, + "log_odds_chosen": 0.4111718535423279, + "log_odds_ratio": -0.512808084487915, + "logits/chosen": 0.22426243126392365, + "logits/rejected": -0.2763628363609314, + "logps/chosen": -1.8473613262176514, + "logps/rejected": -2.2056474685668945, + "loss": 1.9977, + "nll_loss": 1.9464530944824219, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18473613262176514, + "rewards/margins": 0.03582862764596939, + "rewards/rejected": -0.22056476771831512, + "step": 136 + }, + { + "epoch": 0.37832240248532967, + "grad_norm": 0.25576797127723694, + "learning_rate": 4.990641333139455e-06, + "log_odds_chosen": 0.435663640499115, + "log_odds_ratio": -0.5039057731628418, + "logits/chosen": 0.29907894134521484, + "logits/rejected": -0.34149447083473206, + "logps/chosen": -1.7228469848632812, + "logps/rejected": -2.093015670776367, + "loss": 1.8956, + "nll_loss": 1.8452366590499878, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17228470742702484, + "rewards/margins": 0.037016861140728, + "rewards/rejected": -0.20930157601833344, + "step": 137 + }, + { + "epoch": 0.3810838798757335, + "grad_norm": 0.2576519250869751, + "learning_rate": 4.989935734988098e-06, + "log_odds_chosen": 0.32478412985801697, + "log_odds_ratio": -0.5477995872497559, + "logits/chosen": 0.3516974151134491, + "logits/rejected": -0.4254434406757355, + "logps/chosen": -1.8142131567001343, + "logps/rejected": -2.0922889709472656, + "loss": 1.9776, + "nll_loss": 1.922855257987976, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18142130970954895, + "rewards/margins": 0.02780758962035179, + "rewards/rejected": -0.20922890305519104, + "step": 138 + }, + { + "epoch": 0.3838453572661374, + "grad_norm": 0.2815987765789032, + "learning_rate": 4.989204548894589e-06, + "log_odds_chosen": 0.32539641857147217, + "log_odds_ratio": -0.5531569719314575, + "logits/chosen": 0.4096101224422455, + "logits/rejected": -0.19687263667583466, + "logps/chosen": -1.8913159370422363, + "logps/rejected": -2.1725118160247803, + "loss": 2.0515, + "nll_loss": 1.99615478515625, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18913161754608154, + "rewards/margins": 0.028119584545493126, + "rewards/rejected": -0.21725118160247803, + "step": 139 + }, + { + "epoch": 0.3866068346565413, + "grad_norm": 0.25471100211143494, + "learning_rate": 4.988447782372996e-06, + "log_odds_chosen": 0.3162972331047058, + "log_odds_ratio": -0.553735077381134, + "logits/chosen": 0.2287934273481369, + "logits/rejected": -0.36916857957839966, + "logps/chosen": -1.717573642730713, + "logps/rejected": -1.986279010772705, + "loss": 1.8883, + "nll_loss": 1.8329135179519653, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17175735533237457, + "rewards/margins": 0.026870528236031532, + "rewards/rejected": -0.19862788915634155, + "step": 140 + }, + { + "epoch": 0.3893683120469451, + "grad_norm": 0.23676469922065735, + "learning_rate": 4.9876654432002655e-06, + "log_odds_chosen": 0.41759181022644043, + "log_odds_ratio": -0.5150130391120911, + "logits/chosen": 0.3617481589317322, + "logits/rejected": -0.5005238056182861, + "logps/chosen": -1.7590631246566772, + "logps/rejected": -2.120802640914917, + "loss": 1.9118, + "nll_loss": 1.8603452444076538, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17590634524822235, + "rewards/margins": 0.03617396205663681, + "rewards/rejected": -0.21208029985427856, + "step": 141 + }, + { + "epoch": 0.392129789437349, + "grad_norm": 0.23819445073604584, + "learning_rate": 4.986857539416144e-06, + "log_odds_chosen": 0.514154851436615, + "log_odds_ratio": -0.47449907660484314, + "logits/chosen": 0.2892986834049225, + "logits/rejected": -0.5338828563690186, + "logps/chosen": -1.732762336730957, + "logps/rejected": -2.1715476512908936, + "loss": 1.8923, + "nll_loss": 1.8448883295059204, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17327623069286346, + "rewards/margins": 0.04387851804494858, + "rewards/rejected": -0.21715475618839264, + "step": 142 + }, + { + "epoch": 0.39489126682775283, + "grad_norm": 0.2597619891166687, + "learning_rate": 4.986024079323092e-06, + "log_odds_chosen": 0.21202370524406433, + "log_odds_ratio": -0.5989823341369629, + "logits/chosen": 0.31219571828842163, + "logits/rejected": -0.2251828908920288, + "logps/chosen": -1.7001993656158447, + "logps/rejected": -1.8785269260406494, + "loss": 1.9001, + "nll_loss": 1.840247392654419, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17001992464065552, + "rewards/margins": 0.017832759767770767, + "rewards/rejected": -0.18785269558429718, + "step": 143 + }, + { + "epoch": 0.3976527442181567, + "grad_norm": 0.24414244294166565, + "learning_rate": 4.985165071486201e-06, + "log_odds_chosen": 0.42103180289268494, + "log_odds_ratio": -0.506747305393219, + "logits/chosen": 0.24320174753665924, + "logits/rejected": -0.47630739212036133, + "logps/chosen": -1.813495397567749, + "logps/rejected": -2.177159309387207, + "loss": 1.9628, + "nll_loss": 1.9121257066726685, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18134953081607819, + "rewards/margins": 0.03636639565229416, + "rewards/rejected": -0.21771591901779175, + "step": 144 + }, + { + "epoch": 0.4004142216085606, + "grad_norm": 0.25120964646339417, + "learning_rate": 4.984280524733107e-06, + "log_odds_chosen": 0.4766540825366974, + "log_odds_ratio": -0.4867539405822754, + "logits/chosen": 0.23771557211875916, + "logits/rejected": -0.3588354289531708, + "logps/chosen": -1.7543095350265503, + "logps/rejected": -2.1605966091156006, + "loss": 1.92, + "nll_loss": 1.8713701963424683, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17543095350265503, + "rewards/margins": 0.040628716349601746, + "rewards/rejected": -0.21605966985225677, + "step": 145 + }, + { + "epoch": 0.40317569899896444, + "grad_norm": 0.23762984573841095, + "learning_rate": 4.983370448153896e-06, + "log_odds_chosen": 0.3954075574874878, + "log_odds_ratio": -0.5178768038749695, + "logits/chosen": 0.2783900201320648, + "logits/rejected": -0.3363473117351532, + "logps/chosen": -1.7586162090301514, + "logps/rejected": -2.097499370574951, + "loss": 1.9169, + "nll_loss": 1.8651440143585205, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1758616417646408, + "rewards/margins": 0.033888280391693115, + "rewards/rejected": -0.20974992215633392, + "step": 146 + }, + { + "epoch": 0.4059371763893683, + "grad_norm": 0.250926673412323, + "learning_rate": 4.9824348511010115e-06, + "log_odds_chosen": 0.37487757205963135, + "log_odds_ratio": -0.5328046083450317, + "logits/chosen": 0.3899734616279602, + "logits/rejected": -0.3820725679397583, + "logps/chosen": -1.8538525104522705, + "logps/rejected": -2.1806483268737793, + "loss": 2.0058, + "nll_loss": 1.9525387287139893, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18538527190685272, + "rewards/margins": 0.03267957270145416, + "rewards/rejected": -0.2180648297071457, + "step": 147 + }, + { + "epoch": 0.40869865377977216, + "grad_norm": 0.24456721544265747, + "learning_rate": 4.981473743189163e-06, + "log_odds_chosen": 0.6152381300926208, + "log_odds_ratio": -0.4415377974510193, + "logits/chosen": 0.15124794840812683, + "logits/rejected": -0.6728506684303284, + "logps/chosen": -1.630881905555725, + "logps/rejected": -2.1548542976379395, + "loss": 1.8038, + "nll_loss": 1.7596325874328613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16308818757534027, + "rewards/margins": 0.05239725112915039, + "rewards/rejected": -0.21548543870449066, + "step": 148 + }, + { + "epoch": 0.41146013117017605, + "grad_norm": 0.2503897547721863, + "learning_rate": 4.98048713429522e-06, + "log_odds_chosen": 0.4051974415779114, + "log_odds_ratio": -0.5133127570152283, + "logits/chosen": 0.3494684100151062, + "logits/rejected": -0.11723777651786804, + "logps/chosen": -1.7437934875488281, + "logps/rejected": -2.086435079574585, + "loss": 1.9057, + "nll_loss": 1.8543728590011597, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1743793487548828, + "rewards/margins": 0.034264158457517624, + "rewards/rejected": -0.20864351093769073, + "step": 149 + }, + { + "epoch": 0.41422160856057993, + "grad_norm": 0.26401370763778687, + "learning_rate": 4.979475034558115e-06, + "log_odds_chosen": 0.35506629943847656, + "log_odds_ratio": -0.534768283367157, + "logits/chosen": 0.4498825669288635, + "logits/rejected": -0.32746273279190063, + "logps/chosen": -1.8957396745681763, + "logps/rejected": -2.205925464630127, + "loss": 2.042, + "nll_loss": 1.988513469696045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1895739734172821, + "rewards/margins": 0.0310185756534338, + "rewards/rejected": -0.22059254348278046, + "step": 150 + }, + { + "epoch": 0.41698308595098377, + "grad_norm": 0.23593436181545258, + "learning_rate": 4.978437454378741e-06, + "log_odds_chosen": 0.36985495686531067, + "log_odds_ratio": -0.5328302979469299, + "logits/chosen": 0.4250433146953583, + "logits/rejected": -0.38277071714401245, + "logps/chosen": -1.7761732339859009, + "logps/rejected": -2.0909667015075684, + "loss": 1.942, + "nll_loss": 1.8887526988983154, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17761734127998352, + "rewards/margins": 0.03147934749722481, + "rewards/rejected": -0.20909668505191803, + "step": 151 + }, + { + "epoch": 0.41974456334138766, + "grad_norm": 0.26158878207206726, + "learning_rate": 4.977374404419838e-06, + "log_odds_chosen": 0.39601171016693115, + "log_odds_ratio": -0.5228374004364014, + "logits/chosen": 0.42059236764907837, + "logits/rejected": -0.3891626000404358, + "logps/chosen": -1.7959572076797485, + "logps/rejected": -2.140069007873535, + "loss": 1.9499, + "nll_loss": 1.8975740671157837, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1795957237482071, + "rewards/margins": 0.03441117703914642, + "rewards/rejected": -0.21400688588619232, + "step": 152 + }, + { + "epoch": 0.4225060407317915, + "grad_norm": 0.2561565339565277, + "learning_rate": 4.976285895605888e-06, + "log_odds_chosen": 0.4899553656578064, + "log_odds_ratio": -0.4833296239376068, + "logits/chosen": 0.37404921650886536, + "logits/rejected": -0.5243082046508789, + "logps/chosen": -1.7399574518203735, + "logps/rejected": -2.1531152725219727, + "loss": 1.8971, + "nll_loss": 1.8487193584442139, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1739957481622696, + "rewards/margins": 0.04131579399108887, + "rewards/rejected": -0.21531155705451965, + "step": 153 + }, + { + "epoch": 0.4252675181221954, + "grad_norm": 0.2550884485244751, + "learning_rate": 4.9751719391230055e-06, + "log_odds_chosen": 0.25457680225372314, + "log_odds_ratio": -0.5827968120574951, + "logits/chosen": 0.426510751247406, + "logits/rejected": -0.38418132066726685, + "logps/chosen": -1.710974931716919, + "logps/rejected": -1.9265732765197754, + "loss": 1.8985, + "nll_loss": 1.840250015258789, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17109748721122742, + "rewards/margins": 0.02155984938144684, + "rewards/rejected": -0.19265732169151306, + "step": 154 + }, + { + "epoch": 0.42802899551259926, + "grad_norm": 0.25417274236679077, + "learning_rate": 4.974032546418816e-06, + "log_odds_chosen": 0.47524771094322205, + "log_odds_ratio": -0.49181026220321655, + "logits/chosen": 0.43296438455581665, + "logits/rejected": -0.41207337379455566, + "logps/chosen": -1.756546139717102, + "logps/rejected": -2.1634457111358643, + "loss": 1.9365, + "nll_loss": 1.8873004913330078, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17565463483333588, + "rewards/margins": 0.04068994149565697, + "rewards/rejected": -0.21634456515312195, + "step": 155 + }, + { + "epoch": 0.4307904729030031, + "grad_norm": 0.24467967450618744, + "learning_rate": 4.9728677292023405e-06, + "log_odds_chosen": 0.19028525054454803, + "log_odds_ratio": -0.6092196702957153, + "logits/chosen": 0.5236613750457764, + "logits/rejected": -0.3672065734863281, + "logps/chosen": -1.851803183555603, + "logps/rejected": -2.0145559310913086, + "loss": 2.0125, + "nll_loss": 1.9516232013702393, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18518033623695374, + "rewards/margins": 0.016275260597467422, + "rewards/rejected": -0.20145559310913086, + "step": 156 + }, + { + "epoch": 0.433551950293407, + "grad_norm": 0.24561214447021484, + "learning_rate": 4.971677499443882e-06, + "log_odds_chosen": 0.34714600443840027, + "log_odds_ratio": -0.538092315196991, + "logits/chosen": 0.42429813742637634, + "logits/rejected": -0.4320365786552429, + "logps/chosen": -1.7550561428070068, + "logps/rejected": -2.0503358840942383, + "loss": 1.9273, + "nll_loss": 1.873533010482788, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1755056083202362, + "rewards/margins": 0.029527965933084488, + "rewards/rejected": -0.2050335705280304, + "step": 157 + }, + { + "epoch": 0.4363134276838108, + "grad_norm": 0.23480936884880066, + "learning_rate": 4.97046186937489e-06, + "log_odds_chosen": 0.3259715735912323, + "log_odds_ratio": -0.5574356913566589, + "logits/chosen": 0.33684611320495605, + "logits/rejected": -0.48997414112091064, + "logps/chosen": -1.733799934387207, + "logps/rejected": -2.011017084121704, + "loss": 1.8941, + "nll_loss": 1.8383519649505615, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17338000237941742, + "rewards/margins": 0.027721701189875603, + "rewards/rejected": -0.20110172033309937, + "step": 158 + }, + { + "epoch": 0.4390749050742147, + "grad_norm": 0.23975060880184174, + "learning_rate": 4.9692208514878445e-06, + "log_odds_chosen": 0.2344578355550766, + "log_odds_ratio": -0.5858049392700195, + "logits/chosen": 0.4837522804737091, + "logits/rejected": -0.2647075951099396, + "logps/chosen": -1.8640694618225098, + "logps/rejected": -2.064138412475586, + "loss": 2.0361, + "nll_loss": 1.9775654077529907, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1864069551229477, + "rewards/margins": 0.02000689134001732, + "rewards/rejected": -0.2064138650894165, + "step": 159 + }, + { + "epoch": 0.4418363824646186, + "grad_norm": 0.25472894310951233, + "learning_rate": 4.967954458536126e-06, + "log_odds_chosen": 0.34545353055000305, + "log_odds_ratio": -0.5416699051856995, + "logits/chosen": 0.4849855303764343, + "logits/rejected": -0.14890551567077637, + "logps/chosen": -1.6954622268676758, + "logps/rejected": -1.9883880615234375, + "loss": 1.8756, + "nll_loss": 1.821388602256775, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1695462316274643, + "rewards/margins": 0.029292574152350426, + "rewards/rejected": -0.19883880019187927, + "step": 160 + }, + { + "epoch": 0.4445978598550224, + "grad_norm": 0.2551233172416687, + "learning_rate": 4.96666270353388e-06, + "log_odds_chosen": 0.34612518548965454, + "log_odds_ratio": -0.5391160845756531, + "logits/chosen": 0.4446476995944977, + "logits/rejected": -0.4060593545436859, + "logps/chosen": -1.7577486038208008, + "logps/rejected": -2.0534539222717285, + "loss": 1.9068, + "nll_loss": 1.8529114723205566, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17577485740184784, + "rewards/margins": 0.029570531100034714, + "rewards/rejected": -0.20534539222717285, + "step": 161 + }, + { + "epoch": 0.4473593372454263, + "grad_norm": 0.2522425651550293, + "learning_rate": 4.965345599755888e-06, + "log_odds_chosen": 0.3961338400840759, + "log_odds_ratio": -0.520170271396637, + "logits/chosen": 0.38032418489456177, + "logits/rejected": -0.48995065689086914, + "logps/chosen": -1.815320372581482, + "logps/rejected": -2.156792640686035, + "loss": 1.9657, + "nll_loss": 1.9137252569198608, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18153204023838043, + "rewards/margins": 0.0341472253203392, + "rewards/rejected": -0.21567925810813904, + "step": 162 + }, + { + "epoch": 0.45012081463583015, + "grad_norm": 0.23575294017791748, + "learning_rate": 4.964003160737429e-06, + "log_odds_chosen": 0.42496663331985474, + "log_odds_ratio": -0.5144410133361816, + "logits/chosen": 0.4265897274017334, + "logits/rejected": -0.6324371695518494, + "logps/chosen": -1.7239199876785278, + "logps/rejected": -2.0822784900665283, + "loss": 1.8878, + "nll_loss": 1.8363168239593506, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17239199578762054, + "rewards/margins": 0.03583585098385811, + "rewards/rejected": -0.20822784304618835, + "step": 163 + }, + { + "epoch": 0.45288229202623403, + "grad_norm": 0.22830626368522644, + "learning_rate": 4.9626354002741424e-06, + "log_odds_chosen": 0.39489883184432983, + "log_odds_ratio": -0.519772469997406, + "logits/chosen": 0.43731188774108887, + "logits/rejected": -0.5182772874832153, + "logps/chosen": -1.6721280813217163, + "logps/rejected": -2.0018677711486816, + "loss": 1.8343, + "nll_loss": 1.782306432723999, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1672128140926361, + "rewards/margins": 0.032973967492580414, + "rewards/rejected": -0.20018678903579712, + "step": 164 + }, + { + "epoch": 0.4556437694166379, + "grad_norm": 0.25069254636764526, + "learning_rate": 4.9612423324218816e-06, + "log_odds_chosen": 0.4574624300003052, + "log_odds_ratio": -0.4939710795879364, + "logits/chosen": 0.4768182933330536, + "logits/rejected": -0.6430546045303345, + "logps/chosen": -1.8401563167572021, + "logps/rejected": -2.23877215385437, + "loss": 1.9978, + "nll_loss": 1.9484000205993652, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18401561677455902, + "rewards/margins": 0.03986157849431038, + "rewards/rejected": -0.2238771915435791, + "step": 165 + }, + { + "epoch": 0.45840524680704176, + "grad_norm": 0.25569233298301697, + "learning_rate": 4.959823971496575e-06, + "log_odds_chosen": 0.31097978353500366, + "log_odds_ratio": -0.5631955862045288, + "logits/chosen": 0.39253953099250793, + "logits/rejected": -0.3464045226573944, + "logps/chosen": -1.7822060585021973, + "logps/rejected": -2.0535507202148438, + "loss": 1.9515, + "nll_loss": 1.8951623439788818, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17822058498859406, + "rewards/margins": 0.027134478092193604, + "rewards/rejected": -0.20535509288311005, + "step": 166 + }, + { + "epoch": 0.46116672419744564, + "grad_norm": 0.2754010260105133, + "learning_rate": 4.958380332074074e-06, + "log_odds_chosen": 0.38330915570259094, + "log_odds_ratio": -0.5226565003395081, + "logits/chosen": 0.5406702756881714, + "logits/rejected": -0.2918284833431244, + "logps/chosen": -1.744588851928711, + "logps/rejected": -2.0728600025177, + "loss": 1.9256, + "nll_loss": 1.873305320739746, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17445889115333557, + "rewards/margins": 0.03282713145017624, + "rewards/rejected": -0.2072860151529312, + "step": 167 + }, + { + "epoch": 0.4639282015878495, + "grad_norm": 0.24440248310565948, + "learning_rate": 4.95691142899001e-06, + "log_odds_chosen": 0.4813528060913086, + "log_odds_ratio": -0.49559223651885986, + "logits/chosen": 0.42939212918281555, + "logits/rejected": -0.5100030303001404, + "logps/chosen": -1.7333720922470093, + "logps/rejected": -2.148804187774658, + "loss": 1.8988, + "nll_loss": 1.8492889404296875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17333722114562988, + "rewards/margins": 0.04154320806264877, + "rewards/rejected": -0.21488040685653687, + "step": 168 + }, + { + "epoch": 0.46668967897825336, + "grad_norm": 0.24897295236587524, + "learning_rate": 4.955417277339633e-06, + "log_odds_chosen": 0.3721994459629059, + "log_odds_ratio": -0.5288943648338318, + "logits/chosen": 0.4496070444583893, + "logits/rejected": -0.5069053173065186, + "logps/chosen": -1.7618987560272217, + "logps/rejected": -2.0793538093566895, + "loss": 1.9112, + "nll_loss": 1.858320713043213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17618988454341888, + "rewards/margins": 0.03174550086259842, + "rewards/rejected": -0.2079353779554367, + "step": 169 + }, + { + "epoch": 0.46945115636865725, + "grad_norm": 0.24506047368049622, + "learning_rate": 4.953897892477664e-06, + "log_odds_chosen": 0.4158302843570709, + "log_odds_ratio": -0.524972677230835, + "logits/chosen": 0.39215707778930664, + "logits/rejected": -0.6964855790138245, + "logps/chosen": -1.6938952207565308, + "logps/rejected": -2.0564043521881104, + "loss": 1.8573, + "nll_loss": 1.8047559261322021, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1693895161151886, + "rewards/margins": 0.03625092655420303, + "rewards/rejected": -0.20564045011997223, + "step": 170 + }, + { + "epoch": 0.4722126337590611, + "grad_norm": 0.25454598665237427, + "learning_rate": 4.952353290018132e-06, + "log_odds_chosen": 0.39419156312942505, + "log_odds_ratio": -0.5228413939476013, + "logits/chosen": 0.4652084410190582, + "logits/rejected": -0.09862995892763138, + "logps/chosen": -1.7650055885314941, + "logps/rejected": -2.102964162826538, + "loss": 1.9365, + "nll_loss": 1.8842390775680542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17650054395198822, + "rewards/margins": 0.033795878291130066, + "rewards/rejected": -0.2102964073419571, + "step": 171 + }, + { + "epoch": 0.474974111149465, + "grad_norm": 0.23166699707508087, + "learning_rate": 4.950783485834218e-06, + "log_odds_chosen": 0.45087021589279175, + "log_odds_ratio": -0.4937146306037903, + "logits/chosen": 0.43644362688064575, + "logits/rejected": -0.45175108313560486, + "logps/chosen": -1.6984238624572754, + "logps/rejected": -2.0791146755218506, + "loss": 1.8668, + "nll_loss": 1.817420482635498, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16984236240386963, + "rewards/margins": 0.03806909918785095, + "rewards/rejected": -0.20791146159172058, + "step": 172 + }, + { + "epoch": 0.4777355885398688, + "grad_norm": 0.22430026531219482, + "learning_rate": 4.949188496058089e-06, + "log_odds_chosen": 0.3452419340610504, + "log_odds_ratio": -0.5469062328338623, + "logits/chosen": 0.42078280448913574, + "logits/rejected": -0.4557611346244812, + "logps/chosen": -1.6979789733886719, + "logps/rejected": -1.9911997318267822, + "loss": 1.8648, + "nll_loss": 1.8101435899734497, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1697978973388672, + "rewards/margins": 0.029322080314159393, + "rewards/rejected": -0.19911997020244598, + "step": 173 + }, + { + "epoch": 0.4804970659302727, + "grad_norm": 0.25257107615470886, + "learning_rate": 4.947568337080733e-06, + "log_odds_chosen": 0.1399567574262619, + "log_odds_ratio": -0.6351712942123413, + "logits/chosen": 0.3786250352859497, + "logits/rejected": -0.433152973651886, + "logps/chosen": -1.8261394500732422, + "logps/rejected": -1.9481170177459717, + "loss": 1.9906, + "nll_loss": 1.9270497560501099, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.18261395394802094, + "rewards/margins": 0.012197760865092278, + "rewards/rejected": -0.19481170177459717, + "step": 174 + }, + { + "epoch": 0.4832585433206766, + "grad_norm": 0.25380203127861023, + "learning_rate": 4.945923025551789e-06, + "log_odds_chosen": 0.37491294741630554, + "log_odds_ratio": -0.5253455638885498, + "logits/chosen": 0.44007402658462524, + "logits/rejected": -0.5158473253250122, + "logps/chosen": -1.7649321556091309, + "logps/rejected": -2.0859415531158447, + "loss": 1.9061, + "nll_loss": 1.8535852432250977, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17649321258068085, + "rewards/margins": 0.032100923359394073, + "rewards/rejected": -0.20859414339065552, + "step": 175 + }, + { + "epoch": 0.4860200207110804, + "grad_norm": 0.23105277121067047, + "learning_rate": 4.944252578379379e-06, + "log_odds_chosen": 0.2923380434513092, + "log_odds_ratio": -0.5638449788093567, + "logits/chosen": 0.3620964288711548, + "logits/rejected": -0.7222499847412109, + "logps/chosen": -1.6879165172576904, + "logps/rejected": -1.9339208602905273, + "loss": 1.8481, + "nll_loss": 1.7917577028274536, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16879163682460785, + "rewards/margins": 0.024600449949502945, + "rewards/rejected": -0.1933920979499817, + "step": 176 + }, + { + "epoch": 0.4887814981014843, + "grad_norm": 0.2548852264881134, + "learning_rate": 4.942557012729933e-06, + "log_odds_chosen": 0.37703031301498413, + "log_odds_ratio": -0.5374451875686646, + "logits/chosen": 0.451770544052124, + "logits/rejected": -0.6359795331954956, + "logps/chosen": -1.7573479413986206, + "logps/rejected": -2.0781285762786865, + "loss": 1.9276, + "nll_loss": 1.873888373374939, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17573478817939758, + "rewards/margins": 0.03207805007696152, + "rewards/rejected": -0.2078128606081009, + "step": 177 + }, + { + "epoch": 0.49154297549188813, + "grad_norm": 0.24361710250377655, + "learning_rate": 4.940836346028011e-06, + "log_odds_chosen": 0.564251184463501, + "log_odds_ratio": -0.4567793309688568, + "logits/chosen": 0.36331912875175476, + "logits/rejected": -0.6997017860412598, + "logps/chosen": -1.7021052837371826, + "logps/rejected": -2.1804330348968506, + "loss": 1.8632, + "nll_loss": 1.8174842596054077, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17021054029464722, + "rewards/margins": 0.047832753509283066, + "rewards/rejected": -0.2180432826280594, + "step": 178 + }, + { + "epoch": 0.494304452882292, + "grad_norm": 0.25140658020973206, + "learning_rate": 4.9390905959561254e-06, + "log_odds_chosen": 0.42371851205825806, + "log_odds_ratio": -0.5053101778030396, + "logits/chosen": 0.4960322678089142, + "logits/rejected": -0.7143914699554443, + "logps/chosen": -1.8060599565505981, + "logps/rejected": -2.168437957763672, + "loss": 1.9361, + "nll_loss": 1.8855350017547607, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18060600757598877, + "rewards/margins": 0.03623779118061066, + "rewards/rejected": -0.21684379875659943, + "step": 179 + }, + { + "epoch": 0.4970659302726959, + "grad_norm": 0.25267475843429565, + "learning_rate": 4.937319780454559e-06, + "log_odds_chosen": 0.24488888680934906, + "log_odds_ratio": -0.5824177861213684, + "logits/chosen": 0.43725982308387756, + "logits/rejected": -0.5809499621391296, + "logps/chosen": -1.7686806917190552, + "logps/rejected": -1.9778132438659668, + "loss": 1.9319, + "nll_loss": 1.873632550239563, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17686808109283447, + "rewards/margins": 0.020913248881697655, + "rewards/rejected": -0.19778132438659668, + "step": 180 + }, + { + "epoch": 0.49982740766309974, + "grad_norm": 0.2517687976360321, + "learning_rate": 4.935523917721182e-06, + "log_odds_chosen": 0.3661497235298157, + "log_odds_ratio": -0.5325534343719482, + "logits/chosen": 0.4761643409729004, + "logits/rejected": -0.4684900641441345, + "logps/chosen": -1.7682512998580933, + "logps/rejected": -2.079028844833374, + "loss": 1.9513, + "nll_loss": 1.8980205059051514, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1768251359462738, + "rewards/margins": 0.031077751889824867, + "rewards/rejected": -0.20790287852287292, + "step": 181 + }, + { + "epoch": 0.5025888850535036, + "grad_norm": 0.23882247507572174, + "learning_rate": 4.933703026211262e-06, + "log_odds_chosen": 0.40052229166030884, + "log_odds_ratio": -0.5199273824691772, + "logits/chosen": 0.5116904377937317, + "logits/rejected": -0.8737332820892334, + "logps/chosen": -1.7753740549087524, + "logps/rejected": -2.1211981773376465, + "loss": 1.9275, + "nll_loss": 1.8754714727401733, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17753739655017853, + "rewards/margins": 0.03458239883184433, + "rewards/rejected": -0.21211980283260345, + "step": 182 + }, + { + "epoch": 0.5053503624439075, + "grad_norm": 0.23901337385177612, + "learning_rate": 4.931857124637276e-06, + "log_odds_chosen": 0.3831828832626343, + "log_odds_ratio": -0.5294877886772156, + "logits/chosen": 0.44559329748153687, + "logits/rejected": -0.512153685092926, + "logps/chosen": -1.660269021987915, + "logps/rejected": -1.9844969511032104, + "loss": 1.8228, + "nll_loss": 1.7698148488998413, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16602689027786255, + "rewards/margins": 0.032422810792922974, + "rewards/rejected": -0.19844970107078552, + "step": 183 + }, + { + "epoch": 0.5081118398343114, + "grad_norm": 0.26273974776268005, + "learning_rate": 4.92998623196872e-06, + "log_odds_chosen": 0.40838003158569336, + "log_odds_ratio": -0.5120916366577148, + "logits/chosen": 0.36793801188468933, + "logits/rejected": -0.883270263671875, + "logps/chosen": -1.7183860540390015, + "logps/rejected": -2.0646042823791504, + "loss": 1.8679, + "nll_loss": 1.8166720867156982, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17183861136436462, + "rewards/margins": 0.034621842205524445, + "rewards/rejected": -0.20646044611930847, + "step": 184 + }, + { + "epoch": 0.5108733172247152, + "grad_norm": 0.24954968690872192, + "learning_rate": 4.92809036743191e-06, + "log_odds_chosen": 0.44134509563446045, + "log_odds_ratio": -0.5009733438491821, + "logits/chosen": 0.38025662302970886, + "logits/rejected": -0.6959909200668335, + "logps/chosen": -1.6237648725509644, + "logps/rejected": -1.991347312927246, + "loss": 1.7726, + "nll_loss": 1.7225314378738403, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1623764932155609, + "rewards/margins": 0.03675824776291847, + "rewards/rejected": -0.19913475215435028, + "step": 185 + }, + { + "epoch": 0.5136347946151191, + "grad_norm": 0.24785040318965912, + "learning_rate": 4.926169550509787e-06, + "log_odds_chosen": 0.304913192987442, + "log_odds_ratio": -0.5551115274429321, + "logits/chosen": 0.49904897809028625, + "logits/rejected": -0.5060792565345764, + "logps/chosen": -1.752536416053772, + "logps/rejected": -2.009927749633789, + "loss": 1.8926, + "nll_loss": 1.837074875831604, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17525365948677063, + "rewards/margins": 0.0257391519844532, + "rewards/rejected": -0.20099279284477234, + "step": 186 + }, + { + "epoch": 0.516396272005523, + "grad_norm": 0.25260496139526367, + "learning_rate": 4.924223800941718e-06, + "log_odds_chosen": 0.18274395167827606, + "log_odds_ratio": -0.6112073659896851, + "logits/chosen": 0.4272249937057495, + "logits/rejected": -0.5964298248291016, + "logps/chosen": -1.7245608568191528, + "logps/rejected": -1.87729012966156, + "loss": 1.8858, + "nll_loss": 1.824722170829773, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17245608568191528, + "rewards/margins": 0.015272947028279305, + "rewards/rejected": -0.18772903084754944, + "step": 187 + }, + { + "epoch": 0.5191577493959268, + "grad_norm": 0.25974321365356445, + "learning_rate": 4.9222531387232885e-06, + "log_odds_chosen": 0.2772579491138458, + "log_odds_ratio": -0.5700020790100098, + "logits/chosen": 0.5035889148712158, + "logits/rejected": -0.6021788120269775, + "logps/chosen": -1.810874581336975, + "logps/rejected": -2.048560380935669, + "loss": 1.9601, + "nll_loss": 1.903147578239441, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.181087464094162, + "rewards/margins": 0.023768583312630653, + "rewards/rejected": -0.2048560529947281, + "step": 188 + }, + { + "epoch": 0.5219192267863307, + "grad_norm": 0.2509472668170929, + "learning_rate": 4.920257584106104e-06, + "log_odds_chosen": 0.2922664284706116, + "log_odds_ratio": -0.5610611438751221, + "logits/chosen": 0.4223700761795044, + "logits/rejected": -0.8161606192588806, + "logps/chosen": -1.7685585021972656, + "logps/rejected": -2.017383575439453, + "loss": 1.9162, + "nll_loss": 1.8600515127182007, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17685584723949432, + "rewards/margins": 0.02488252893090248, + "rewards/rejected": -0.2017383873462677, + "step": 189 + }, + { + "epoch": 0.5246807041767345, + "grad_norm": 0.26333701610565186, + "learning_rate": 4.918237157597574e-06, + "log_odds_chosen": 0.367781400680542, + "log_odds_ratio": -0.5288804769515991, + "logits/chosen": 0.34821969270706177, + "logits/rejected": -0.5903292894363403, + "logps/chosen": -1.7087079286575317, + "logps/rejected": -2.0187838077545166, + "loss": 1.8792, + "nll_loss": 1.8263163566589355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1708707958459854, + "rewards/margins": 0.03100760281085968, + "rewards/rejected": -0.2018783837556839, + "step": 190 + }, + { + "epoch": 0.5274421815671384, + "grad_norm": 0.2659012973308563, + "learning_rate": 4.916191879960708e-06, + "log_odds_chosen": 0.41825154423713684, + "log_odds_ratio": -0.5096725225448608, + "logits/chosen": 0.3918278217315674, + "logits/rejected": -0.7869745492935181, + "logps/chosen": -1.819606900215149, + "logps/rejected": -2.178121328353882, + "loss": 1.9644, + "nll_loss": 1.9134495258331299, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18196068704128265, + "rewards/margins": 0.03585144132375717, + "rewards/rejected": -0.21781213581562042, + "step": 191 + }, + { + "epoch": 0.5302036589575423, + "grad_norm": 0.2279636263847351, + "learning_rate": 4.914121772213898e-06, + "log_odds_chosen": 0.3906615972518921, + "log_odds_ratio": -0.5185546875, + "logits/chosen": 0.4513569176197052, + "logits/rejected": -0.5819852948188782, + "logps/chosen": -1.6937000751495361, + "logps/rejected": -2.0224616527557373, + "loss": 1.8302, + "nll_loss": 1.778322458267212, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16937001049518585, + "rewards/margins": 0.03287617862224579, + "rewards/rejected": -0.20224617421627045, + "step": 192 + }, + { + "epoch": 0.5329651363479462, + "grad_norm": 0.25850534439086914, + "learning_rate": 4.912026855630703e-06, + "log_odds_chosen": 0.4198724031448364, + "log_odds_ratio": -0.5083851218223572, + "logits/chosen": 0.44823789596557617, + "logits/rejected": -0.45316439867019653, + "logps/chosen": -1.7062734365463257, + "logps/rejected": -2.061528205871582, + "loss": 1.8534, + "nll_loss": 1.8025155067443848, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17062735557556152, + "rewards/margins": 0.035525478422641754, + "rewards/rejected": -0.20615282654762268, + "step": 193 + }, + { + "epoch": 0.5357266137383501, + "grad_norm": 0.2361551970243454, + "learning_rate": 4.909907151739634e-06, + "log_odds_chosen": 0.3865026831626892, + "log_odds_ratio": -0.527399480342865, + "logits/chosen": 0.3759016692638397, + "logits/rejected": -0.6874773502349854, + "logps/chosen": -1.6837282180786133, + "logps/rejected": -2.00811505317688, + "loss": 1.8406, + "nll_loss": 1.7878108024597168, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16837282478809357, + "rewards/margins": 0.03243869170546532, + "rewards/rejected": -0.200811505317688, + "step": 194 + }, + { + "epoch": 0.5384880911287538, + "grad_norm": 0.25059905648231506, + "learning_rate": 4.907762682323926e-06, + "log_odds_chosen": 0.5617655515670776, + "log_odds_ratio": -0.4947778284549713, + "logits/chosen": 0.4162059426307678, + "logits/rejected": -0.6656520366668701, + "logps/chosen": -1.6942963600158691, + "logps/rejected": -2.1570441722869873, + "loss": 1.8694, + "nll_loss": 1.8199554681777954, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16942965984344482, + "rewards/margins": 0.046274758875370026, + "rewards/rejected": -0.21570439636707306, + "step": 195 + }, + { + "epoch": 0.5412495685191577, + "grad_norm": 0.24844138324260712, + "learning_rate": 4.905593469421323e-06, + "log_odds_chosen": 0.2864172160625458, + "log_odds_ratio": -0.5658568143844604, + "logits/chosen": 0.40823209285736084, + "logits/rejected": -1.0584533214569092, + "logps/chosen": -1.6995760202407837, + "logps/rejected": -1.9419208765029907, + "loss": 1.8573, + "nll_loss": 1.8007633686065674, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16995760798454285, + "rewards/margins": 0.024234486743807793, + "rewards/rejected": -0.1941920816898346, + "step": 196 + }, + { + "epoch": 0.5440110459095616, + "grad_norm": 0.25183677673339844, + "learning_rate": 4.90339953532384e-06, + "log_odds_chosen": 0.32565394043922424, + "log_odds_ratio": -0.5478559732437134, + "logits/chosen": 0.4504649341106415, + "logits/rejected": -0.6915105581283569, + "logps/chosen": -1.723501443862915, + "logps/rejected": -1.9977482557296753, + "loss": 1.8793, + "nll_loss": 1.8244800567626953, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17235015332698822, + "rewards/margins": 0.027424685657024384, + "rewards/rejected": -0.199774831533432, + "step": 197 + }, + { + "epoch": 0.5467725232999655, + "grad_norm": 0.24120980501174927, + "learning_rate": 4.901180902577549e-06, + "log_odds_chosen": 0.36884185671806335, + "log_odds_ratio": -0.5313685536384583, + "logits/chosen": 0.47393080592155457, + "logits/rejected": -0.8454681038856506, + "logps/chosen": -1.7091484069824219, + "logps/rejected": -2.021761655807495, + "loss": 1.8562, + "nll_loss": 1.80307936668396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17091482877731323, + "rewards/margins": 0.03126133233308792, + "rewards/rejected": -0.20217616856098175, + "step": 198 + }, + { + "epoch": 0.5495340006903694, + "grad_norm": 0.26695704460144043, + "learning_rate": 4.8989375939823305e-06, + "log_odds_chosen": 0.2716186046600342, + "log_odds_ratio": -0.5816208124160767, + "logits/chosen": 0.43514788150787354, + "logits/rejected": -0.9399389028549194, + "logps/chosen": -1.6752939224243164, + "logps/rejected": -1.9062312841415405, + "loss": 1.829, + "nll_loss": 1.770880937576294, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1675294041633606, + "rewards/margins": 0.023093728348612785, + "rewards/rejected": -0.19062313437461853, + "step": 199 + }, + { + "epoch": 0.5522954780807732, + "grad_norm": 0.248373344540596, + "learning_rate": 4.896669632591652e-06, + "log_odds_chosen": 0.43290525674819946, + "log_odds_ratio": -0.5045643448829651, + "logits/chosen": 0.41075634956359863, + "logits/rejected": -0.6819908618927002, + "logps/chosen": -1.6926066875457764, + "logps/rejected": -2.0592057704925537, + "loss": 1.8492, + "nll_loss": 1.7987439632415771, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1692606657743454, + "rewards/margins": 0.03665991127490997, + "rewards/rejected": -0.20592057704925537, + "step": 200 + }, + { + "epoch": 0.5550569554711771, + "grad_norm": 0.2426389902830124, + "learning_rate": 4.894377041712327e-06, + "log_odds_chosen": 0.3046472668647766, + "log_odds_ratio": -0.5560302138328552, + "logits/chosen": 0.4586237370967865, + "logits/rejected": -0.889277458190918, + "logps/chosen": -1.7483210563659668, + "logps/rejected": -2.0069689750671387, + "loss": 1.8995, + "nll_loss": 1.8438801765441895, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17483210563659668, + "rewards/margins": 0.025864800438284874, + "rewards/rejected": -0.2006969153881073, + "step": 201 + }, + { + "epoch": 0.557818432861581, + "grad_norm": 0.2384442389011383, + "learning_rate": 4.892059844904273e-06, + "log_odds_chosen": 0.35407793521881104, + "log_odds_ratio": -0.5373241901397705, + "logits/chosen": 0.3391227722167969, + "logits/rejected": -0.8238649964332581, + "logps/chosen": -1.6727503538131714, + "logps/rejected": -1.971817135810852, + "loss": 1.8314, + "nll_loss": 1.7776691913604736, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16727504134178162, + "rewards/margins": 0.02990666963160038, + "rewards/rejected": -0.19718170166015625, + "step": 202 + }, + { + "epoch": 0.5605799102519848, + "grad_norm": 0.2237497717142105, + "learning_rate": 4.889718065980272e-06, + "log_odds_chosen": 0.42560863494873047, + "log_odds_ratio": -0.5102251768112183, + "logits/chosen": 0.38399800658226013, + "logits/rejected": -0.940986156463623, + "logps/chosen": -1.7369155883789062, + "logps/rejected": -2.0991053581237793, + "loss": 1.8724, + "nll_loss": 1.821379542350769, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1736915558576584, + "rewards/margins": 0.03621895611286163, + "rewards/rejected": -0.2099105268716812, + "step": 203 + }, + { + "epoch": 0.5633413876423887, + "grad_norm": 0.23220422863960266, + "learning_rate": 4.8873517290057265e-06, + "log_odds_chosen": 0.5465662479400635, + "log_odds_ratio": -0.45910075306892395, + "logits/chosen": 0.33986854553222656, + "logits/rejected": -0.9626595973968506, + "logps/chosen": -1.7113467454910278, + "logps/rejected": -2.1808066368103027, + "loss": 1.8594, + "nll_loss": 1.8135318756103516, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17113468050956726, + "rewards/margins": 0.046945974230766296, + "rewards/rejected": -0.21808065474033356, + "step": 204 + }, + { + "epoch": 0.5661028650327925, + "grad_norm": 0.22727084159851074, + "learning_rate": 4.88496085829841e-06, + "log_odds_chosen": 0.3183567523956299, + "log_odds_ratio": -0.5485174059867859, + "logits/chosen": 0.38077837228775024, + "logits/rejected": -0.3718608617782593, + "logps/chosen": -1.682438611984253, + "logps/rejected": -1.9494574069976807, + "loss": 1.8312, + "nll_loss": 1.7763221263885498, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.168243870139122, + "rewards/margins": 0.026701876893639565, + "rewards/rejected": -0.19494575262069702, + "step": 205 + }, + { + "epoch": 0.5688643424231964, + "grad_norm": 0.2435847967863083, + "learning_rate": 4.882545478428219e-06, + "log_odds_chosen": 0.4757111072540283, + "log_odds_ratio": -0.5007758140563965, + "logits/chosen": 0.4316224753856659, + "logits/rejected": -0.9769002199172974, + "logps/chosen": -1.6722346544265747, + "logps/rejected": -2.073765277862549, + "loss": 1.8362, + "nll_loss": 1.7861530780792236, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1672234833240509, + "rewards/margins": 0.04015304520726204, + "rewards/rejected": -0.20737652480602264, + "step": 206 + }, + { + "epoch": 0.5716258198136003, + "grad_norm": 0.28022557497024536, + "learning_rate": 4.880105614216917e-06, + "log_odds_chosen": 0.42549797892570496, + "log_odds_ratio": -0.507975161075592, + "logits/chosen": 0.42804068326950073, + "logits/rejected": -0.8541020750999451, + "logps/chosen": -1.8415427207946777, + "logps/rejected": -2.209498405456543, + "loss": 1.9733, + "nll_loss": 1.9224536418914795, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18415425717830658, + "rewards/margins": 0.03679555654525757, + "rewards/rejected": -0.22094982862472534, + "step": 207 + }, + { + "epoch": 0.5743872972040042, + "grad_norm": 0.26431816816329956, + "learning_rate": 4.8776412907378845e-06, + "log_odds_chosen": 0.3722533583641052, + "log_odds_ratio": -0.5265185832977295, + "logits/chosen": 0.46087294816970825, + "logits/rejected": -0.8973901271820068, + "logps/chosen": -1.7529377937316895, + "logps/rejected": -2.0698721408843994, + "loss": 1.8939, + "nll_loss": 1.8412883281707764, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17529378831386566, + "rewards/margins": 0.03169342502951622, + "rewards/rejected": -0.206987202167511, + "step": 208 + }, + { + "epoch": 0.577148774594408, + "grad_norm": 0.2286507785320282, + "learning_rate": 4.875152533315859e-06, + "log_odds_chosen": 0.42754417657852173, + "log_odds_ratio": -0.5049228668212891, + "logits/chosen": 0.3295610547065735, + "logits/rejected": -0.821418285369873, + "logps/chosen": -1.6234990358352661, + "logps/rejected": -1.9790537357330322, + "loss": 1.7729, + "nll_loss": 1.7223597764968872, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1623499095439911, + "rewards/margins": 0.035555459558963776, + "rewards/rejected": -0.19790537655353546, + "step": 209 + }, + { + "epoch": 0.5799102519848118, + "grad_norm": 0.23942524194717407, + "learning_rate": 4.872639367526672e-06, + "log_odds_chosen": 0.4114229083061218, + "log_odds_ratio": -0.5152523517608643, + "logits/chosen": 0.3920520842075348, + "logits/rejected": -0.7173675298690796, + "logps/chosen": -1.6646122932434082, + "logps/rejected": -2.0126848220825195, + "loss": 1.8132, + "nll_loss": 1.7616626024246216, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16646124422550201, + "rewards/margins": 0.0348072350025177, + "rewards/rejected": -0.20126846432685852, + "step": 210 + }, + { + "epoch": 0.5826717293752157, + "grad_norm": 0.259086012840271, + "learning_rate": 4.870101819196992e-06, + "log_odds_chosen": 0.37038204073905945, + "log_odds_ratio": -0.5485732555389404, + "logits/chosen": 0.33591514825820923, + "logits/rejected": -0.7169915437698364, + "logps/chosen": -1.7096450328826904, + "logps/rejected": -2.028204917907715, + "loss": 1.863, + "nll_loss": 1.8081302642822266, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17096450924873352, + "rewards/margins": 0.031855978071689606, + "rewards/rejected": -0.20282049477100372, + "step": 211 + }, + { + "epoch": 0.5854332067656196, + "grad_norm": 0.23236605525016785, + "learning_rate": 4.8675399144040535e-06, + "log_odds_chosen": 0.48778051137924194, + "log_odds_ratio": -0.4863608777523041, + "logits/chosen": 0.3406648337841034, + "logits/rejected": -1.1020050048828125, + "logps/chosen": -1.6164040565490723, + "logps/rejected": -2.0246694087982178, + "loss": 1.7551, + "nll_loss": 1.7064510583877563, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16164040565490723, + "rewards/margins": 0.04082653671503067, + "rewards/rejected": -0.2024669200181961, + "step": 212 + }, + { + "epoch": 0.5881946841560235, + "grad_norm": 0.2472049593925476, + "learning_rate": 4.864953679475392e-06, + "log_odds_chosen": 0.4090927541255951, + "log_odds_ratio": -0.5185278654098511, + "logits/chosen": 0.3806914687156677, + "logits/rejected": -1.1645220518112183, + "logps/chosen": -1.6705214977264404, + "logps/rejected": -2.010647773742676, + "loss": 1.7979, + "nll_loss": 1.7460622787475586, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16705216467380524, + "rewards/margins": 0.03401262313127518, + "rewards/rejected": -0.201064795255661, + "step": 213 + }, + { + "epoch": 0.5909561615464274, + "grad_norm": 0.26932069659233093, + "learning_rate": 4.862343140988573e-06, + "log_odds_chosen": 0.38164323568344116, + "log_odds_ratio": -0.5293680429458618, + "logits/chosen": 0.35298576951026917, + "logits/rejected": -0.9924853444099426, + "logps/chosen": -1.7351062297821045, + "logps/rejected": -2.062347173690796, + "loss": 1.8652, + "nll_loss": 1.8122597932815552, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1735106110572815, + "rewards/margins": 0.03272408992052078, + "rewards/rejected": -0.20623470842838287, + "step": 214 + }, + { + "epoch": 0.5937176389368312, + "grad_norm": 0.26750069856643677, + "learning_rate": 4.859708325770919e-06, + "log_odds_chosen": 0.3453901410102844, + "log_odds_ratio": -0.5410705804824829, + "logits/chosen": 0.402724027633667, + "logits/rejected": -0.7651013135910034, + "logps/chosen": -1.736122727394104, + "logps/rejected": -2.027735948562622, + "loss": 1.901, + "nll_loss": 1.8469077348709106, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17361226677894592, + "rewards/margins": 0.029161330312490463, + "rewards/rejected": -0.20277361571788788, + "step": 215 + }, + { + "epoch": 0.596479116327235, + "grad_norm": 0.25545212626457214, + "learning_rate": 4.857049260899233e-06, + "log_odds_chosen": 0.42634040117263794, + "log_odds_ratio": -0.5089380145072937, + "logits/chosen": 0.2999016046524048, + "logits/rejected": -0.9428575038909912, + "logps/chosen": -1.6035995483398438, + "logps/rejected": -1.9544857740402222, + "loss": 1.7566, + "nll_loss": 1.7056996822357178, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1603599637746811, + "rewards/margins": 0.03508862853050232, + "rewards/rejected": -0.19544857740402222, + "step": 216 + }, + { + "epoch": 0.5992405937176389, + "grad_norm": 0.2341543734073639, + "learning_rate": 4.854365973699519e-06, + "log_odds_chosen": 0.29267483949661255, + "log_odds_ratio": -0.5718191862106323, + "logits/chosen": 0.32583779096603394, + "logits/rejected": -0.7815302610397339, + "logps/chosen": -1.6951828002929688, + "logps/rejected": -1.9353394508361816, + "loss": 1.8596, + "nll_loss": 1.8024464845657349, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16951829195022583, + "rewards/margins": 0.024015672504901886, + "rewards/rejected": -0.19353395700454712, + "step": 217 + }, + { + "epoch": 0.6020020711080428, + "grad_norm": 0.24366186559200287, + "learning_rate": 4.851658491746707e-06, + "log_odds_chosen": 0.3909580707550049, + "log_odds_ratio": -0.5210633277893066, + "logits/chosen": 0.25465232133865356, + "logits/rejected": -0.9531072378158569, + "logps/chosen": -1.6791120767593384, + "logps/rejected": -2.0105059146881104, + "loss": 1.8187, + "nll_loss": 1.7665891647338867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16791123151779175, + "rewards/margins": 0.03313935548067093, + "rewards/rejected": -0.20105057954788208, + "step": 218 + }, + { + "epoch": 0.6047635484984467, + "grad_norm": 0.24001233279705048, + "learning_rate": 4.848926842864361e-06, + "log_odds_chosen": 0.4899923503398895, + "log_odds_ratio": -0.48127448558807373, + "logits/chosen": 0.25574028491973877, + "logits/rejected": -1.256288766860962, + "logps/chosen": -1.6502352952957153, + "logps/rejected": -2.064323902130127, + "loss": 1.7891, + "nll_loss": 1.740965723991394, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.165023535490036, + "rewards/margins": 0.04140886664390564, + "rewards/rejected": -0.20643240213394165, + "step": 219 + }, + { + "epoch": 0.6075250258888505, + "grad_norm": 0.23009978234767914, + "learning_rate": 4.846171055124401e-06, + "log_odds_chosen": 0.3280143141746521, + "log_odds_ratio": -0.55921870470047, + "logits/chosen": 0.35827893018722534, + "logits/rejected": -0.7289958000183105, + "logps/chosen": -1.7372446060180664, + "logps/rejected": -2.02004075050354, + "loss": 1.8677, + "nll_loss": 1.811750888824463, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1737244725227356, + "rewards/margins": 0.028279609978199005, + "rewards/rejected": -0.2020040899515152, + "step": 220 + }, + { + "epoch": 0.6102865032792544, + "grad_norm": 0.24643385410308838, + "learning_rate": 4.843391156846811e-06, + "log_odds_chosen": 0.49847206473350525, + "log_odds_ratio": -0.4771438539028168, + "logits/chosen": 0.38028058409690857, + "logits/rejected": -0.8463162779808044, + "logps/chosen": -1.6676466464996338, + "logps/rejected": -2.0871989727020264, + "loss": 1.8045, + "nll_loss": 1.756779432296753, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16676466166973114, + "rewards/margins": 0.04195523262023926, + "rewards/rejected": -0.2087198942899704, + "step": 221 + }, + { + "epoch": 0.6130479806696583, + "grad_norm": 0.23624517023563385, + "learning_rate": 4.8405871765993435e-06, + "log_odds_chosen": 0.5461086630821228, + "log_odds_ratio": -0.4661559462547302, + "logits/chosen": 0.2758100926876068, + "logits/rejected": -0.9698406457901001, + "logps/chosen": -1.716646671295166, + "logps/rejected": -2.178037643432617, + "loss": 1.8539, + "nll_loss": 1.807306170463562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17166468501091003, + "rewards/margins": 0.04613909870386124, + "rewards/rejected": -0.21780376136302948, + "step": 222 + }, + { + "epoch": 0.6158094580600622, + "grad_norm": 0.24216988682746887, + "learning_rate": 4.837759143197237e-06, + "log_odds_chosen": 0.33277636766433716, + "log_odds_ratio": -0.543420672416687, + "logits/chosen": 0.25846540927886963, + "logits/rejected": -1.1584213972091675, + "logps/chosen": -1.6766406297683716, + "logps/rejected": -1.959399700164795, + "loss": 1.818, + "nll_loss": 1.7636725902557373, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1676640659570694, + "rewards/margins": 0.028275907039642334, + "rewards/rejected": -0.19593995809555054, + "step": 223 + }, + { + "epoch": 0.618570935450466, + "grad_norm": 0.22247816622257233, + "learning_rate": 4.834907085702909e-06, + "log_odds_chosen": 0.42663368582725525, + "log_odds_ratio": -0.5054609775543213, + "logits/chosen": 0.2946387827396393, + "logits/rejected": -0.8483214378356934, + "logps/chosen": -1.6203222274780273, + "logps/rejected": -1.9752922058105469, + "loss": 1.7654, + "nll_loss": 1.7148088216781616, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16203221678733826, + "rewards/margins": 0.03549701347947121, + "rewards/rejected": -0.19752921164035797, + "step": 224 + }, + { + "epoch": 0.6213324128408698, + "grad_norm": 0.23438099026679993, + "learning_rate": 4.832031033425663e-06, + "log_odds_chosen": 0.4974308907985687, + "log_odds_ratio": -0.4770738184452057, + "logits/chosen": 0.22860127687454224, + "logits/rejected": -1.1651134490966797, + "logps/chosen": -1.5770584344863892, + "logps/rejected": -1.98891282081604, + "loss": 1.7128, + "nll_loss": 1.6650840044021606, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15770584344863892, + "rewards/margins": 0.04118544980883598, + "rewards/rejected": -0.198891282081604, + "step": 225 + }, + { + "epoch": 0.6240938902312737, + "grad_norm": 0.2453928291797638, + "learning_rate": 4.829131015921386e-06, + "log_odds_chosen": 0.336994469165802, + "log_odds_ratio": -0.5416135787963867, + "logits/chosen": 0.40966925024986267, + "logits/rejected": -0.7484245300292969, + "logps/chosen": -1.7475402355194092, + "logps/rejected": -2.035691499710083, + "loss": 1.8881, + "nll_loss": 1.833910346031189, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17475402355194092, + "rewards/margins": 0.0288151316344738, + "rewards/rejected": -0.2035691738128662, + "step": 226 + }, + { + "epoch": 0.6268553676216776, + "grad_norm": 0.22821033000946045, + "learning_rate": 4.826207062992245e-06, + "log_odds_chosen": 0.47645366191864014, + "log_odds_ratio": -0.4914059638977051, + "logits/chosen": 0.2782054543495178, + "logits/rejected": -0.9900674223899841, + "logps/chosen": -1.8174843788146973, + "logps/rejected": -2.2320914268493652, + "loss": 1.95, + "nll_loss": 1.9008376598358154, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1817484349012375, + "rewards/margins": 0.041460707783699036, + "rewards/rejected": -0.22320915758609772, + "step": 227 + }, + { + "epoch": 0.6296168450120815, + "grad_norm": 0.2569068968296051, + "learning_rate": 4.82325920468638e-06, + "log_odds_chosen": 0.2413053661584854, + "log_odds_ratio": -0.5858334302902222, + "logits/chosen": 0.32741737365722656, + "logits/rejected": -0.9221272468566895, + "logps/chosen": -1.7791626453399658, + "logps/rejected": -1.9837150573730469, + "loss": 1.9203, + "nll_loss": 1.8617504835128784, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1779162585735321, + "rewards/margins": 0.020455272868275642, + "rewards/rejected": -0.1983715444803238, + "step": 228 + }, + { + "epoch": 0.6323783224024854, + "grad_norm": 0.2548074424266815, + "learning_rate": 4.820287471297598e-06, + "log_odds_chosen": 0.4002269208431244, + "log_odds_ratio": -0.5239609479904175, + "logits/chosen": 0.21428313851356506, + "logits/rejected": -0.9460724592208862, + "logps/chosen": -1.6473997831344604, + "logps/rejected": -1.9846879243850708, + "loss": 1.7941, + "nll_loss": 1.7416696548461914, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1647399663925171, + "rewards/margins": 0.03372883051633835, + "rewards/rejected": -0.19846880435943604, + "step": 229 + }, + { + "epoch": 0.6351397997928891, + "grad_norm": 0.22254504263401031, + "learning_rate": 4.817291893365055e-06, + "log_odds_chosen": 0.46622079610824585, + "log_odds_ratio": -0.4967763423919678, + "logits/chosen": 0.07556484639644623, + "logits/rejected": -1.116629719734192, + "logps/chosen": -1.5138487815856934, + "logps/rejected": -1.8917471170425415, + "loss": 1.675, + "nll_loss": 1.625287413597107, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15138490498065948, + "rewards/margins": 0.03778982535004616, + "rewards/rejected": -0.18917471170425415, + "step": 230 + }, + { + "epoch": 0.637901277183293, + "grad_norm": 0.28197240829467773, + "learning_rate": 4.81427250167295e-06, + "log_odds_chosen": 0.24183571338653564, + "log_odds_ratio": -0.583247721195221, + "logits/chosen": 0.15436850488185883, + "logits/rejected": -1.1790400743484497, + "logps/chosen": -1.6747009754180908, + "logps/rejected": -1.8760582208633423, + "loss": 1.8184, + "nll_loss": 1.7600996494293213, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16747009754180908, + "rewards/margins": 0.020135723054409027, + "rewards/rejected": -0.1876058131456375, + "step": 231 + }, + { + "epoch": 0.6406627545736969, + "grad_norm": 0.24836094677448273, + "learning_rate": 4.811229327250204e-06, + "log_odds_chosen": 0.45771628618240356, + "log_odds_ratio": -0.5010443329811096, + "logits/chosen": 0.25003117322921753, + "logits/rejected": -1.1424946784973145, + "logps/chosen": -1.714363694190979, + "logps/rejected": -2.1092028617858887, + "loss": 1.8544, + "nll_loss": 1.8042795658111572, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1714363694190979, + "rewards/margins": 0.03948391228914261, + "rewards/rejected": -0.2109202891588211, + "step": 232 + }, + { + "epoch": 0.6434242319641008, + "grad_norm": 0.23518826067447662, + "learning_rate": 4.8081624013701435e-06, + "log_odds_chosen": 0.4311872124671936, + "log_odds_ratio": -0.5104647278785706, + "logits/chosen": 0.1991138905286789, + "logits/rejected": -1.3669785261154175, + "logps/chosen": -1.6888374090194702, + "logps/rejected": -2.053189992904663, + "loss": 1.8231, + "nll_loss": 1.7720434665679932, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16888374090194702, + "rewards/margins": 0.03643525391817093, + "rewards/rejected": -0.20531900227069855, + "step": 233 + }, + { + "epoch": 0.6461857093545047, + "grad_norm": 0.23462118208408356, + "learning_rate": 4.805071755550177e-06, + "log_odds_chosen": 0.394045889377594, + "log_odds_ratio": -0.5184462070465088, + "logits/chosen": 0.2896556854248047, + "logits/rejected": -1.3101093769073486, + "logps/chosen": -1.713099479675293, + "logps/rejected": -2.048048496246338, + "loss": 1.8398, + "nll_loss": 1.7879210710525513, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1713099479675293, + "rewards/margins": 0.03349488973617554, + "rewards/rejected": -0.20480485260486603, + "step": 234 + }, + { + "epoch": 0.6489471867449085, + "grad_norm": 0.23507662117481232, + "learning_rate": 4.8019574215514705e-06, + "log_odds_chosen": 0.3388752043247223, + "log_odds_ratio": -0.5402787327766418, + "logits/chosen": 0.20577961206436157, + "logits/rejected": -0.8569263219833374, + "logps/chosen": -1.6788839101791382, + "logps/rejected": -1.9616367816925049, + "loss": 1.8351, + "nll_loss": 1.7810907363891602, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16788838803768158, + "rewards/margins": 0.028275297954678535, + "rewards/rejected": -0.19616368412971497, + "step": 235 + }, + { + "epoch": 0.6517086641353124, + "grad_norm": 0.2070939689874649, + "learning_rate": 4.7988194313786275e-06, + "log_odds_chosen": 0.37065887451171875, + "log_odds_ratio": -0.5329977869987488, + "logits/chosen": 0.22679734230041504, + "logits/rejected": -0.9689663052558899, + "logps/chosen": -1.629687786102295, + "logps/rejected": -1.9382522106170654, + "loss": 1.7528, + "nll_loss": 1.6994796991348267, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16296879947185516, + "rewards/margins": 0.03085644729435444, + "rewards/rejected": -0.19382524490356445, + "step": 236 + }, + { + "epoch": 0.6544701415257163, + "grad_norm": 0.2251017838716507, + "learning_rate": 4.795657817279349e-06, + "log_odds_chosen": 0.3826752305030823, + "log_odds_ratio": -0.5331037044525146, + "logits/chosen": 0.143580362200737, + "logits/rejected": -1.0959559679031372, + "logps/chosen": -1.568124532699585, + "logps/rejected": -1.8851323127746582, + "loss": 1.7164, + "nll_loss": 1.663122296333313, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15681245923042297, + "rewards/margins": 0.03170077130198479, + "rewards/rejected": -0.18851323425769806, + "step": 237 + }, + { + "epoch": 0.6572316189161201, + "grad_norm": 0.24112077057361603, + "learning_rate": 4.7924726117441135e-06, + "log_odds_chosen": 0.45494502782821655, + "log_odds_ratio": -0.5097072124481201, + "logits/chosen": 0.12531203031539917, + "logits/rejected": -1.1530197858810425, + "logps/chosen": -1.6881966590881348, + "logps/rejected": -2.074317216873169, + "loss": 1.8279, + "nll_loss": 1.7769427299499512, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16881968080997467, + "rewards/margins": 0.03861205279827118, + "rewards/rejected": -0.20743173360824585, + "step": 238 + }, + { + "epoch": 0.659993096306524, + "grad_norm": 0.23394882678985596, + "learning_rate": 4.789263847505835e-06, + "log_odds_chosen": 0.48239994049072266, + "log_odds_ratio": -0.484576016664505, + "logits/chosen": 0.19327585399150848, + "logits/rejected": -1.0120890140533447, + "logps/chosen": -1.673211693763733, + "logps/rejected": -2.078700542449951, + "loss": 1.8042, + "nll_loss": 1.7557491064071655, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16732117533683777, + "rewards/margins": 0.04054888337850571, + "rewards/rejected": -0.20787005126476288, + "step": 239 + }, + { + "epoch": 0.6627545736969278, + "grad_norm": 0.24529801309108734, + "learning_rate": 4.786031557539532e-06, + "log_odds_chosen": 0.6541503667831421, + "log_odds_ratio": -0.42781609296798706, + "logits/chosen": 0.1258109211921692, + "logits/rejected": -1.3018333911895752, + "logps/chosen": -1.6192426681518555, + "logps/rejected": -2.17771577835083, + "loss": 1.7489, + "nll_loss": 1.7061513662338257, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16192427277565002, + "rewards/margins": 0.055847302079200745, + "rewards/rejected": -0.21777155995368958, + "step": 240 + }, + { + "epoch": 0.6655160510873317, + "grad_norm": 0.21988217532634735, + "learning_rate": 4.782775775061983e-06, + "log_odds_chosen": 0.4216863214969635, + "log_odds_ratio": -0.5078084468841553, + "logits/chosen": 0.22119548916816711, + "logits/rejected": -1.0060391426086426, + "logps/chosen": -1.6245155334472656, + "logps/rejected": -1.9757689237594604, + "loss": 1.7524, + "nll_loss": 1.7016619443893433, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16245155036449432, + "rewards/margins": 0.03512535244226456, + "rewards/rejected": -0.19757691025733948, + "step": 241 + }, + { + "epoch": 0.6682775284777356, + "grad_norm": 0.20372723042964935, + "learning_rate": 4.779496533531393e-06, + "log_odds_chosen": 0.5198055505752563, + "log_odds_ratio": -0.4691554009914398, + "logits/chosen": 0.20225608348846436, + "logits/rejected": -1.1654834747314453, + "logps/chosen": -1.5811524391174316, + "logps/rejected": -2.01274037361145, + "loss": 1.7073, + "nll_loss": 1.6603847742080688, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15811526775360107, + "rewards/margins": 0.043158773332834244, + "rewards/rejected": -0.20127403736114502, + "step": 242 + }, + { + "epoch": 0.6710390058681395, + "grad_norm": 0.22578665614128113, + "learning_rate": 4.7761938666470405e-06, + "log_odds_chosen": 0.4481002688407898, + "log_odds_ratio": -0.5030589699745178, + "logits/chosen": 0.21000558137893677, + "logits/rejected": -1.1474783420562744, + "logps/chosen": -1.6968859434127808, + "logps/rejected": -2.079129695892334, + "loss": 1.8331, + "nll_loss": 1.7828097343444824, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16968859732151031, + "rewards/margins": 0.03822438418865204, + "rewards/rejected": -0.20791295170783997, + "step": 243 + }, + { + "epoch": 0.6738004832585434, + "grad_norm": 0.21789832413196564, + "learning_rate": 4.7728678083489375e-06, + "log_odds_chosen": 0.3310457170009613, + "log_odds_ratio": -0.5467555522918701, + "logits/chosen": 0.21008087694644928, + "logits/rejected": -1.071950912475586, + "logps/chosen": -1.6163996458053589, + "logps/rejected": -1.8934293985366821, + "loss": 1.7531, + "nll_loss": 1.6983906030654907, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1616399735212326, + "rewards/margins": 0.027702966704964638, + "rewards/rejected": -0.1893429309129715, + "step": 244 + }, + { + "epoch": 0.6765619606489471, + "grad_norm": 0.21386297047138214, + "learning_rate": 4.7695183928174804e-06, + "log_odds_chosen": 0.4576282799243927, + "log_odds_ratio": -0.4972766637802124, + "logits/chosen": 0.11432070285081863, + "logits/rejected": -1.215945839881897, + "logps/chosen": -1.6433749198913574, + "logps/rejected": -2.0280091762542725, + "loss": 1.7829, + "nll_loss": 1.7331418991088867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16433750092983246, + "rewards/margins": 0.03846340626478195, + "rewards/rejected": -0.2028008997440338, + "step": 245 + }, + { + "epoch": 0.679323438039351, + "grad_norm": 0.23491686582565308, + "learning_rate": 4.766145654473096e-06, + "log_odds_chosen": 0.37732306122779846, + "log_odds_ratio": -0.5303936004638672, + "logits/chosen": 0.18962648510932922, + "logits/rejected": -1.3029686212539673, + "logps/chosen": -1.716407299041748, + "logps/rejected": -2.0386903285980225, + "loss": 1.8439, + "nll_loss": 1.790850043296814, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17164072394371033, + "rewards/margins": 0.03222829848527908, + "rewards/rejected": -0.2038690447807312, + "step": 246 + }, + { + "epoch": 0.6820849154297549, + "grad_norm": 0.2103213518857956, + "learning_rate": 4.762749627975888e-06, + "log_odds_chosen": 0.40977245569229126, + "log_odds_ratio": -0.5140390396118164, + "logits/chosen": 0.0982648953795433, + "logits/rejected": -1.1915839910507202, + "logps/chosen": -1.557995319366455, + "logps/rejected": -1.8929803371429443, + "loss": 1.6972, + "nll_loss": 1.6458226442337036, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1557995229959488, + "rewards/margins": 0.033498503267765045, + "rewards/rejected": -0.18929803371429443, + "step": 247 + }, + { + "epoch": 0.6848463928201588, + "grad_norm": 0.2225971668958664, + "learning_rate": 4.7593303482252835e-06, + "log_odds_chosen": 0.36240053176879883, + "log_odds_ratio": -0.5350769758224487, + "logits/chosen": 0.16154634952545166, + "logits/rejected": -0.829785943031311, + "logps/chosen": -1.6611778736114502, + "logps/rejected": -1.9659464359283447, + "loss": 1.7928, + "nll_loss": 1.7392576932907104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16611778736114502, + "rewards/margins": 0.03047686442732811, + "rewards/rejected": -0.19659464061260223, + "step": 248 + }, + { + "epoch": 0.6876078702105627, + "grad_norm": 0.19730301201343536, + "learning_rate": 4.755887850359673e-06, + "log_odds_chosen": 0.598614513874054, + "log_odds_ratio": -0.4566551446914673, + "logits/chosen": 0.10718496143817902, + "logits/rejected": -1.343011498451233, + "logps/chosen": -1.5110077857971191, + "logps/rejected": -2.002279043197632, + "loss": 1.6562, + "nll_loss": 1.6105355024337769, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1511007696390152, + "rewards/margins": 0.04912712052464485, + "rewards/rejected": -0.20022790133953094, + "step": 249 + }, + { + "epoch": 0.6903693476009665, + "grad_norm": 0.2222234308719635, + "learning_rate": 4.752422169756048e-06, + "log_odds_chosen": 0.4699355959892273, + "log_odds_ratio": -0.4920656085014343, + "logits/chosen": 0.1497102826833725, + "logits/rejected": -1.4104342460632324, + "logps/chosen": -1.6844984292984009, + "logps/rejected": -2.0840110778808594, + "loss": 1.8276, + "nll_loss": 1.7784277200698853, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16844984889030457, + "rewards/margins": 0.03995127975940704, + "rewards/rejected": -0.20840111374855042, + "step": 250 + }, + { + "epoch": 0.6931308249913704, + "grad_norm": 0.2100534588098526, + "learning_rate": 4.748933342029639e-06, + "log_odds_chosen": 0.5825514793395996, + "log_odds_ratio": -0.4510309398174286, + "logits/chosen": 0.1121891662478447, + "logits/rejected": -1.3764441013336182, + "logps/chosen": -1.560391902923584, + "logps/rejected": -2.048152446746826, + "loss": 1.7094, + "nll_loss": 1.6642597913742065, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15603917837142944, + "rewards/margins": 0.0487760454416275, + "rewards/rejected": -0.20481522381305695, + "step": 251 + }, + { + "epoch": 0.6958923023817742, + "grad_norm": 0.20206160843372345, + "learning_rate": 4.745421403033548e-06, + "log_odds_chosen": 0.4050910174846649, + "log_odds_ratio": -0.5125004649162292, + "logits/chosen": 0.10816405713558197, + "logits/rejected": -1.1761468648910522, + "logps/chosen": -1.6354482173919678, + "logps/rejected": -1.9715569019317627, + "loss": 1.7615, + "nll_loss": 1.7102546691894531, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16354484856128693, + "rewards/margins": 0.033610858023166656, + "rewards/rejected": -0.1971556842327118, + "step": 252 + }, + { + "epoch": 0.6986537797721781, + "grad_norm": 0.2380354106426239, + "learning_rate": 4.741886388858384e-06, + "log_odds_chosen": 0.3426484167575836, + "log_odds_ratio": -0.543499231338501, + "logits/chosen": 0.17392773926258087, + "logits/rejected": -1.2038366794586182, + "logps/chosen": -1.6468617916107178, + "logps/rejected": -1.928429365158081, + "loss": 1.79, + "nll_loss": 1.7356586456298828, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1646861881017685, + "rewards/margins": 0.02815674990415573, + "rewards/rejected": -0.19284293055534363, + "step": 253 + }, + { + "epoch": 0.701415257162582, + "grad_norm": 0.2207070291042328, + "learning_rate": 4.738328335831883e-06, + "log_odds_chosen": 0.3862311542034149, + "log_odds_ratio": -0.5232746005058289, + "logits/chosen": 0.13721241056919098, + "logits/rejected": -1.4115301370620728, + "logps/chosen": -1.612336277961731, + "logps/rejected": -1.9344983100891113, + "loss": 1.7501, + "nll_loss": 1.6977391242980957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16123361885547638, + "rewards/margins": 0.03221620246767998, + "rewards/rejected": -0.19344982504844666, + "step": 254 + }, + { + "epoch": 0.7041767345529858, + "grad_norm": 0.2061997801065445, + "learning_rate": 4.734747280518549e-06, + "log_odds_chosen": 0.47361427545547485, + "log_odds_ratio": -0.48697221279144287, + "logits/chosen": 0.03231241926550865, + "logits/rejected": -1.5338258743286133, + "logps/chosen": -1.6500543355941772, + "logps/rejected": -2.0473241806030273, + "loss": 1.7861, + "nll_loss": 1.737368106842041, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16500544548034668, + "rewards/margins": 0.039726972579956055, + "rewards/rejected": -0.20473241806030273, + "step": 255 + }, + { + "epoch": 0.7069382119433897, + "grad_norm": 0.20688936114311218, + "learning_rate": 4.7311432597192655e-06, + "log_odds_chosen": 0.39435428380966187, + "log_odds_ratio": -0.5248243808746338, + "logits/chosen": 0.03083261288702488, + "logits/rejected": -1.4570684432983398, + "logps/chosen": -1.648856520652771, + "logps/rejected": -1.9787046909332275, + "loss": 1.7765, + "nll_loss": 1.7240355014801025, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16488566994667053, + "rewards/margins": 0.03298482671380043, + "rewards/rejected": -0.19787049293518066, + "step": 256 + }, + { + "epoch": 0.7096996893337936, + "grad_norm": 0.22945424914360046, + "learning_rate": 4.72751631047092e-06, + "log_odds_chosen": 0.5417366027832031, + "log_odds_ratio": -0.46491706371307373, + "logits/chosen": -0.006479084491729736, + "logits/rejected": -1.1154435873031616, + "logps/chosen": -1.6812236309051514, + "logps/rejected": -2.1395444869995117, + "loss": 1.8262, + "nll_loss": 1.7797247171401978, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16812236607074738, + "rewards/margins": 0.0458320677280426, + "rewards/rejected": -0.21395443379878998, + "step": 257 + }, + { + "epoch": 0.7124611667241975, + "grad_norm": 0.22095176577568054, + "learning_rate": 4.72386647004603e-06, + "log_odds_chosen": 0.4106917977333069, + "log_odds_ratio": -0.5133163332939148, + "logits/chosen": 0.1232781782746315, + "logits/rejected": -1.1046854257583618, + "logps/chosen": -1.6649987697601318, + "logps/rejected": -2.0084848403930664, + "loss": 1.811, + "nll_loss": 1.7596914768218994, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16649989783763885, + "rewards/margins": 0.03434859216213226, + "rewards/rejected": -0.20084848999977112, + "step": 258 + }, + { + "epoch": 0.7152226441146013, + "grad_norm": 0.22131314873695374, + "learning_rate": 4.720193775952352e-06, + "log_odds_chosen": 0.2770199477672577, + "log_odds_ratio": -0.5675607919692993, + "logits/chosen": 0.09717811644077301, + "logits/rejected": -1.1877737045288086, + "logps/chosen": -1.6597505807876587, + "logps/rejected": -1.889330267906189, + "loss": 1.7861, + "nll_loss": 1.729378581047058, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16597506403923035, + "rewards/margins": 0.022957956418395042, + "rewards/rejected": -0.18893301486968994, + "step": 259 + }, + { + "epoch": 0.7179841215050051, + "grad_norm": 0.2108003944158554, + "learning_rate": 4.716498265932501e-06, + "log_odds_chosen": 0.5200653076171875, + "log_odds_ratio": -0.47012218832969666, + "logits/chosen": -0.01136242039501667, + "logits/rejected": -1.1188238859176636, + "logps/chosen": -1.4902641773223877, + "logps/rejected": -1.915147066116333, + "loss": 1.6221, + "nll_loss": 1.575091004371643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14902642369270325, + "rewards/margins": 0.04248826950788498, + "rewards/rejected": -0.19151470065116882, + "step": 260 + }, + { + "epoch": 0.720745598895409, + "grad_norm": 0.22381585836410522, + "learning_rate": 4.712779977963559e-06, + "log_odds_chosen": 0.39622795581817627, + "log_odds_ratio": -0.5201148390769958, + "logits/chosen": 0.10159610211849213, + "logits/rejected": -0.859241247177124, + "logps/chosen": -1.5900012254714966, + "logps/rejected": -1.917615294456482, + "loss": 1.7383, + "nll_loss": 1.6862510442733765, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15900012850761414, + "rewards/margins": 0.03276140242815018, + "rewards/rejected": -0.1917615383863449, + "step": 261 + }, + { + "epoch": 0.7235070762858129, + "grad_norm": 0.2046324461698532, + "learning_rate": 4.7090389502566884e-06, + "log_odds_chosen": 0.5120800733566284, + "log_odds_ratio": -0.47404342889785767, + "logits/chosen": 0.017112823203206062, + "logits/rejected": -1.1166574954986572, + "logps/chosen": -1.6049623489379883, + "logps/rejected": -2.0321714878082275, + "loss": 1.7168, + "nll_loss": 1.6693758964538574, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16049623489379883, + "rewards/margins": 0.042720913887023926, + "rewards/rejected": -0.20321716368198395, + "step": 262 + }, + { + "epoch": 0.7262685536762168, + "grad_norm": 0.20920297503471375, + "learning_rate": 4.705275221256738e-06, + "log_odds_chosen": 0.42567548155784607, + "log_odds_ratio": -0.5072777271270752, + "logits/chosen": 0.11223579943180084, + "logits/rejected": -1.1794720888137817, + "logps/chosen": -1.6436134576797485, + "logps/rejected": -2.0019237995147705, + "loss": 1.7837, + "nll_loss": 1.7329978942871094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16436134278774261, + "rewards/margins": 0.035831037908792496, + "rewards/rejected": -0.2001923769712448, + "step": 263 + }, + { + "epoch": 0.7290300310666207, + "grad_norm": 0.215216726064682, + "learning_rate": 4.701488829641845e-06, + "log_odds_chosen": 0.3972318172454834, + "log_odds_ratio": -0.5163053870201111, + "logits/chosen": 0.07385722547769547, + "logits/rejected": -1.1585350036621094, + "logps/chosen": -1.5981651544570923, + "logps/rejected": -1.9258975982666016, + "loss": 1.7361, + "nll_loss": 1.6844836473464966, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15981650352478027, + "rewards/margins": 0.032773248851299286, + "rewards/rejected": -0.19258975982666016, + "step": 264 + }, + { + "epoch": 0.7317915084570245, + "grad_norm": 0.20215153694152832, + "learning_rate": 4.697679814323044e-06, + "log_odds_chosen": 0.373761922121048, + "log_odds_ratio": -0.5314816832542419, + "logits/chosen": 0.0431194081902504, + "logits/rejected": -1.3062759637832642, + "logps/chosen": -1.6237438917160034, + "logps/rejected": -1.9311178922653198, + "loss": 1.746, + "nll_loss": 1.6928824186325073, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1623743772506714, + "rewards/margins": 0.030737407505512238, + "rewards/rejected": -0.19311177730560303, + "step": 265 + }, + { + "epoch": 0.7345529858474283, + "grad_norm": 0.2043098360300064, + "learning_rate": 4.693848214443858e-06, + "log_odds_chosen": 0.3958456516265869, + "log_odds_ratio": -0.5203114151954651, + "logits/chosen": 0.016560683026909828, + "logits/rejected": -1.4892038106918335, + "logps/chosen": -1.690280795097351, + "logps/rejected": -2.024770736694336, + "loss": 1.8103, + "nll_loss": 1.7582213878631592, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16902808845043182, + "rewards/margins": 0.03344898298382759, + "rewards/rejected": -0.2024770826101303, + "step": 266 + }, + { + "epoch": 0.7373144632378322, + "grad_norm": 0.2200057953596115, + "learning_rate": 4.689994069379905e-06, + "log_odds_chosen": 0.6603919863700867, + "log_odds_ratio": -0.4194304049015045, + "logits/chosen": 0.0796060711145401, + "logits/rejected": -1.6784858703613281, + "logps/chosen": -1.5854812860488892, + "logps/rejected": -2.141174554824829, + "loss": 1.7061, + "nll_loss": 1.6641736030578613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15854813158512115, + "rewards/margins": 0.05556933209300041, + "rewards/rejected": -0.21411746740341187, + "step": 267 + }, + { + "epoch": 0.7400759406282361, + "grad_norm": 0.207722008228302, + "learning_rate": 4.686117418738489e-06, + "log_odds_chosen": 0.4980109930038452, + "log_odds_ratio": -0.477446049451828, + "logits/chosen": 0.0639527440071106, + "logits/rejected": -1.2904075384140015, + "logps/chosen": -1.632141351699829, + "logps/rejected": -2.049818515777588, + "loss": 1.7618, + "nll_loss": 1.7140535116195679, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16321413218975067, + "rewards/margins": 0.041767701506614685, + "rewards/rejected": -0.20498183369636536, + "step": 268 + }, + { + "epoch": 0.74283741801864, + "grad_norm": 0.20242716372013092, + "learning_rate": 4.6822183023581945e-06, + "log_odds_chosen": 0.42001479864120483, + "log_odds_ratio": -0.517326831817627, + "logits/chosen": 0.07094614952802658, + "logits/rejected": -1.5565531253814697, + "logps/chosen": -1.6155306100845337, + "logps/rejected": -1.9663753509521484, + "loss": 1.7551, + "nll_loss": 1.7033692598342896, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1615530550479889, + "rewards/margins": 0.035084471106529236, + "rewards/rejected": -0.19663754105567932, + "step": 269 + }, + { + "epoch": 0.7455988954090439, + "grad_norm": 0.19576282799243927, + "learning_rate": 4.678296760308474e-06, + "log_odds_chosen": 0.34514501690864563, + "log_odds_ratio": -0.5403321385383606, + "logits/chosen": -0.05599237233400345, + "logits/rejected": -1.4408526420593262, + "logps/chosen": -1.5795042514801025, + "logps/rejected": -1.8630210161209106, + "loss": 1.7087, + "nll_loss": 1.6546752452850342, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15795043110847473, + "rewards/margins": 0.028351658955216408, + "rewards/rejected": -0.1863020956516266, + "step": 270 + }, + { + "epoch": 0.7483603727994477, + "grad_norm": 0.22211046516895294, + "learning_rate": 4.674352832889239e-06, + "log_odds_chosen": 0.5620113611221313, + "log_odds_ratio": -0.4555080533027649, + "logits/chosen": -0.005663935095071793, + "logits/rejected": -1.2683230638504028, + "logps/chosen": -1.6520458459854126, + "logps/rejected": -2.1268177032470703, + "loss": 1.7996, + "nll_loss": 1.7540650367736816, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16520458459854126, + "rewards/margins": 0.04747716709971428, + "rewards/rejected": -0.21268175542354584, + "step": 271 + }, + { + "epoch": 0.7511218501898516, + "grad_norm": 0.1922963708639145, + "learning_rate": 4.670386560630446e-06, + "log_odds_chosen": 0.444943368434906, + "log_odds_ratio": -0.5015792846679688, + "logits/chosen": -0.06714704632759094, + "logits/rejected": -1.379449725151062, + "logps/chosen": -1.5659842491149902, + "logps/rejected": -1.9318914413452148, + "loss": 1.6989, + "nll_loss": 1.6487019062042236, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15659843385219574, + "rewards/margins": 0.036590706557035446, + "rewards/rejected": -0.19318914413452148, + "step": 272 + }, + { + "epoch": 0.7538833275802554, + "grad_norm": 0.19772395491600037, + "learning_rate": 4.66639798429168e-06, + "log_odds_chosen": 0.5909014940261841, + "log_odds_ratio": -0.4440915584564209, + "logits/chosen": -0.010252359323203564, + "logits/rejected": -1.6117687225341797, + "logps/chosen": -1.6227033138275146, + "logps/rejected": -2.1234261989593506, + "loss": 1.7368, + "nll_loss": 1.69236421585083, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16227032244205475, + "rewards/margins": 0.05007229745388031, + "rewards/rejected": -0.21234261989593506, + "step": 273 + }, + { + "epoch": 0.7566448049706593, + "grad_norm": 0.21296410262584686, + "learning_rate": 4.6623871448617345e-06, + "log_odds_chosen": 0.37625253200531006, + "log_odds_ratio": -0.5279226303100586, + "logits/chosen": -0.11291900277137756, + "logits/rejected": -1.474963665008545, + "logps/chosen": -1.5688221454620361, + "logps/rejected": -1.8770910501480103, + "loss": 1.6973, + "nll_loss": 1.6444581747055054, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15688221156597137, + "rewards/margins": 0.03082689456641674, + "rewards/rejected": -0.18770912289619446, + "step": 274 + }, + { + "epoch": 0.7594062823610632, + "grad_norm": 0.21504846215248108, + "learning_rate": 4.6583540835581885e-06, + "log_odds_chosen": 0.446832537651062, + "log_odds_ratio": -0.5026010870933533, + "logits/chosen": -0.0652085542678833, + "logits/rejected": -1.2765593528747559, + "logps/chosen": -1.5856618881225586, + "logps/rejected": -1.9579052925109863, + "loss": 1.7239, + "nll_loss": 1.6736685037612915, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1585661768913269, + "rewards/margins": 0.037224359810352325, + "rewards/rejected": -0.19579055905342102, + "step": 275 + }, + { + "epoch": 0.762167759751467, + "grad_norm": 0.1951994001865387, + "learning_rate": 4.654298841826988e-06, + "log_odds_chosen": 0.38752269744873047, + "log_odds_ratio": -0.5222585797309875, + "logits/chosen": -0.09070023894309998, + "logits/rejected": -1.2031896114349365, + "logps/chosen": -1.5128428936004639, + "logps/rejected": -1.8278968334197998, + "loss": 1.6382, + "nll_loss": 1.586016058921814, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15128430724143982, + "rewards/margins": 0.031505391001701355, + "rewards/rejected": -0.18278968334197998, + "step": 276 + }, + { + "epoch": 0.7649292371418709, + "grad_norm": 0.20356132090091705, + "learning_rate": 4.6502214613420164e-06, + "log_odds_chosen": 0.661637008190155, + "log_odds_ratio": -0.4188510775566101, + "logits/chosen": -0.07356397807598114, + "logits/rejected": -1.270960807800293, + "logps/chosen": -1.4853070974349976, + "logps/rejected": -2.030834674835205, + "loss": 1.6179, + "nll_loss": 1.5760544538497925, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14853070676326752, + "rewards/margins": 0.054552774876356125, + "rewards/rejected": -0.20308347046375275, + "step": 277 + }, + { + "epoch": 0.7676907145322748, + "grad_norm": 0.21808800101280212, + "learning_rate": 4.646121984004666e-06, + "log_odds_chosen": 0.535017192363739, + "log_odds_ratio": -0.4694536030292511, + "logits/chosen": -0.058985427021980286, + "logits/rejected": -1.204667329788208, + "logps/chosen": -1.6129412651062012, + "logps/rejected": -2.0625391006469727, + "loss": 1.7558, + "nll_loss": 1.7088611125946045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1612941473722458, + "rewards/margins": 0.04495978727936745, + "rewards/rejected": -0.20625391602516174, + "step": 278 + }, + { + "epoch": 0.7704521919226787, + "grad_norm": 0.21462294459342957, + "learning_rate": 4.642000451943409e-06, + "log_odds_chosen": 0.4302963316440582, + "log_odds_ratio": -0.5060604214668274, + "logits/chosen": 0.040374599397182465, + "logits/rejected": -1.187546968460083, + "logps/chosen": -1.6709057092666626, + "logps/rejected": -2.035818576812744, + "loss": 1.7989, + "nll_loss": 1.7482545375823975, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16709057986736298, + "rewards/margins": 0.03649128973484039, + "rewards/rejected": -0.20358186960220337, + "step": 279 + }, + { + "epoch": 0.7732136693130826, + "grad_norm": 0.19471004605293274, + "learning_rate": 4.637856907513366e-06, + "log_odds_chosen": 0.5729100704193115, + "log_odds_ratio": -0.45996299386024475, + "logits/chosen": -0.0875316932797432, + "logits/rejected": -1.2949634790420532, + "logps/chosen": -1.6255837678909302, + "logps/rejected": -2.107527256011963, + "loss": 1.7409, + "nll_loss": 1.6949416399002075, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1625583916902542, + "rewards/margins": 0.04819435626268387, + "rewards/rejected": -0.21075274050235748, + "step": 280 + }, + { + "epoch": 0.7759751467034863, + "grad_norm": 0.20401322841644287, + "learning_rate": 4.633691393295865e-06, + "log_odds_chosen": 0.3522525429725647, + "log_odds_ratio": -0.5362752676010132, + "logits/chosen": -0.07532086223363876, + "logits/rejected": -1.3928956985473633, + "logps/chosen": -1.6393187046051025, + "logps/rejected": -1.9348175525665283, + "loss": 1.7628, + "nll_loss": 1.7092012166976929, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16393187642097473, + "rewards/margins": 0.02954990416765213, + "rewards/rejected": -0.19348177313804626, + "step": 281 + }, + { + "epoch": 0.7787366240938902, + "grad_norm": 0.19418881833553314, + "learning_rate": 4.629503952098011e-06, + "log_odds_chosen": 0.6238572597503662, + "log_odds_ratio": -0.43670332431793213, + "logits/chosen": -0.08066678047180176, + "logits/rejected": -1.6776320934295654, + "logps/chosen": -1.5727202892303467, + "logps/rejected": -2.097163200378418, + "loss": 1.6952, + "nll_loss": 1.6514896154403687, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15727202594280243, + "rewards/margins": 0.05244428664445877, + "rewards/rejected": -0.2097163200378418, + "step": 282 + }, + { + "epoch": 0.7814981014842941, + "grad_norm": 0.19461016356945038, + "learning_rate": 4.6252946269522406e-06, + "log_odds_chosen": 0.41456982493400574, + "log_odds_ratio": -0.5249388813972473, + "logits/chosen": -0.09783484041690826, + "logits/rejected": -1.5897575616836548, + "logps/chosen": -1.5939539670944214, + "logps/rejected": -1.9444385766983032, + "loss": 1.7211, + "nll_loss": 1.6686402559280396, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15939539670944214, + "rewards/margins": 0.0350484773516655, + "rewards/rejected": -0.19444386661052704, + "step": 283 + }, + { + "epoch": 0.784259578874698, + "grad_norm": 0.2026386708021164, + "learning_rate": 4.621063461115882e-06, + "log_odds_chosen": 0.42722252011299133, + "log_odds_ratio": -0.5157784223556519, + "logits/chosen": -0.04370000213384628, + "logits/rejected": -1.5634028911590576, + "logps/chosen": -1.6642380952835083, + "logps/rejected": -2.0263044834136963, + "loss": 1.7849, + "nll_loss": 1.733304738998413, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16642381250858307, + "rewards/margins": 0.03620663285255432, + "rewards/rejected": -0.20263046026229858, + "step": 284 + }, + { + "epoch": 0.7870210562651019, + "grad_norm": 0.20439012348651886, + "learning_rate": 4.6168104980707105e-06, + "log_odds_chosen": 0.4680078625679016, + "log_odds_ratio": -0.4906473755836487, + "logits/chosen": -0.09853056073188782, + "logits/rejected": -1.5130950212478638, + "logps/chosen": -1.5937169790267944, + "logps/rejected": -1.9836535453796387, + "loss": 1.7178, + "nll_loss": 1.668696641921997, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15937167406082153, + "rewards/margins": 0.038993678987026215, + "rewards/rejected": -0.19836536049842834, + "step": 285 + }, + { + "epoch": 0.7897825336555057, + "grad_norm": 0.1877906173467636, + "learning_rate": 4.612535781522504e-06, + "log_odds_chosen": 0.37254124879837036, + "log_odds_ratio": -0.525785505771637, + "logits/chosen": -0.08526084572076797, + "logits/rejected": -1.4526267051696777, + "logps/chosen": -1.5574032068252563, + "logps/rejected": -1.863482117652893, + "loss": 1.6777, + "nll_loss": 1.6251548528671265, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15574032068252563, + "rewards/margins": 0.03060789778828621, + "rewards/rejected": -0.18634822964668274, + "step": 286 + }, + { + "epoch": 0.7925440110459095, + "grad_norm": 0.19580195844173431, + "learning_rate": 4.6082393554005855e-06, + "log_odds_chosen": 0.5562997460365295, + "log_odds_ratio": -0.4619132876396179, + "logits/chosen": 0.01294594258069992, + "logits/rejected": -1.4304238557815552, + "logps/chosen": -1.5266389846801758, + "logps/rejected": -1.9919108152389526, + "loss": 1.6616, + "nll_loss": 1.6153795719146729, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.152663916349411, + "rewards/margins": 0.046527184545993805, + "rewards/rejected": -0.19919107854366302, + "step": 287 + }, + { + "epoch": 0.7953054884363134, + "grad_norm": 0.1980113834142685, + "learning_rate": 4.6039212638573835e-06, + "log_odds_chosen": 0.3346819281578064, + "log_odds_ratio": -0.5443198680877686, + "logits/chosen": -0.15275517106056213, + "logits/rejected": -1.6041910648345947, + "logps/chosen": -1.6962547302246094, + "logps/rejected": -1.9770543575286865, + "loss": 1.8067, + "nll_loss": 1.752286672592163, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16962547600269318, + "rewards/margins": 0.028079960495233536, + "rewards/rejected": -0.1977054327726364, + "step": 288 + }, + { + "epoch": 0.7980669658267173, + "grad_norm": 0.2273865044116974, + "learning_rate": 4.599581551267969e-06, + "log_odds_chosen": 0.5466289520263672, + "log_odds_ratio": -0.46321243047714233, + "logits/chosen": 0.03993244469165802, + "logits/rejected": -1.286005973815918, + "logps/chosen": -1.6738882064819336, + "logps/rejected": -2.1377532482147217, + "loss": 1.8, + "nll_loss": 1.7537211179733276, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16738884150981903, + "rewards/margins": 0.046386465430259705, + "rewards/rejected": -0.21377530694007874, + "step": 289 + }, + { + "epoch": 0.8008284432171212, + "grad_norm": 0.20244112610816956, + "learning_rate": 4.5952202622296015e-06, + "log_odds_chosen": 0.3539222180843353, + "log_odds_ratio": -0.53514164686203, + "logits/chosen": -0.11812002211809158, + "logits/rejected": -1.4588350057601929, + "logps/chosen": -1.5985196828842163, + "logps/rejected": -1.8924399614334106, + "loss": 1.7271, + "nll_loss": 1.6736091375350952, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15985198318958282, + "rewards/margins": 0.029392031952738762, + "rewards/rejected": -0.18924400210380554, + "step": 290 + }, + { + "epoch": 0.803589920607525, + "grad_norm": 0.20306488871574402, + "learning_rate": 4.590837441561277e-06, + "log_odds_chosen": 0.4768354296684265, + "log_odds_ratio": -0.49164149165153503, + "logits/chosen": -0.09056994318962097, + "logits/rejected": -1.4742063283920288, + "logps/chosen": -1.6228840351104736, + "logps/rejected": -2.022940158843994, + "loss": 1.7341, + "nll_loss": 1.6849125623703003, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1622883826494217, + "rewards/margins": 0.040005628019571304, + "rewards/rejected": -0.2022940218448639, + "step": 291 + }, + { + "epoch": 0.8063513979979289, + "grad_norm": 0.19205592572689056, + "learning_rate": 4.586433134303257e-06, + "log_odds_chosen": 0.5734292268753052, + "log_odds_ratio": -0.44972163438796997, + "logits/chosen": -0.07964983582496643, + "logits/rejected": -1.3166429996490479, + "logps/chosen": -1.4941394329071045, + "logps/rejected": -1.9636625051498413, + "loss": 1.6364, + "nll_loss": 1.5913902521133423, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14941394329071045, + "rewards/margins": 0.04695230349898338, + "rewards/rejected": -0.19636625051498413, + "step": 292 + }, + { + "epoch": 0.8091128753883328, + "grad_norm": 0.20325274765491486, + "learning_rate": 4.582007385716614e-06, + "log_odds_chosen": 0.4024369418621063, + "log_odds_ratio": -0.5175961852073669, + "logits/chosen": -0.07684268802404404, + "logits/rejected": -1.502617597579956, + "logps/chosen": -1.6142741441726685, + "logps/rejected": -1.9492229223251343, + "loss": 1.7574, + "nll_loss": 1.7056207656860352, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16142742335796356, + "rewards/margins": 0.033494893461465836, + "rewards/rejected": -0.1949223130941391, + "step": 293 + }, + { + "epoch": 0.8118743527787367, + "grad_norm": 0.20083336532115936, + "learning_rate": 4.57756024128276e-06, + "log_odds_chosen": 0.5593162178993225, + "log_odds_ratio": -0.45524081587791443, + "logits/chosen": -0.17213015258312225, + "logits/rejected": -1.3977055549621582, + "logps/chosen": -1.5258231163024902, + "logps/rejected": -1.9875348806381226, + "loss": 1.6559, + "nll_loss": 1.610384225845337, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1525823026895523, + "rewards/margins": 0.04617120325565338, + "rewards/rejected": -0.1987534910440445, + "step": 294 + }, + { + "epoch": 0.8146358301691405, + "grad_norm": 0.19001390039920807, + "learning_rate": 4.573091746702988e-06, + "log_odds_chosen": 0.5878125429153442, + "log_odds_ratio": -0.44817692041397095, + "logits/chosen": -0.16976626217365265, + "logits/rejected": -1.3345102071762085, + "logps/chosen": -1.5175349712371826, + "logps/rejected": -2.003750801086426, + "loss": 1.6573, + "nll_loss": 1.612461805343628, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1517535001039505, + "rewards/margins": 0.048621561378240585, + "rewards/rejected": -0.20037508010864258, + "step": 295 + }, + { + "epoch": 0.8173973075595443, + "grad_norm": 0.18019473552703857, + "learning_rate": 4.5686019478979915e-06, + "log_odds_chosen": 0.6261818408966064, + "log_odds_ratio": -0.43093276023864746, + "logits/chosen": -0.045413050800561905, + "logits/rejected": -1.8382006883621216, + "logps/chosen": -1.5509614944458008, + "logps/rejected": -2.0732412338256836, + "loss": 1.6609, + "nll_loss": 1.617802619934082, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1550961434841156, + "rewards/margins": 0.05222797393798828, + "rewards/rejected": -0.20732411742210388, + "step": 296 + }, + { + "epoch": 0.8201587849499482, + "grad_norm": 0.20908498764038086, + "learning_rate": 4.564090891007401e-06, + "log_odds_chosen": 0.5261347889900208, + "log_odds_ratio": -0.4701959192752838, + "logits/chosen": -0.1871597170829773, + "logits/rejected": -1.5989326238632202, + "logps/chosen": -1.648465633392334, + "logps/rejected": -2.0937981605529785, + "loss": 1.7682, + "nll_loss": 1.7211995124816895, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16484656929969788, + "rewards/margins": 0.04453325271606445, + "rewards/rejected": -0.20937982201576233, + "step": 297 + }, + { + "epoch": 0.8229202623403521, + "grad_norm": 0.188289076089859, + "learning_rate": 4.559558622389304e-06, + "log_odds_chosen": 0.5777133703231812, + "log_odds_ratio": -0.4563387632369995, + "logits/chosen": -0.13260145485401154, + "logits/rejected": -1.7002828121185303, + "logps/chosen": -1.5394691228866577, + "logps/rejected": -2.0138742923736572, + "loss": 1.6561, + "nll_loss": 1.6105040311813354, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1539469212293625, + "rewards/margins": 0.047440510243177414, + "rewards/rejected": -0.201387420296669, + "step": 298 + }, + { + "epoch": 0.825681739730756, + "grad_norm": 0.19627498090267181, + "learning_rate": 4.555005188619776e-06, + "log_odds_chosen": 0.5525295734405518, + "log_odds_ratio": -0.4643310308456421, + "logits/chosen": -0.20222769677639008, + "logits/rejected": -1.6397647857666016, + "logps/chosen": -1.5733509063720703, + "logps/rejected": -2.0376038551330566, + "loss": 1.6892, + "nll_loss": 1.642791748046875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1573350876569748, + "rewards/margins": 0.04642530530691147, + "rewards/rejected": -0.20376040041446686, + "step": 299 + }, + { + "epoch": 0.8284432171211599, + "grad_norm": 0.2084610015153885, + "learning_rate": 4.55043063649239e-06, + "log_odds_chosen": 0.6468226313591003, + "log_odds_ratio": -0.4257048964500427, + "logits/chosen": -0.23111680150032043, + "logits/rejected": -1.959672212600708, + "logps/chosen": -1.5862557888031006, + "logps/rejected": -2.1299490928649902, + "loss": 1.7007, + "nll_loss": 1.6581330299377441, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15862558782100677, + "rewards/margins": 0.054369326680898666, + "rewards/rejected": -0.21299490332603455, + "step": 300 + }, + { + "epoch": 0.8312046945115636, + "grad_norm": 0.18870840966701508, + "learning_rate": 4.54583501301775e-06, + "log_odds_chosen": 0.5869650840759277, + "log_odds_ratio": -0.45076417922973633, + "logits/chosen": -0.1821564882993698, + "logits/rejected": -1.9625955820083618, + "logps/chosen": -1.6092771291732788, + "logps/rejected": -2.104759693145752, + "loss": 1.7134, + "nll_loss": 1.668313980102539, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16092771291732788, + "rewards/margins": 0.04954826086759567, + "rewards/rejected": -0.21047596633434296, + "step": 301 + }, + { + "epoch": 0.8339661719019675, + "grad_norm": 0.1871059238910675, + "learning_rate": 4.541218365422997e-06, + "log_odds_chosen": 0.6819782257080078, + "log_odds_ratio": -0.4187195301055908, + "logits/chosen": -0.2113940417766571, + "logits/rejected": -1.5733145475387573, + "logps/chosen": -1.4704827070236206, + "logps/rejected": -2.02266001701355, + "loss": 1.5975, + "nll_loss": 1.5556209087371826, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14704826474189758, + "rewards/margins": 0.05521773174405098, + "rewards/rejected": -0.20226599276065826, + "step": 302 + }, + { + "epoch": 0.8367276492923714, + "grad_norm": 0.1986740678548813, + "learning_rate": 4.536580741151328e-06, + "log_odds_chosen": 0.427354633808136, + "log_odds_ratio": -0.5044746398925781, + "logits/chosen": -0.17407816648483276, + "logits/rejected": -1.2958568334579468, + "logps/chosen": -1.6126006841659546, + "logps/rejected": -1.9662362337112427, + "loss": 1.7285, + "nll_loss": 1.678093433380127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16126006841659546, + "rewards/margins": 0.03536355867981911, + "rewards/rejected": -0.19662362337112427, + "step": 303 + }, + { + "epoch": 0.8394891266827753, + "grad_norm": 0.20910044014453888, + "learning_rate": 4.531922187861507e-06, + "log_odds_chosen": 0.6855639219284058, + "log_odds_ratio": -0.41133368015289307, + "logits/chosen": -0.17175668478012085, + "logits/rejected": -1.4931915998458862, + "logps/chosen": -1.491321086883545, + "logps/rejected": -2.059537649154663, + "loss": 1.631, + "nll_loss": 1.589834451675415, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1491321176290512, + "rewards/margins": 0.056821659207344055, + "rewards/rejected": -0.20595377683639526, + "step": 304 + }, + { + "epoch": 0.8422506040731792, + "grad_norm": 0.2135782539844513, + "learning_rate": 4.527242753427378e-06, + "log_odds_chosen": 0.4499048590660095, + "log_odds_ratio": -0.4998108744621277, + "logits/chosen": -0.23703832924365997, + "logits/rejected": -1.2717434167861938, + "logps/chosen": -1.6932170391082764, + "logps/rejected": -2.074625015258789, + "loss": 1.8238, + "nll_loss": 1.7738076448440552, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1693217009305954, + "rewards/margins": 0.038140811026096344, + "rewards/rejected": -0.20746250450611115, + "step": 305 + }, + { + "epoch": 0.845012081463583, + "grad_norm": 0.1993100643157959, + "learning_rate": 4.522542485937369e-06, + "log_odds_chosen": 0.6878387331962585, + "log_odds_ratio": -0.4140303134918213, + "logits/chosen": -0.17333604395389557, + "logits/rejected": -1.6610162258148193, + "logps/chosen": -1.5485466718673706, + "logps/rejected": -2.122166395187378, + "loss": 1.6836, + "nll_loss": 1.6422399282455444, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1548546552658081, + "rewards/margins": 0.057361967861652374, + "rewards/rejected": -0.21221664547920227, + "step": 306 + }, + { + "epoch": 0.8477735588539869, + "grad_norm": 0.20089736580848694, + "learning_rate": 4.5178214336940015e-06, + "log_odds_chosen": 0.5427862405776978, + "log_odds_ratio": -0.4643491506576538, + "logits/chosen": -0.1863488256931305, + "logits/rejected": -1.4939963817596436, + "logps/chosen": -1.556074857711792, + "logps/rejected": -2.007810592651367, + "loss": 1.6867, + "nll_loss": 1.640239953994751, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15560749173164368, + "rewards/margins": 0.04517355561256409, + "rewards/rejected": -0.20078104734420776, + "step": 307 + }, + { + "epoch": 0.8505350362443908, + "grad_norm": 0.19465354084968567, + "learning_rate": 4.513079645213391e-06, + "log_odds_chosen": 0.6006003618240356, + "log_odds_ratio": -0.4448007047176361, + "logits/chosen": -0.21229855716228485, + "logits/rejected": -1.460700273513794, + "logps/chosen": -1.4619970321655273, + "logps/rejected": -1.951730728149414, + "loss": 1.5808, + "nll_loss": 1.5363428592681885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14619971811771393, + "rewards/margins": 0.04897337406873703, + "rewards/rejected": -0.19517306983470917, + "step": 308 + }, + { + "epoch": 0.8532965136347946, + "grad_norm": 0.22921526432037354, + "learning_rate": 4.508317169224752e-06, + "log_odds_chosen": 0.28590530157089233, + "log_odds_ratio": -0.5672011375427246, + "logits/chosen": -0.3028235137462616, + "logits/rejected": -1.3352372646331787, + "logps/chosen": -1.5896085500717163, + "logps/rejected": -1.8217551708221436, + "loss": 1.7142, + "nll_loss": 1.6574809551239014, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15896086394786835, + "rewards/margins": 0.0232146717607975, + "rewards/rejected": -0.18217553198337555, + "step": 309 + }, + { + "epoch": 0.8560579910251985, + "grad_norm": 0.20834492146968842, + "learning_rate": 4.5035340546698915e-06, + "log_odds_chosen": 0.5302640795707703, + "log_odds_ratio": -0.47237080335617065, + "logits/chosen": -0.13226553797721863, + "logits/rejected": -1.6689988374710083, + "logps/chosen": -1.5815542936325073, + "logps/rejected": -2.017597198486328, + "loss": 1.7019, + "nll_loss": 1.6546366214752197, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1581554412841797, + "rewards/margins": 0.0436042957007885, + "rewards/rejected": -0.2017597258090973, + "step": 310 + }, + { + "epoch": 0.8588194684156023, + "grad_norm": 0.2186000496149063, + "learning_rate": 4.4987303507027155e-06, + "log_odds_chosen": 0.3287478983402252, + "log_odds_ratio": -0.549705982208252, + "logits/chosen": -0.11287423223257065, + "logits/rejected": -1.349088430404663, + "logps/chosen": -1.6460450887680054, + "logps/rejected": -1.9183804988861084, + "loss": 1.7738, + "nll_loss": 1.718807339668274, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16460449993610382, + "rewards/margins": 0.027233552187681198, + "rewards/rejected": -0.19183805584907532, + "step": 311 + }, + { + "epoch": 0.8615809458060062, + "grad_norm": 0.19687776267528534, + "learning_rate": 4.493906106688712e-06, + "log_odds_chosen": 0.6679433584213257, + "log_odds_ratio": -0.41612738370895386, + "logits/chosen": -0.1544291228055954, + "logits/rejected": -1.6344720125198364, + "logps/chosen": -1.5363831520080566, + "logps/rejected": -2.0934810638427734, + "loss": 1.6699, + "nll_loss": 1.6282765865325928, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1536383181810379, + "rewards/margins": 0.05570977181196213, + "rewards/rejected": -0.20934809744358063, + "step": 312 + }, + { + "epoch": 0.8643424231964101, + "grad_norm": 0.18424390256404877, + "learning_rate": 4.4890613722044526e-06, + "log_odds_chosen": 0.5501468181610107, + "log_odds_ratio": -0.4591679573059082, + "logits/chosen": -0.21150004863739014, + "logits/rejected": -1.5727708339691162, + "logps/chosen": -1.529168963432312, + "logps/rejected": -1.9859933853149414, + "loss": 1.6331, + "nll_loss": 1.587223768234253, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15291690826416016, + "rewards/margins": 0.04568243771791458, + "rewards/rejected": -0.19859933853149414, + "step": 313 + }, + { + "epoch": 0.867103900586814, + "grad_norm": 0.20299668610095978, + "learning_rate": 4.484196197037082e-06, + "log_odds_chosen": 0.5865733027458191, + "log_odds_ratio": -0.4465975761413574, + "logits/chosen": -0.13622766733169556, + "logits/rejected": -1.5913082361221313, + "logps/chosen": -1.5737249851226807, + "logps/rejected": -2.0672624111175537, + "loss": 1.691, + "nll_loss": 1.6463013887405396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15737250447273254, + "rewards/margins": 0.04935373738408089, + "rewards/rejected": -0.20672622323036194, + "step": 314 + }, + { + "epoch": 0.8698653779772179, + "grad_norm": 0.19080907106399536, + "learning_rate": 4.4793106311838e-06, + "log_odds_chosen": 0.556647002696991, + "log_odds_ratio": -0.46323782205581665, + "logits/chosen": -0.2553403377532959, + "logits/rejected": -1.4489879608154297, + "logps/chosen": -1.562751054763794, + "logps/rejected": -2.0281355381011963, + "loss": 1.6685, + "nll_loss": 1.6221669912338257, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15627512335777283, + "rewards/margins": 0.04653845354914665, + "rewards/rejected": -0.20281356573104858, + "step": 315 + }, + { + "epoch": 0.8726268553676216, + "grad_norm": 0.21013295650482178, + "learning_rate": 4.474404724851356e-06, + "log_odds_chosen": 0.5066735148429871, + "log_odds_ratio": -0.48284393548965454, + "logits/chosen": -0.18822398781776428, + "logits/rejected": -1.6378931999206543, + "logps/chosen": -1.6176685094833374, + "logps/rejected": -2.041299819946289, + "loss": 1.7356, + "nll_loss": 1.687273621559143, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16176684200763702, + "rewards/margins": 0.04236314073204994, + "rewards/rejected": -0.20412999391555786, + "step": 316 + }, + { + "epoch": 0.8753883327580255, + "grad_norm": 0.2111610472202301, + "learning_rate": 4.469478528455529e-06, + "log_odds_chosen": 0.4905741810798645, + "log_odds_ratio": -0.4830451011657715, + "logits/chosen": -0.3353807330131531, + "logits/rejected": -1.3652547597885132, + "logps/chosen": -1.6042860746383667, + "logps/rejected": -2.011214017868042, + "loss": 1.7255, + "nll_loss": 1.677234411239624, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16042861342430115, + "rewards/margins": 0.04069279134273529, + "rewards/rejected": -0.20112140476703644, + "step": 317 + }, + { + "epoch": 0.8781498101484294, + "grad_norm": 0.20293498039245605, + "learning_rate": 4.464532092620607e-06, + "log_odds_chosen": 0.523048996925354, + "log_odds_ratio": -0.47193020582199097, + "logits/chosen": -0.18486103415489197, + "logits/rejected": -1.4264463186264038, + "logps/chosen": -1.638154149055481, + "logps/rejected": -2.0816562175750732, + "loss": 1.7478, + "nll_loss": 1.7006094455718994, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1638154238462448, + "rewards/margins": 0.04435021057724953, + "rewards/rejected": -0.20816563069820404, + "step": 318 + }, + { + "epoch": 0.8809112875388333, + "grad_norm": 0.17995508015155792, + "learning_rate": 4.4595654681788715e-06, + "log_odds_chosen": 0.647110641002655, + "log_odds_ratio": -0.428227037191391, + "logits/chosen": -0.23671314120292664, + "logits/rejected": -1.8144798278808594, + "logps/chosen": -1.5134212970733643, + "logps/rejected": -2.050638437271118, + "loss": 1.6212, + "nll_loss": 1.5783425569534302, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15134215354919434, + "rewards/margins": 0.053721703588962555, + "rewards/rejected": -0.2050638496875763, + "step": 319 + }, + { + "epoch": 0.8836727649292372, + "grad_norm": 0.20032061636447906, + "learning_rate": 4.454578706170075e-06, + "log_odds_chosen": 0.35773491859436035, + "log_odds_ratio": -0.5383328795433044, + "logits/chosen": -0.1582796573638916, + "logits/rejected": -1.6286273002624512, + "logps/chosen": -1.6196863651275635, + "logps/rejected": -1.9159085750579834, + "loss": 1.7371, + "nll_loss": 1.6832914352416992, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1619686335325241, + "rewards/margins": 0.02962222881615162, + "rewards/rejected": -0.19159086048603058, + "step": 320 + }, + { + "epoch": 0.886434242319641, + "grad_norm": 0.2055967003107071, + "learning_rate": 4.449571857840911e-06, + "log_odds_chosen": 0.586254358291626, + "log_odds_ratio": -0.45622390508651733, + "logits/chosen": -0.09812657535076141, + "logits/rejected": -1.682092547416687, + "logps/chosen": -1.5706590414047241, + "logps/rejected": -2.0691263675689697, + "loss": 1.6908, + "nll_loss": 1.6451488733291626, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15706591308116913, + "rewards/margins": 0.04984673112630844, + "rewards/rejected": -0.20691262185573578, + "step": 321 + }, + { + "epoch": 0.8891957197100449, + "grad_norm": 0.2170560508966446, + "learning_rate": 4.444544974644493e-06, + "log_odds_chosen": 0.42108941078186035, + "log_odds_ratio": -0.5122984647750854, + "logits/chosen": -0.20832902193069458, + "logits/rejected": -1.468292236328125, + "logps/chosen": -1.5347344875335693, + "logps/rejected": -1.8808050155639648, + "loss": 1.6619, + "nll_loss": 1.61066472530365, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15347345173358917, + "rewards/margins": 0.03460706025362015, + "rewards/rejected": -0.18808050453662872, + "step": 322 + }, + { + "epoch": 0.8919571971004487, + "grad_norm": 0.20523445308208466, + "learning_rate": 4.4394981082398254e-06, + "log_odds_chosen": 0.4536086320877075, + "log_odds_ratio": -0.5041500926017761, + "logits/chosen": -0.26630857586860657, + "logits/rejected": -1.5968291759490967, + "logps/chosen": -1.5988963842391968, + "logps/rejected": -1.9788540601730347, + "loss": 1.7272, + "nll_loss": 1.676782250404358, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15988965332508087, + "rewards/margins": 0.037995755672454834, + "rewards/rejected": -0.19788537919521332, + "step": 323 + }, + { + "epoch": 0.8947186744908526, + "grad_norm": 0.21546316146850586, + "learning_rate": 4.434431310491267e-06, + "log_odds_chosen": 0.6247555017471313, + "log_odds_ratio": -0.43717044591903687, + "logits/chosen": -0.27398669719696045, + "logits/rejected": -1.5846881866455078, + "logps/chosen": -1.660336971282959, + "logps/rejected": -2.190629243850708, + "loss": 1.7647, + "nll_loss": 1.7209898233413696, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16603372991085052, + "rewards/margins": 0.05302921682596207, + "rewards/rejected": -0.2190629243850708, + "step": 324 + }, + { + "epoch": 0.8974801518812565, + "grad_norm": 0.21981805562973022, + "learning_rate": 4.429344633468005e-06, + "log_odds_chosen": 0.5096076130867004, + "log_odds_ratio": -0.4749549627304077, + "logits/chosen": -0.22183284163475037, + "logits/rejected": -1.6317569017410278, + "logps/chosen": -1.528499722480774, + "logps/rejected": -1.9499012231826782, + "loss": 1.6449, + "nll_loss": 1.597440242767334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1528499722480774, + "rewards/margins": 0.0421401672065258, + "rewards/rejected": -0.1949901133775711, + "step": 325 + }, + { + "epoch": 0.9002416292716603, + "grad_norm": 0.23145584762096405, + "learning_rate": 4.424238129443515e-06, + "log_odds_chosen": 0.4823288321495056, + "log_odds_ratio": -0.48343366384506226, + "logits/chosen": -0.1781751811504364, + "logits/rejected": -1.2557498216629028, + "logps/chosen": -1.6358767747879028, + "logps/rejected": -2.0398380756378174, + "loss": 1.7597, + "nll_loss": 1.7113655805587769, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16358768939971924, + "rewards/margins": 0.04039612039923668, + "rewards/rejected": -0.20398379862308502, + "step": 326 + }, + { + "epoch": 0.9030031066620642, + "grad_norm": 0.21986520290374756, + "learning_rate": 4.4191118508950286e-06, + "log_odds_chosen": 0.5763283371925354, + "log_odds_ratio": -0.4508843421936035, + "logits/chosen": -0.2733724117279053, + "logits/rejected": -1.4346749782562256, + "logps/chosen": -1.5722405910491943, + "logps/rejected": -2.0537078380584717, + "loss": 1.7093, + "nll_loss": 1.6641618013381958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15722407400608063, + "rewards/margins": 0.048146720975637436, + "rewards/rejected": -0.20537079870700836, + "step": 327 + }, + { + "epoch": 0.9057645840524681, + "grad_norm": 0.19916215538978577, + "learning_rate": 4.413965850502987e-06, + "log_odds_chosen": 0.6543524265289307, + "log_odds_ratio": -0.42543232440948486, + "logits/chosen": -0.28132179379463196, + "logits/rejected": -1.6163063049316406, + "logps/chosen": -1.4477849006652832, + "logps/rejected": -1.9837119579315186, + "loss": 1.5427, + "nll_loss": 1.5002020597457886, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14477849006652832, + "rewards/margins": 0.05359271913766861, + "rewards/rejected": -0.19837118685245514, + "step": 328 + }, + { + "epoch": 0.908526061442872, + "grad_norm": 0.20786328613758087, + "learning_rate": 4.408800181150509e-06, + "log_odds_chosen": 0.7103330492973328, + "log_odds_ratio": -0.405758798122406, + "logits/chosen": -0.27468955516815186, + "logits/rejected": -1.7239093780517578, + "logps/chosen": -1.5232479572296143, + "logps/rejected": -2.113443374633789, + "loss": 1.6308, + "nll_loss": 1.5901868343353271, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15232481062412262, + "rewards/margins": 0.0590195432305336, + "rewards/rejected": -0.21134433150291443, + "step": 329 + }, + { + "epoch": 0.9112875388332758, + "grad_norm": 0.21173468232154846, + "learning_rate": 4.4036148959228365e-06, + "log_odds_chosen": 0.5421361327171326, + "log_odds_ratio": -0.45979243516921997, + "logits/chosen": -0.19346265494823456, + "logits/rejected": -1.3866770267486572, + "logps/chosen": -1.5939050912857056, + "logps/rejected": -2.0476269721984863, + "loss": 1.7068, + "nll_loss": 1.660808801651001, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15939049422740936, + "rewards/margins": 0.045372217893600464, + "rewards/rejected": -0.20476271212100983, + "step": 330 + }, + { + "epoch": 0.9140490162236796, + "grad_norm": 0.20240680873394012, + "learning_rate": 4.3984100481068e-06, + "log_odds_chosen": 0.5311475992202759, + "log_odds_ratio": -0.47231799364089966, + "logits/chosen": -0.10619790852069855, + "logits/rejected": -1.8089945316314697, + "logps/chosen": -1.528999924659729, + "logps/rejected": -1.9681410789489746, + "loss": 1.6383, + "nll_loss": 1.5910669565200806, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15289999544620514, + "rewards/margins": 0.04391412436962128, + "rewards/rejected": -0.19681411981582642, + "step": 331 + }, + { + "epoch": 0.9168104936140835, + "grad_norm": 0.18672843277454376, + "learning_rate": 4.3931856911902635e-06, + "log_odds_chosen": 0.6374708414077759, + "log_odds_ratio": -0.433654248714447, + "logits/chosen": -0.31257641315460205, + "logits/rejected": -1.766423225402832, + "logps/chosen": -1.5535167455673218, + "logps/rejected": -2.087543487548828, + "loss": 1.6746, + "nll_loss": 1.631203293800354, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1553516834974289, + "rewards/margins": 0.05340268462896347, + "rewards/rejected": -0.20875434577465057, + "step": 332 + }, + { + "epoch": 0.9195719710044874, + "grad_norm": 0.20278650522232056, + "learning_rate": 4.387941878861578e-06, + "log_odds_chosen": 0.442794531583786, + "log_odds_ratio": -0.5059284567832947, + "logits/chosen": -0.2818780243396759, + "logits/rejected": -1.41943359375, + "logps/chosen": -1.475003957748413, + "logps/rejected": -1.838566541671753, + "loss": 1.5936, + "nll_loss": 1.5429768562316895, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1475003957748413, + "rewards/margins": 0.03635626286268234, + "rewards/rejected": -0.18385665118694305, + "step": 333 + }, + { + "epoch": 0.9223334483948913, + "grad_norm": 0.21072961390018463, + "learning_rate": 4.382678665009028e-06, + "log_odds_chosen": 0.5503413677215576, + "log_odds_ratio": -0.4611153304576874, + "logits/chosen": -0.24195455014705658, + "logits/rejected": -1.77567720413208, + "logps/chosen": -1.6017379760742188, + "logps/rejected": -2.0620176792144775, + "loss": 1.7086, + "nll_loss": 1.6624857187271118, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16017380356788635, + "rewards/margins": 0.04602799564599991, + "rewards/rejected": -0.20620179176330566, + "step": 334 + }, + { + "epoch": 0.9250949257852952, + "grad_norm": 0.21240665018558502, + "learning_rate": 4.3773961037202784e-06, + "log_odds_chosen": 0.5787122249603271, + "log_odds_ratio": -0.4496590793132782, + "logits/chosen": -0.24867500364780426, + "logits/rejected": -1.7745471000671387, + "logps/chosen": -1.6565507650375366, + "logps/rejected": -2.1457858085632324, + "loss": 1.7518, + "nll_loss": 1.706835150718689, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16565507650375366, + "rewards/margins": 0.04892349988222122, + "rewards/rejected": -0.21457859873771667, + "step": 335 + }, + { + "epoch": 0.927856403175699, + "grad_norm": 0.20056602358818054, + "learning_rate": 4.37209424928182e-06, + "log_odds_chosen": 0.5505763292312622, + "log_odds_ratio": -0.4596712589263916, + "logits/chosen": -0.18031375110149384, + "logits/rejected": -1.4376002550125122, + "logps/chosen": -1.557979941368103, + "logps/rejected": -2.017378091812134, + "loss": 1.6716, + "nll_loss": 1.6256715059280396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1557980179786682, + "rewards/margins": 0.04593981057405472, + "rewards/rejected": -0.20173780620098114, + "step": 336 + }, + { + "epoch": 0.9306178805661028, + "grad_norm": 0.18808571994304657, + "learning_rate": 4.366773156178413e-06, + "log_odds_chosen": 0.42912667989730835, + "log_odds_ratio": -0.5055819153785706, + "logits/chosen": -0.3135528564453125, + "logits/rejected": -1.5518022775650024, + "logps/chosen": -1.481281042098999, + "logps/rejected": -1.826270341873169, + "loss": 1.5991, + "nll_loss": 1.5485769510269165, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14812810719013214, + "rewards/margins": 0.0344989076256752, + "rewards/rejected": -0.18262703716754913, + "step": 337 + }, + { + "epoch": 0.9333793579565067, + "grad_norm": 0.21728971600532532, + "learning_rate": 4.361432879092518e-06, + "log_odds_chosen": 0.5635970234870911, + "log_odds_ratio": -0.4642696976661682, + "logits/chosen": -0.32608091831207275, + "logits/rejected": -1.478355050086975, + "logps/chosen": -1.5051325559616089, + "logps/rejected": -1.974266529083252, + "loss": 1.6273, + "nll_loss": 1.5808420181274414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15051327645778656, + "rewards/margins": 0.04691339656710625, + "rewards/rejected": -0.1974266618490219, + "step": 338 + }, + { + "epoch": 0.9361408353469106, + "grad_norm": 0.20923133194446564, + "learning_rate": 4.356073472903747e-06, + "log_odds_chosen": 0.5899335145950317, + "log_odds_ratio": -0.44590240716934204, + "logits/chosen": -0.17045272886753082, + "logits/rejected": -1.5179771184921265, + "logps/chosen": -1.4507163763046265, + "logps/rejected": -1.9340299367904663, + "loss": 1.5672, + "nll_loss": 1.5226441621780396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14507164061069489, + "rewards/margins": 0.048331368714571, + "rewards/rejected": -0.1934029906988144, + "step": 339 + }, + { + "epoch": 0.9389023127373145, + "grad_norm": 0.20438268780708313, + "learning_rate": 4.350694992688289e-06, + "log_odds_chosen": 0.6362269520759583, + "log_odds_ratio": -0.43733319640159607, + "logits/chosen": -0.14821594953536987, + "logits/rejected": -1.5649621486663818, + "logps/chosen": -1.4848688840866089, + "logps/rejected": -2.009694814682007, + "loss": 1.6031, + "nll_loss": 1.5594022274017334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1484868973493576, + "rewards/margins": 0.05248260498046875, + "rewards/rejected": -0.20096951723098755, + "step": 340 + }, + { + "epoch": 0.9416637901277183, + "grad_norm": 0.20621763169765472, + "learning_rate": 4.345297493718352e-06, + "log_odds_chosen": 0.547203540802002, + "log_odds_ratio": -0.462637722492218, + "logits/chosen": -0.31976550817489624, + "logits/rejected": -1.318708062171936, + "logps/chosen": -1.4836325645446777, + "logps/rejected": -1.930004358291626, + "loss": 1.6093, + "nll_loss": 1.563034176826477, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14836326241493225, + "rewards/margins": 0.044637180864810944, + "rewards/rejected": -0.1930004358291626, + "step": 341 + }, + { + "epoch": 0.9444252675181222, + "grad_norm": 0.18609599769115448, + "learning_rate": 4.339881031461588e-06, + "log_odds_chosen": 0.47142109274864197, + "log_odds_ratio": -0.48913687467575073, + "logits/chosen": -0.26566094160079956, + "logits/rejected": -1.550631046295166, + "logps/chosen": -1.4443541765213013, + "logps/rejected": -1.8148910999298096, + "loss": 1.578, + "nll_loss": 1.5290637016296387, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14443542063236237, + "rewards/margins": 0.03705369308590889, + "rewards/rejected": -0.18148910999298096, + "step": 342 + }, + { + "epoch": 0.9471867449085261, + "grad_norm": 0.1996508240699768, + "learning_rate": 4.334445661580527e-06, + "log_odds_chosen": 0.5693928599357605, + "log_odds_ratio": -0.4520787000656128, + "logits/chosen": -0.33670923113822937, + "logits/rejected": -1.7492713928222656, + "logps/chosen": -1.5564992427825928, + "logps/rejected": -2.030904531478882, + "loss": 1.6725, + "nll_loss": 1.6272705793380737, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15564994513988495, + "rewards/margins": 0.04744052141904831, + "rewards/rejected": -0.20309044420719147, + "step": 343 + }, + { + "epoch": 0.94994822229893, + "grad_norm": 0.19215835630893707, + "learning_rate": 4.328991439932003e-06, + "log_odds_chosen": 0.6314505934715271, + "log_odds_ratio": -0.4288046658039093, + "logits/chosen": -0.29215654730796814, + "logits/rejected": -1.5209310054779053, + "logps/chosen": -1.486976981163025, + "logps/rejected": -2.0075106620788574, + "loss": 1.6116, + "nll_loss": 1.568747878074646, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14869770407676697, + "rewards/margins": 0.05205334722995758, + "rewards/rejected": -0.20075105130672455, + "step": 344 + }, + { + "epoch": 0.9527096996893338, + "grad_norm": 0.20573773980140686, + "learning_rate": 4.323518422566586e-06, + "log_odds_chosen": 0.7072029113769531, + "log_odds_ratio": -0.40426695346832275, + "logits/chosen": -0.23872965574264526, + "logits/rejected": -1.5439509153366089, + "logps/chosen": -1.5577975511550903, + "logps/rejected": -2.149085760116577, + "loss": 1.6903, + "nll_loss": 1.649876594543457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15577976405620575, + "rewards/margins": 0.059128809720277786, + "rewards/rejected": -0.21490855515003204, + "step": 345 + }, + { + "epoch": 0.9554711770797376, + "grad_norm": 0.1889713704586029, + "learning_rate": 4.318026665727993e-06, + "log_odds_chosen": 0.6957321166992188, + "log_odds_ratio": -0.411748468875885, + "logits/chosen": -0.3132311701774597, + "logits/rejected": -1.7383476495742798, + "logps/chosen": -1.4288955926895142, + "logps/rejected": -1.9945143461227417, + "loss": 1.5422, + "nll_loss": 1.501028299331665, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14288955926895142, + "rewards/margins": 0.056561872363090515, + "rewards/rejected": -0.19945143163204193, + "step": 346 + }, + { + "epoch": 0.9582326544701415, + "grad_norm": 0.21370814740657806, + "learning_rate": 4.3125162258525265e-06, + "log_odds_chosen": 0.37931889295578003, + "log_odds_ratio": -0.5239338874816895, + "logits/chosen": -0.34714582562446594, + "logits/rejected": -1.3666698932647705, + "logps/chosen": -1.6313103437423706, + "logps/rejected": -1.9475239515304565, + "loss": 1.7528, + "nll_loss": 1.7004497051239014, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16313102841377258, + "rewards/margins": 0.031621355563402176, + "rewards/rejected": -0.19475241005420685, + "step": 347 + }, + { + "epoch": 0.9609941318605454, + "grad_norm": 0.20202378928661346, + "learning_rate": 4.3069871595684795e-06, + "log_odds_chosen": 0.5231636762619019, + "log_odds_ratio": -0.4718437194824219, + "logits/chosen": -0.3397434949874878, + "logits/rejected": -1.5151662826538086, + "logps/chosen": -1.4959080219268799, + "logps/rejected": -1.9243054389953613, + "loss": 1.6233, + "nll_loss": 1.5760971307754517, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14959080517292023, + "rewards/margins": 0.04283975064754486, + "rewards/rejected": -0.1924305558204651, + "step": 348 + }, + { + "epoch": 0.9637556092509493, + "grad_norm": 0.1996474266052246, + "learning_rate": 4.3014395236955635e-06, + "log_odds_chosen": 0.6398702263832092, + "log_odds_ratio": -0.4294753670692444, + "logits/chosen": -0.2723667621612549, + "logits/rejected": -1.6896021366119385, + "logps/chosen": -1.4915456771850586, + "logps/rejected": -2.0213863849639893, + "loss": 1.6075, + "nll_loss": 1.5645424127578735, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14915457367897034, + "rewards/margins": 0.0529840886592865, + "rewards/rejected": -0.20213866233825684, + "step": 349 + }, + { + "epoch": 0.9665170866413532, + "grad_norm": 0.18923406302928925, + "learning_rate": 4.295873375244319e-06, + "log_odds_chosen": 0.5657058358192444, + "log_odds_ratio": -0.4569021463394165, + "logits/chosen": -0.2960559129714966, + "logits/rejected": -1.5297155380249023, + "logps/chosen": -1.446962833404541, + "logps/rejected": -1.9055999517440796, + "loss": 1.5727, + "nll_loss": 1.527005672454834, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14469628036022186, + "rewards/margins": 0.045863717794418335, + "rewards/rejected": -0.1905599981546402, + "step": 350 + }, + { + "epoch": 0.9692785640317569, + "grad_norm": 0.2020733803510666, + "learning_rate": 4.290288771415536e-06, + "log_odds_chosen": 0.47521698474884033, + "log_odds_ratio": -0.48997536301612854, + "logits/chosen": -0.2961908280849457, + "logits/rejected": -1.738245964050293, + "logps/chosen": -1.6070820093154907, + "logps/rejected": -2.003466844558716, + "loss": 1.7209, + "nll_loss": 1.6719499826431274, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16070818901062012, + "rewards/margins": 0.039638496935367584, + "rewards/rejected": -0.2003466784954071, + "step": 351 + }, + { + "epoch": 0.9720400414221608, + "grad_norm": 0.19740743935108185, + "learning_rate": 4.284685769599658e-06, + "log_odds_chosen": 0.5527662038803101, + "log_odds_ratio": -0.4617147445678711, + "logits/chosen": -0.3190363645553589, + "logits/rejected": -1.6859633922576904, + "logps/chosen": -1.5147120952606201, + "logps/rejected": -1.9768071174621582, + "loss": 1.6243, + "nll_loss": 1.5781550407409668, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15147122740745544, + "rewards/margins": 0.04620949178934097, + "rewards/rejected": -0.19768072664737701, + "step": 352 + }, + { + "epoch": 0.9748015188125647, + "grad_norm": 0.19644393026828766, + "learning_rate": 4.279064427376199e-06, + "log_odds_chosen": 0.5512232780456543, + "log_odds_ratio": -0.45938748121261597, + "logits/chosen": -0.2991534173488617, + "logits/rejected": -1.5584678649902344, + "logps/chosen": -1.5441019535064697, + "logps/rejected": -2.002403736114502, + "loss": 1.6453, + "nll_loss": 1.5993587970733643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1544102132320404, + "rewards/margins": 0.045830175280570984, + "rewards/rejected": -0.2002403736114502, + "step": 353 + }, + { + "epoch": 0.9775629962029686, + "grad_norm": 0.19680863618850708, + "learning_rate": 4.273424802513145e-06, + "log_odds_chosen": 0.5857518911361694, + "log_odds_ratio": -0.4452309012413025, + "logits/chosen": -0.2941315174102783, + "logits/rejected": -1.6820390224456787, + "logps/chosen": -1.5470666885375977, + "logps/rejected": -2.031536102294922, + "loss": 1.6621, + "nll_loss": 1.6175867319107056, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1547066569328308, + "rewards/margins": 0.04844695329666138, + "rewards/rejected": -0.2031536102294922, + "step": 354 + }, + { + "epoch": 0.9803244735933725, + "grad_norm": 0.18272706866264343, + "learning_rate": 4.267766952966369e-06, + "log_odds_chosen": 0.7239515781402588, + "log_odds_ratio": -0.4105169177055359, + "logits/chosen": -0.23595450818538666, + "logits/rejected": -1.5807809829711914, + "logps/chosen": -1.3758444786071777, + "logps/rejected": -1.9615696668624878, + "loss": 1.4949, + "nll_loss": 1.4538719654083252, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13758444786071777, + "rewards/margins": 0.058572523295879364, + "rewards/rejected": -0.19615697860717773, + "step": 355 + }, + { + "epoch": 0.9830859509837763, + "grad_norm": 0.19880905747413635, + "learning_rate": 4.26209093687903e-06, + "log_odds_chosen": 0.4604690670967102, + "log_odds_ratio": -0.4963679313659668, + "logits/chosen": -0.30030739307403564, + "logits/rejected": -1.8233280181884766, + "logps/chosen": -1.5396702289581299, + "logps/rejected": -1.922170639038086, + "loss": 1.6588, + "nll_loss": 1.6091995239257812, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.153967022895813, + "rewards/margins": 0.038250040262937546, + "rewards/rejected": -0.19221706688404083, + "step": 356 + }, + { + "epoch": 0.9858474283741802, + "grad_norm": 0.19747234880924225, + "learning_rate": 4.2563968125809734e-06, + "log_odds_chosen": 0.5938950777053833, + "log_odds_ratio": -0.44322288036346436, + "logits/chosen": -0.17701445519924164, + "logits/rejected": -1.6618235111236572, + "logps/chosen": -1.60313880443573, + "logps/rejected": -2.101062774658203, + "loss": 1.7054, + "nll_loss": 1.6611095666885376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16031388938426971, + "rewards/margins": 0.04979238659143448, + "rewards/rejected": -0.2101062536239624, + "step": 357 + }, + { + "epoch": 0.988608905764584, + "grad_norm": 0.19918540120124817, + "learning_rate": 4.2506846385881375e-06, + "log_odds_chosen": 0.6773942708969116, + "log_odds_ratio": -0.4168775975704193, + "logits/chosen": -0.4877479672431946, + "logits/rejected": -1.6255730390548706, + "logps/chosen": -1.409053087234497, + "logps/rejected": -1.9528459310531616, + "loss": 1.5337, + "nll_loss": 1.491982340812683, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14090532064437866, + "rewards/margins": 0.05437929555773735, + "rewards/rejected": -0.19528460502624512, + "step": 358 + }, + { + "epoch": 0.9913703831549879, + "grad_norm": 0.1901620328426361, + "learning_rate": 4.2449544736019486e-06, + "log_odds_chosen": 0.5646210312843323, + "log_odds_ratio": -0.45523136854171753, + "logits/chosen": -0.22285765409469604, + "logits/rejected": -1.4501441717147827, + "logps/chosen": -1.479009985923767, + "logps/rejected": -1.9385974407196045, + "loss": 1.5958, + "nll_loss": 1.5503125190734863, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1479010134935379, + "rewards/margins": 0.045958735048770905, + "rewards/rejected": -0.1938597410917282, + "step": 359 + }, + { + "epoch": 0.9941318605453918, + "grad_norm": 0.18198014795780182, + "learning_rate": 4.239206376508716e-06, + "log_odds_chosen": 0.7485941648483276, + "log_odds_ratio": -0.39402005076408386, + "logits/chosen": -0.25095412135124207, + "logits/rejected": -1.9625945091247559, + "logps/chosen": -1.4998773336410522, + "logps/rejected": -2.127000570297241, + "loss": 1.6257, + "nll_loss": 1.5863466262817383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14998774230480194, + "rewards/margins": 0.06271231919527054, + "rewards/rejected": -0.21270006895065308, + "step": 360 + }, + { + "epoch": 0.9968933379357956, + "grad_norm": 0.18113547563552856, + "learning_rate": 4.233440406379032e-06, + "log_odds_chosen": 0.560468852519989, + "log_odds_ratio": -0.45691755414009094, + "logits/chosen": -0.35995978116989136, + "logits/rejected": -1.630347490310669, + "logps/chosen": -1.4223926067352295, + "logps/rejected": -1.871058702468872, + "loss": 1.5472, + "nll_loss": 1.501501202583313, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1422392576932907, + "rewards/margins": 0.04486660659313202, + "rewards/rejected": -0.18710586428642273, + "step": 361 + }, + { + "epoch": 0.9996548153261995, + "grad_norm": 0.1977638155221939, + "learning_rate": 4.227656622467162e-06, + "log_odds_chosen": 0.6151151657104492, + "log_odds_ratio": -0.4389611482620239, + "logits/chosen": -0.34485432505607605, + "logits/rejected": -1.5607213973999023, + "logps/chosen": -1.528998851776123, + "logps/rejected": -2.0421009063720703, + "loss": 1.6442, + "nll_loss": 1.6003334522247314, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15289989113807678, + "rewards/margins": 0.05131019651889801, + "rewards/rejected": -0.2042100876569748, + "step": 362 + }, + { + "epoch": 1.0, + "grad_norm": 0.4796704351902008, + "learning_rate": 4.221855084210433e-06, + "log_odds_chosen": 0.42105579376220703, + "log_odds_ratio": -0.5046184659004211, + "logits/chosen": -0.522539496421814, + "logits/rejected": -2.0535831451416016, + "logps/chosen": -1.5670783519744873, + "logps/rejected": -1.9137303829193115, + "loss": 1.6859, + "nll_loss": 1.635398030281067, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15670783817768097, + "rewards/margins": 0.034665197134017944, + "rewards/rejected": -0.1913730353116989, + "step": 363 + }, + { + "epoch": 1.0027614773904039, + "grad_norm": 0.2030143141746521, + "learning_rate": 4.2160358512286266e-06, + "log_odds_chosen": 0.5737169981002808, + "log_odds_ratio": -0.45416390895843506, + "logits/chosen": -0.2930692732334137, + "logits/rejected": -1.6667262315750122, + "logps/chosen": -1.5078061819076538, + "logps/rejected": -1.9778721332550049, + "loss": 1.6304, + "nll_loss": 1.5849525928497314, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1507806032896042, + "rewards/margins": 0.04700660705566406, + "rewards/rejected": -0.19778722524642944, + "step": 364 + }, + { + "epoch": 1.0055229547808078, + "grad_norm": 0.18416902422904968, + "learning_rate": 4.210198983323366e-06, + "log_odds_chosen": 0.6264990568161011, + "log_odds_ratio": -0.4338659346103668, + "logits/chosen": -0.2520604133605957, + "logits/rejected": -1.8486303091049194, + "logps/chosen": -1.5509365797042847, + "logps/rejected": -2.071831703186035, + "loss": 1.6494, + "nll_loss": 1.606053113937378, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15509365499019623, + "rewards/margins": 0.052089497447013855, + "rewards/rejected": -0.20718316733837128, + "step": 365 + }, + { + "epoch": 1.0082844321712117, + "grad_norm": 0.19729849696159363, + "learning_rate": 4.204344540477499e-06, + "log_odds_chosen": 0.6402697563171387, + "log_odds_ratio": -0.429515540599823, + "logits/chosen": -0.288376122713089, + "logits/rejected": -1.6332948207855225, + "logps/chosen": -1.5338780879974365, + "logps/rejected": -2.068819046020508, + "loss": 1.6531, + "nll_loss": 1.6101782321929932, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15338779985904694, + "rewards/margins": 0.053494103252887726, + "rewards/rejected": -0.20688191056251526, + "step": 366 + }, + { + "epoch": 1.0110459095616156, + "grad_norm": 0.18270985782146454, + "learning_rate": 4.1984725828544855e-06, + "log_odds_chosen": 0.6716368198394775, + "log_odds_ratio": -0.41607195138931274, + "logits/chosen": -0.27237242460250854, + "logits/rejected": -1.8720418214797974, + "logps/chosen": -1.4672292470932007, + "logps/rejected": -2.0190887451171875, + "loss": 1.5786, + "nll_loss": 1.5369707345962524, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1467229127883911, + "rewards/margins": 0.0551859587430954, + "rewards/rejected": -0.2019088715314865, + "step": 367 + }, + { + "epoch": 1.0138073869520194, + "grad_norm": 0.20039811730384827, + "learning_rate": 4.192583170797775e-06, + "log_odds_chosen": 0.5032901167869568, + "log_odds_ratio": -0.4751865863800049, + "logits/chosen": -0.3330274224281311, + "logits/rejected": -1.4479092359542847, + "logps/chosen": -1.5037407875061035, + "logps/rejected": -1.9160493612289429, + "loss": 1.6307, + "nll_loss": 1.5831364393234253, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15037408471107483, + "rewards/margins": 0.041230857372283936, + "rewards/rejected": -0.19160494208335876, + "step": 368 + }, + { + "epoch": 1.016568864342423, + "grad_norm": 0.1855594515800476, + "learning_rate": 4.186676364830187e-06, + "log_odds_chosen": 0.5586062669754028, + "log_odds_ratio": -0.45506125688552856, + "logits/chosen": -0.23738795518875122, + "logits/rejected": -1.6253269910812378, + "logps/chosen": -1.566367745399475, + "logps/rejected": -2.030472755432129, + "loss": 1.6675, + "nll_loss": 1.6219737529754639, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1566367745399475, + "rewards/margins": 0.04641049727797508, + "rewards/rejected": -0.20304730534553528, + "step": 369 + }, + { + "epoch": 1.019330341732827, + "grad_norm": 0.2150687575340271, + "learning_rate": 4.1807522256532925e-06, + "log_odds_chosen": 0.6354942321777344, + "log_odds_ratio": -0.43560659885406494, + "logits/chosen": -0.29759910702705383, + "logits/rejected": -1.7936424016952515, + "logps/chosen": -1.621231198310852, + "logps/rejected": -2.1618242263793945, + "loss": 1.7339, + "nll_loss": 1.6903626918792725, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16212311387062073, + "rewards/margins": 0.054059334099292755, + "rewards/rejected": -0.21618244051933289, + "step": 370 + }, + { + "epoch": 1.0220918191232309, + "grad_norm": 0.18705667555332184, + "learning_rate": 4.174810814146789e-06, + "log_odds_chosen": 0.6377235651016235, + "log_odds_ratio": -0.4334976375102997, + "logits/chosen": -0.17882226407527924, + "logits/rejected": -1.4699535369873047, + "logps/chosen": -1.5574746131896973, + "logps/rejected": -2.092029094696045, + "loss": 1.6697, + "nll_loss": 1.626399040222168, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1557474583387375, + "rewards/margins": 0.05345546454191208, + "rewards/rejected": -0.20920291543006897, + "step": 371 + }, + { + "epoch": 1.0248532965136348, + "grad_norm": 0.18511676788330078, + "learning_rate": 4.1688521913678706e-06, + "log_odds_chosen": 0.7460950613021851, + "log_odds_ratio": -0.39828044176101685, + "logits/chosen": -0.24553045630455017, + "logits/rejected": -1.5528843402862549, + "logps/chosen": -1.506624460220337, + "logps/rejected": -2.1312265396118164, + "loss": 1.6125, + "nll_loss": 1.5727205276489258, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15066243708133698, + "rewards/margins": 0.06246021389961243, + "rewards/rejected": -0.2131226509809494, + "step": 372 + }, + { + "epoch": 1.0276147739040387, + "grad_norm": 0.18531759083271027, + "learning_rate": 4.162876418550606e-06, + "log_odds_chosen": 0.5151762962341309, + "log_odds_ratio": -0.47239360213279724, + "logits/chosen": -0.3635602593421936, + "logits/rejected": -1.655716061592102, + "logps/chosen": -1.509564995765686, + "logps/rejected": -1.9321736097335815, + "loss": 1.6228, + "nll_loss": 1.5756025314331055, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15095649659633636, + "rewards/margins": 0.042260847985744476, + "rewards/rejected": -0.19321735203266144, + "step": 373 + }, + { + "epoch": 1.0303762512944425, + "grad_norm": 0.19160981476306915, + "learning_rate": 4.156883557105308e-06, + "log_odds_chosen": 0.537889301776886, + "log_odds_ratio": -0.46930158138275146, + "logits/chosen": -0.3415728509426117, + "logits/rejected": -1.7609002590179443, + "logps/chosen": -1.4833260774612427, + "logps/rejected": -1.925763726234436, + "loss": 1.5902, + "nll_loss": 1.5432225465774536, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1483326107263565, + "rewards/margins": 0.044243764132261276, + "rewards/rejected": -0.1925763636827469, + "step": 374 + }, + { + "epoch": 1.0331377286848464, + "grad_norm": 0.2014252096414566, + "learning_rate": 4.150873668617899e-06, + "log_odds_chosen": 0.5346254110336304, + "log_odds_ratio": -0.4631119966506958, + "logits/chosen": -0.4020942449569702, + "logits/rejected": -1.6286017894744873, + "logps/chosen": -1.554813265800476, + "logps/rejected": -1.9967365264892578, + "loss": 1.6587, + "nll_loss": 1.6124264001846313, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15548132359981537, + "rewards/margins": 0.0441923588514328, + "rewards/rejected": -0.19967366755008698, + "step": 375 + }, + { + "epoch": 1.0358992060752503, + "grad_norm": 0.1909962296485901, + "learning_rate": 4.144846814849282e-06, + "log_odds_chosen": 0.4685186743736267, + "log_odds_ratio": -0.4881165623664856, + "logits/chosen": -0.19894683361053467, + "logits/rejected": -1.4885890483856201, + "logps/chosen": -1.5590081214904785, + "logps/rejected": -1.9445595741271973, + "loss": 1.6834, + "nll_loss": 1.634606957435608, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15590080618858337, + "rewards/margins": 0.03855516016483307, + "rewards/rejected": -0.19445598125457764, + "step": 376 + }, + { + "epoch": 1.0386606834656542, + "grad_norm": 0.1877821981906891, + "learning_rate": 4.138803057734705e-06, + "log_odds_chosen": 0.7200895547866821, + "log_odds_ratio": -0.4008093476295471, + "logits/chosen": -0.25993677973747253, + "logits/rejected": -1.574484944343567, + "logps/chosen": -1.456033706665039, + "logps/rejected": -2.0489182472229004, + "loss": 1.5752, + "nll_loss": 1.5350782871246338, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14560337364673615, + "rewards/margins": 0.05928843468427658, + "rewards/rejected": -0.20489181578159332, + "step": 377 + }, + { + "epoch": 1.041422160856058, + "grad_norm": 0.17346148192882538, + "learning_rate": 4.132742459383122e-06, + "log_odds_chosen": 0.7830832004547119, + "log_odds_ratio": -0.3833864629268646, + "logits/chosen": -0.27740198373794556, + "logits/rejected": -1.7669141292572021, + "logps/chosen": -1.380853533744812, + "logps/rejected": -2.0105137825012207, + "loss": 1.4985, + "nll_loss": 1.4602102041244507, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13808535039424896, + "rewards/margins": 0.06296603381633759, + "rewards/rejected": -0.20105136930942535, + "step": 378 + }, + { + "epoch": 1.0441836382464618, + "grad_norm": 0.183350071310997, + "learning_rate": 4.126665082076559e-06, + "log_odds_chosen": 0.5126218795776367, + "log_odds_ratio": -0.47373971343040466, + "logits/chosen": -0.3141680955886841, + "logits/rejected": -1.3418911695480347, + "logps/chosen": -1.516774296760559, + "logps/rejected": -1.933377742767334, + "loss": 1.6442, + "nll_loss": 1.5968701839447021, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1516774296760559, + "rewards/margins": 0.04166034609079361, + "rewards/rejected": -0.19333778321743011, + "step": 379 + }, + { + "epoch": 1.0469451156368657, + "grad_norm": 0.18558627367019653, + "learning_rate": 4.120570988269472e-06, + "log_odds_chosen": 0.651531994342804, + "log_odds_ratio": -0.423582524061203, + "logits/chosen": -0.38820552825927734, + "logits/rejected": -1.826667070388794, + "logps/chosen": -1.5054341554641724, + "logps/rejected": -2.0457534790039062, + "loss": 1.5926, + "nll_loss": 1.5502351522445679, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1505434215068817, + "rewards/margins": 0.05403192341327667, + "rewards/rejected": -0.20457535982131958, + "step": 380 + }, + { + "epoch": 1.0497065930272695, + "grad_norm": 0.19876988232135773, + "learning_rate": 4.114460240588101e-06, + "log_odds_chosen": 0.7609922885894775, + "log_odds_ratio": -0.3903619050979614, + "logits/chosen": -0.28415167331695557, + "logits/rejected": -1.509800910949707, + "logps/chosen": -1.5427088737487793, + "logps/rejected": -2.184783458709717, + "loss": 1.6545, + "nll_loss": 1.6154515743255615, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15427090227603912, + "rewards/margins": 0.06420743465423584, + "rewards/rejected": -0.21847833693027496, + "step": 381 + }, + { + "epoch": 1.0524680704176734, + "grad_norm": 0.17985652387142181, + "learning_rate": 4.1083329018298356e-06, + "log_odds_chosen": 0.7214679718017578, + "log_odds_ratio": -0.401907354593277, + "logits/chosen": -0.2644188404083252, + "logits/rejected": -1.569467544555664, + "logps/chosen": -1.4487037658691406, + "logps/rejected": -2.043267250061035, + "loss": 1.5525, + "nll_loss": 1.5123515129089355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14487037062644958, + "rewards/margins": 0.05945635586977005, + "rewards/rejected": -0.20432673394680023, + "step": 382 + }, + { + "epoch": 1.0552295478080773, + "grad_norm": 0.19042105972766876, + "learning_rate": 4.102189034962561e-06, + "log_odds_chosen": 0.6154542565345764, + "log_odds_ratio": -0.4352980852127075, + "logits/chosen": -0.21143808960914612, + "logits/rejected": -1.4588818550109863, + "logps/chosen": -1.583728551864624, + "logps/rejected": -2.1008310317993164, + "loss": 1.6812, + "nll_loss": 1.6376571655273438, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15837284922599792, + "rewards/margins": 0.051710255444049835, + "rewards/rejected": -0.21008309721946716, + "step": 383 + }, + { + "epoch": 1.0579910251984812, + "grad_norm": 0.202115997672081, + "learning_rate": 4.096028703124014e-06, + "log_odds_chosen": 0.6282423138618469, + "log_odds_ratio": -0.43798086047172546, + "logits/chosen": -0.3399620056152344, + "logits/rejected": -1.302710771560669, + "logps/chosen": -1.5835597515106201, + "logps/rejected": -2.1131136417388916, + "loss": 1.67, + "nll_loss": 1.6261749267578125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1583559811115265, + "rewards/margins": 0.05295538902282715, + "rewards/rejected": -0.21131137013435364, + "step": 384 + }, + { + "epoch": 1.060752502588885, + "grad_norm": 0.18484216928482056, + "learning_rate": 4.089851969621138e-06, + "log_odds_chosen": 0.7135899662971497, + "log_odds_ratio": -0.41620948910713196, + "logits/chosen": -0.28658849000930786, + "logits/rejected": -1.5913586616516113, + "logps/chosen": -1.425489902496338, + "logps/rejected": -2.015927791595459, + "loss": 1.5551, + "nll_loss": 1.5135202407836914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14254900813102722, + "rewards/margins": 0.059043798595666885, + "rewards/rejected": -0.20159278810024261, + "step": 385 + }, + { + "epoch": 1.063513979979289, + "grad_norm": 0.20954498648643494, + "learning_rate": 4.083658897929425e-06, + "log_odds_chosen": 0.6113450527191162, + "log_odds_ratio": -0.4348878562450409, + "logits/chosen": -0.3510580360889435, + "logits/rejected": -1.5097743272781372, + "logps/chosen": -1.5490520000457764, + "logps/rejected": -2.056283950805664, + "loss": 1.6599, + "nll_loss": 1.616385817527771, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15490520000457764, + "rewards/margins": 0.050723206251859665, + "rewards/rejected": -0.2056283950805664, + "step": 386 + }, + { + "epoch": 1.0662754573696929, + "grad_norm": 0.19627171754837036, + "learning_rate": 4.077449551692268e-06, + "log_odds_chosen": 0.5390780568122864, + "log_odds_ratio": -0.46470367908477783, + "logits/chosen": -0.23739197850227356, + "logits/rejected": -1.621540904045105, + "logps/chosen": -1.555280089378357, + "logps/rejected": -2.001728057861328, + "loss": 1.6682, + "nll_loss": 1.6217542886734009, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1555280089378357, + "rewards/margins": 0.044644795358181, + "rewards/rejected": -0.20017282664775848, + "step": 387 + }, + { + "epoch": 1.0690369347600965, + "grad_norm": 0.1885172724723816, + "learning_rate": 4.071223994720309e-06, + "log_odds_chosen": 0.7198303937911987, + "log_odds_ratio": -0.40047964453697205, + "logits/chosen": -0.38902541995048523, + "logits/rejected": -1.71836519241333, + "logps/chosen": -1.4929300546646118, + "logps/rejected": -2.0929007530212402, + "loss": 1.6048, + "nll_loss": 1.5647118091583252, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14929300546646118, + "rewards/margins": 0.0599970780313015, + "rewards/rejected": -0.20929010212421417, + "step": 388 + }, + { + "epoch": 1.0717984121505004, + "grad_norm": 0.20288851857185364, + "learning_rate": 4.064982290990777e-06, + "log_odds_chosen": 0.5762845277786255, + "log_odds_ratio": -0.45192721486091614, + "logits/chosen": -0.3292975425720215, + "logits/rejected": -1.4449775218963623, + "logps/chosen": -1.4430099725723267, + "logps/rejected": -1.9106184244155884, + "loss": 1.5566, + "nll_loss": 1.511430263519287, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14430099725723267, + "rewards/margins": 0.04676084965467453, + "rewards/rejected": -0.1910618394613266, + "step": 389 + }, + { + "epoch": 1.0745598895409043, + "grad_norm": 0.18857373297214508, + "learning_rate": 4.058724504646834e-06, + "log_odds_chosen": 0.698142945766449, + "log_odds_ratio": -0.41126978397369385, + "logits/chosen": -0.3490840494632721, + "logits/rejected": -1.5612983703613281, + "logps/chosen": -1.4951398372650146, + "logps/rejected": -2.07562255859375, + "loss": 1.6046, + "nll_loss": 1.563432216644287, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14951398968696594, + "rewards/margins": 0.058048274368047714, + "rewards/rejected": -0.20756226778030396, + "step": 390 + }, + { + "epoch": 1.0773213669313082, + "grad_norm": 0.18817338347434998, + "learning_rate": 4.0524506999969185e-06, + "log_odds_chosen": 0.5495921969413757, + "log_odds_ratio": -0.46190890669822693, + "logits/chosen": -0.32165098190307617, + "logits/rejected": -1.7347975969314575, + "logps/chosen": -1.5509448051452637, + "logps/rejected": -2.0019354820251465, + "loss": 1.6579, + "nll_loss": 1.6116652488708496, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15509448945522308, + "rewards/margins": 0.045099057257175446, + "rewards/rejected": -0.20019352436065674, + "step": 391 + }, + { + "epoch": 1.080082844321712, + "grad_norm": 0.18304969370365143, + "learning_rate": 4.046160941514079e-06, + "log_odds_chosen": 0.7243356704711914, + "log_odds_ratio": -0.4019787311553955, + "logits/chosen": -0.20833127200603485, + "logits/rejected": -1.3766155242919922, + "logps/chosen": -1.4449529647827148, + "logps/rejected": -2.0372161865234375, + "loss": 1.5421, + "nll_loss": 1.5019174814224243, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14449530839920044, + "rewards/margins": 0.05922630429267883, + "rewards/rejected": -0.20372159779071808, + "step": 392 + }, + { + "epoch": 1.082844321712116, + "grad_norm": 0.23034730553627014, + "learning_rate": 4.039855293835316e-06, + "log_odds_chosen": 0.6121989488601685, + "log_odds_ratio": -0.4352457523345947, + "logits/chosen": -0.37496262788772583, + "logits/rejected": -1.6959614753723145, + "logps/chosen": -1.4758626222610474, + "logps/rejected": -1.9751558303833008, + "loss": 1.5737, + "nll_loss": 1.530150294303894, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14758625626564026, + "rewards/margins": 0.04992932081222534, + "rewards/rejected": -0.1975155770778656, + "step": 393 + }, + { + "epoch": 1.0856057991025199, + "grad_norm": 0.17257992923259735, + "learning_rate": 4.033533821760917e-06, + "log_odds_chosen": 0.7732821702957153, + "log_odds_ratio": -0.3887802064418793, + "logits/chosen": -0.3277055025100708, + "logits/rejected": -1.868857741355896, + "logps/chosen": -1.4420204162597656, + "logps/rejected": -2.0812883377075195, + "loss": 1.5466, + "nll_loss": 1.5076910257339478, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14420203864574432, + "rewards/margins": 0.06392678618431091, + "rewards/rejected": -0.20812883973121643, + "step": 394 + }, + { + "epoch": 1.0883672764929238, + "grad_norm": 0.1858333796262741, + "learning_rate": 4.027196590253786e-06, + "log_odds_chosen": 0.4944222569465637, + "log_odds_ratio": -0.4831688404083252, + "logits/chosen": -0.28738462924957275, + "logits/rejected": -1.3927885293960571, + "logps/chosen": -1.4958670139312744, + "logps/rejected": -1.8963196277618408, + "loss": 1.6172, + "nll_loss": 1.5688387155532837, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14958669245243073, + "rewards/margins": 0.04004526883363724, + "rewards/rejected": -0.18963195383548737, + "step": 395 + }, + { + "epoch": 1.0911287538833276, + "grad_norm": 0.1886051446199417, + "learning_rate": 4.020843664438783e-06, + "log_odds_chosen": 0.6189723610877991, + "log_odds_ratio": -0.4430191218852997, + "logits/chosen": -0.30028706789016724, + "logits/rejected": -1.7108986377716064, + "logps/chosen": -1.446290135383606, + "logps/rejected": -1.9520014524459839, + "loss": 1.5704, + "nll_loss": 1.5261311531066895, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14462901651859283, + "rewards/margins": 0.05057113245129585, + "rewards/rejected": -0.1952001452445984, + "step": 396 + }, + { + "epoch": 1.0938902312737315, + "grad_norm": 0.189390167593956, + "learning_rate": 4.01447510960205e-06, + "log_odds_chosen": 0.7106426358222961, + "log_odds_ratio": -0.40562015771865845, + "logits/chosen": -0.4158485531806946, + "logits/rejected": -1.7321726083755493, + "logps/chosen": -1.5334806442260742, + "logps/rejected": -2.1250576972961426, + "loss": 1.6332, + "nll_loss": 1.592668890953064, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15334807336330414, + "rewards/margins": 0.059157684445381165, + "rewards/rejected": -0.2125057429075241, + "step": 397 + }, + { + "epoch": 1.0966517086641354, + "grad_norm": 0.19324593245983124, + "learning_rate": 4.008090991190341e-06, + "log_odds_chosen": 0.49737095832824707, + "log_odds_ratio": -0.47871023416519165, + "logits/chosen": -0.20496344566345215, + "logits/rejected": -1.5982781648635864, + "logps/chosen": -1.570819616317749, + "logps/rejected": -1.9830747842788696, + "loss": 1.6702, + "nll_loss": 1.622377634048462, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1570819765329361, + "rewards/margins": 0.04122550040483475, + "rewards/rejected": -0.19830746948719025, + "step": 398 + }, + { + "epoch": 1.099413186054539, + "grad_norm": 0.17873111367225647, + "learning_rate": 4.001691374810352e-06, + "log_odds_chosen": 0.7614402770996094, + "log_odds_ratio": -0.39445751905441284, + "logits/chosen": -0.27544835209846497, + "logits/rejected": -1.992354154586792, + "logps/chosen": -1.5202884674072266, + "logps/rejected": -2.1587905883789062, + "loss": 1.6232, + "nll_loss": 1.5837651491165161, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15202882885932922, + "rewards/margins": 0.06385020911693573, + "rewards/rejected": -0.21587903797626495, + "step": 399 + }, + { + "epoch": 1.102174663444943, + "grad_norm": 0.17042513191699982, + "learning_rate": 3.99527632622804e-06, + "log_odds_chosen": 0.6560631990432739, + "log_odds_ratio": -0.4252350628376007, + "logits/chosen": -0.3314496576786041, + "logits/rejected": -1.559665322303772, + "logps/chosen": -1.4409160614013672, + "logps/rejected": -1.9774514436721802, + "loss": 1.5645, + "nll_loss": 1.5220184326171875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14409160614013672, + "rewards/margins": 0.05365355312824249, + "rewards/rejected": -0.19774514436721802, + "step": 400 + }, + { + "epoch": 1.1049361408353469, + "grad_norm": 0.1929718255996704, + "learning_rate": 3.988845911367957e-06, + "log_odds_chosen": 0.6236564517021179, + "log_odds_ratio": -0.4364378750324249, + "logits/chosen": -0.3177638649940491, + "logits/rejected": -1.5883958339691162, + "logps/chosen": -1.598113775253296, + "logps/rejected": -2.1222128868103027, + "loss": 1.6877, + "nll_loss": 1.644063949584961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1598113626241684, + "rewards/margins": 0.052409932017326355, + "rewards/rejected": -0.21222129464149475, + "step": 401 + }, + { + "epoch": 1.1076976182257507, + "grad_norm": 0.17265060544013977, + "learning_rate": 3.982400196312565e-06, + "log_odds_chosen": 0.6908576488494873, + "log_odds_ratio": -0.4076445698738098, + "logits/chosen": -0.28831660747528076, + "logits/rejected": -1.4703201055526733, + "logps/chosen": -1.4102579355239868, + "logps/rejected": -1.9662861824035645, + "loss": 1.5263, + "nll_loss": 1.4855272769927979, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14102579653263092, + "rewards/margins": 0.05560281127691269, + "rewards/rejected": -0.1966286301612854, + "step": 402 + }, + { + "epoch": 1.1104590956161546, + "grad_norm": 0.18258056044578552, + "learning_rate": 3.975939247301558e-06, + "log_odds_chosen": 0.6171894669532776, + "log_odds_ratio": -0.43477940559387207, + "logits/chosen": -0.34336021542549133, + "logits/rejected": -1.718552589416504, + "logps/chosen": -1.5480486154556274, + "logps/rejected": -2.061230421066284, + "loss": 1.6639, + "nll_loss": 1.620387077331543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15480485558509827, + "rewards/margins": 0.051318198442459106, + "rewards/rejected": -0.20612303912639618, + "step": 403 + }, + { + "epoch": 1.1132205730065585, + "grad_norm": 0.17894716560840607, + "learning_rate": 3.969463130731183e-06, + "log_odds_chosen": 0.601898729801178, + "log_odds_ratio": -0.44408684968948364, + "logits/chosen": -0.2595762610435486, + "logits/rejected": -1.671970248222351, + "logps/chosen": -1.480761170387268, + "logps/rejected": -1.973817229270935, + "loss": 1.5901, + "nll_loss": 1.5457121133804321, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.148076131939888, + "rewards/margins": 0.049305595457553864, + "rewards/rejected": -0.19738171994686127, + "step": 404 + }, + { + "epoch": 1.1159820503969624, + "grad_norm": 0.182702898979187, + "learning_rate": 3.9629719131535595e-06, + "log_odds_chosen": 0.5538443922996521, + "log_odds_ratio": -0.463712602853775, + "logits/chosen": -0.3739258646965027, + "logits/rejected": -1.5419493913650513, + "logps/chosen": -1.4951484203338623, + "logps/rejected": -1.9511198997497559, + "loss": 1.6103, + "nll_loss": 1.563896656036377, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14951485395431519, + "rewards/margins": 0.04559716209769249, + "rewards/rejected": -0.19511200487613678, + "step": 405 + }, + { + "epoch": 1.1187435277873663, + "grad_norm": 0.1713375300168991, + "learning_rate": 3.9564656612759904e-06, + "log_odds_chosen": 0.6431328058242798, + "log_odds_ratio": -0.42764732241630554, + "logits/chosen": -0.29896292090415955, + "logits/rejected": -1.8772965669631958, + "logps/chosen": -1.4999200105667114, + "logps/rejected": -2.031973361968994, + "loss": 1.6098, + "nll_loss": 1.5670844316482544, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14999200403690338, + "rewards/margins": 0.05320533737540245, + "rewards/rejected": -0.20319733023643494, + "step": 406 + }, + { + "epoch": 1.1215050051777702, + "grad_norm": 0.1880570352077484, + "learning_rate": 3.94994444196028e-06, + "log_odds_chosen": 0.6853294968605042, + "log_odds_ratio": -0.41755619645118713, + "logits/chosen": -0.2672858238220215, + "logits/rejected": -1.5635608434677124, + "logps/chosen": -1.5350513458251953, + "logps/rejected": -2.1063671112060547, + "loss": 1.6214, + "nll_loss": 1.579624056816101, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1535051316022873, + "rewards/margins": 0.0571315735578537, + "rewards/rejected": -0.21063672006130219, + "step": 407 + }, + { + "epoch": 1.124266482568174, + "grad_norm": 0.17640192806720734, + "learning_rate": 3.943408322222049e-06, + "log_odds_chosen": 0.6473301649093628, + "log_odds_ratio": -0.430767297744751, + "logits/chosen": -0.25322699546813965, + "logits/rejected": -1.3362030982971191, + "logps/chosen": -1.4611663818359375, + "logps/rejected": -1.9962221384048462, + "loss": 1.5796, + "nll_loss": 1.5365506410598755, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14611662924289703, + "rewards/margins": 0.053505584597587585, + "rewards/rejected": -0.19962221384048462, + "step": 408 + }, + { + "epoch": 1.127027959958578, + "grad_norm": 0.1975967288017273, + "learning_rate": 3.936857369230037e-06, + "log_odds_chosen": 0.5013847351074219, + "log_odds_ratio": -0.48179811239242554, + "logits/chosen": -0.24628515541553497, + "logits/rejected": -1.5175528526306152, + "logps/chosen": -1.5307085514068604, + "logps/rejected": -1.9439282417297363, + "loss": 1.6331, + "nll_loss": 1.584876537322998, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.153070867061615, + "rewards/margins": 0.04132195562124252, + "rewards/rejected": -0.1943928301334381, + "step": 409 + }, + { + "epoch": 1.1297894373489816, + "grad_norm": 0.19114039838314056, + "learning_rate": 3.930291650305424e-06, + "log_odds_chosen": 0.6988283395767212, + "log_odds_ratio": -0.40597397089004517, + "logits/chosen": -0.25875231623649597, + "logits/rejected": -1.3210163116455078, + "logps/chosen": -1.4922490119934082, + "logps/rejected": -2.0712697505950928, + "loss": 1.5831, + "nll_loss": 1.5425041913986206, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1492249071598053, + "rewards/margins": 0.05790204927325249, + "rewards/rejected": -0.20712696015834808, + "step": 410 + }, + { + "epoch": 1.1325509147393855, + "grad_norm": 0.1878093183040619, + "learning_rate": 3.92371123292113e-06, + "log_odds_chosen": 0.6247770190238953, + "log_odds_ratio": -0.43160274624824524, + "logits/chosen": -0.2506449520587921, + "logits/rejected": -1.691251516342163, + "logps/chosen": -1.5277273654937744, + "logps/rejected": -2.0446317195892334, + "loss": 1.6273, + "nll_loss": 1.5841726064682007, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15277275443077087, + "rewards/margins": 0.051690444350242615, + "rewards/rejected": -0.2044631838798523, + "step": 411 + }, + { + "epoch": 1.1353123921297894, + "grad_norm": 0.1842024326324463, + "learning_rate": 3.917116184701125e-06, + "log_odds_chosen": 0.7279251217842102, + "log_odds_ratio": -0.4019698202610016, + "logits/chosen": -0.2969970107078552, + "logits/rejected": -1.9143730401992798, + "logps/chosen": -1.4923046827316284, + "logps/rejected": -2.099824905395508, + "loss": 1.5904, + "nll_loss": 1.5501734018325806, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1492304652929306, + "rewards/margins": 0.0607520155608654, + "rewards/rejected": -0.2099824696779251, + "step": 412 + }, + { + "epoch": 1.1380738695201933, + "grad_norm": 0.2017175853252411, + "learning_rate": 3.910506573419734e-06, + "log_odds_chosen": 0.8388389348983765, + "log_odds_ratio": -0.3656638562679291, + "logits/chosen": -0.27687397599220276, + "logits/rejected": -1.5076192617416382, + "logps/chosen": -1.4898481369018555, + "logps/rejected": -2.190901279449463, + "loss": 1.5889, + "nll_loss": 1.5523051023483276, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14898481965065002, + "rewards/margins": 0.07010531425476074, + "rewards/rejected": -0.21909013390541077, + "step": 413 + }, + { + "epoch": 1.1408353469105972, + "grad_norm": 0.18677456676959991, + "learning_rate": 3.903882467000938e-06, + "log_odds_chosen": 0.789838433265686, + "log_odds_ratio": -0.37885501980781555, + "logits/chosen": -0.2613790035247803, + "logits/rejected": -1.6607636213302612, + "logps/chosen": -1.5320932865142822, + "logps/rejected": -2.194924831390381, + "loss": 1.6278, + "nll_loss": 1.5898661613464355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15320934355258942, + "rewards/margins": 0.06628316640853882, + "rewards/rejected": -0.21949250996112823, + "step": 414 + }, + { + "epoch": 1.143596824301001, + "grad_norm": 0.18295292556285858, + "learning_rate": 3.897243933517679e-06, + "log_odds_chosen": 0.697784423828125, + "log_odds_ratio": -0.40981873869895935, + "logits/chosen": -0.36293089389801025, + "logits/rejected": -1.7186784744262695, + "logps/chosen": -1.4446918964385986, + "logps/rejected": -2.0167624950408936, + "loss": 1.5485, + "nll_loss": 1.50748872756958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14446918666362762, + "rewards/margins": 0.05720707029104233, + "rewards/rejected": -0.20167624950408936, + "step": 415 + }, + { + "epoch": 1.146358301691405, + "grad_norm": 0.18878361582756042, + "learning_rate": 3.890591041191162e-06, + "log_odds_chosen": 0.7591959834098816, + "log_odds_ratio": -0.39290934801101685, + "logits/chosen": -0.2790437340736389, + "logits/rejected": -1.4805063009262085, + "logps/chosen": -1.4139214754104614, + "logps/rejected": -2.03341007232666, + "loss": 1.5176, + "nll_loss": 1.478288173675537, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14139214158058167, + "rewards/margins": 0.06194887310266495, + "rewards/rejected": -0.20334100723266602, + "step": 416 + }, + { + "epoch": 1.1491197790818088, + "grad_norm": 0.2020062804222107, + "learning_rate": 3.883923858390149e-06, + "log_odds_chosen": 0.75853431224823, + "log_odds_ratio": -0.38675639033317566, + "logits/chosen": -0.3220999538898468, + "logits/rejected": -1.7704015970230103, + "logps/chosen": -1.5146489143371582, + "logps/rejected": -2.149966239929199, + "loss": 1.6306, + "nll_loss": 1.5919427871704102, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15146489441394806, + "rewards/margins": 0.06353174149990082, + "rewards/rejected": -0.21499663591384888, + "step": 417 + }, + { + "epoch": 1.1518812564722127, + "grad_norm": 0.17892806231975555, + "learning_rate": 3.8772424536302565e-06, + "log_odds_chosen": 0.5580451488494873, + "log_odds_ratio": -0.4568861126899719, + "logits/chosen": -0.3027711510658264, + "logits/rejected": -1.707384467124939, + "logps/chosen": -1.5017491579055786, + "logps/rejected": -1.9595609903335571, + "loss": 1.5943, + "nll_loss": 1.5486559867858887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15017491579055786, + "rewards/margins": 0.04578119143843651, + "rewards/rejected": -0.19595609605312347, + "step": 418 + }, + { + "epoch": 1.1546427338626164, + "grad_norm": 0.19901612401008606, + "learning_rate": 3.870546895573258e-06, + "log_odds_chosen": 0.6981452703475952, + "log_odds_ratio": -0.4115220010280609, + "logits/chosen": -0.3455246686935425, + "logits/rejected": -1.3123235702514648, + "logps/chosen": -1.479371428489685, + "logps/rejected": -2.0550196170806885, + "loss": 1.5787, + "nll_loss": 1.5375878810882568, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14793714880943298, + "rewards/margins": 0.05756482481956482, + "rewards/rejected": -0.2055019587278366, + "step": 419 + }, + { + "epoch": 1.1574042112530203, + "grad_norm": 0.1906924694776535, + "learning_rate": 3.863837253026372e-06, + "log_odds_chosen": 0.7406031489372253, + "log_odds_ratio": -0.40153640508651733, + "logits/chosen": -0.37284234166145325, + "logits/rejected": -1.6314030885696411, + "logps/chosen": -1.4350590705871582, + "logps/rejected": -2.0481374263763428, + "loss": 1.5474, + "nll_loss": 1.507211446762085, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14350590109825134, + "rewards/margins": 0.061307840049266815, + "rewards/rejected": -0.20481374859809875, + "step": 420 + }, + { + "epoch": 1.1601656886434242, + "grad_norm": 0.19268250465393066, + "learning_rate": 3.857113594941556e-06, + "log_odds_chosen": 0.5946828126907349, + "log_odds_ratio": -0.44611191749572754, + "logits/chosen": -0.3333417475223541, + "logits/rejected": -1.820539116859436, + "logps/chosen": -1.5357310771942139, + "logps/rejected": -2.030311346054077, + "loss": 1.6374, + "nll_loss": 1.5927873849868774, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15357311069965363, + "rewards/margins": 0.04945802688598633, + "rewards/rejected": -0.20303113758563995, + "step": 421 + }, + { + "epoch": 1.162927166033828, + "grad_norm": 0.1852002888917923, + "learning_rate": 3.8503759904148005e-06, + "log_odds_chosen": 0.753965437412262, + "log_odds_ratio": -0.39210668206214905, + "logits/chosen": -0.2807456851005554, + "logits/rejected": -1.7219178676605225, + "logps/chosen": -1.4659725427627563, + "logps/rejected": -2.0921542644500732, + "loss": 1.5699, + "nll_loss": 1.5306528806686401, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1465972512960434, + "rewards/margins": 0.0626181811094284, + "rewards/rejected": -0.209215447306633, + "step": 422 + }, + { + "epoch": 1.165688643424232, + "grad_norm": 0.18899178504943848, + "learning_rate": 3.843624508685416e-06, + "log_odds_chosen": 0.6049898266792297, + "log_odds_ratio": -0.43925538659095764, + "logits/chosen": -0.3452371060848236, + "logits/rejected": -1.7807867527008057, + "logps/chosen": -1.5395519733428955, + "logps/rejected": -2.040437698364258, + "loss": 1.6476, + "nll_loss": 1.603638768196106, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15395519137382507, + "rewards/margins": 0.05008859187364578, + "rewards/rejected": -0.20404377579689026, + "step": 423 + }, + { + "epoch": 1.1684501208146358, + "grad_norm": 0.1812516748905182, + "learning_rate": 3.8368592191353246e-06, + "log_odds_chosen": 0.6994820833206177, + "log_odds_ratio": -0.412604957818985, + "logits/chosen": -0.32768282294273376, + "logits/rejected": -1.7750557661056519, + "logps/chosen": -1.4714299440383911, + "logps/rejected": -2.0467658042907715, + "loss": 1.5632, + "nll_loss": 1.5219401121139526, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14714302122592926, + "rewards/margins": 0.05753358453512192, + "rewards/rejected": -0.20467659831047058, + "step": 424 + }, + { + "epoch": 1.1712115982050397, + "grad_norm": 0.18136382102966309, + "learning_rate": 3.830080191288342e-06, + "log_odds_chosen": 0.6253668665885925, + "log_odds_ratio": -0.42983123660087585, + "logits/chosen": -0.2867172360420227, + "logits/rejected": -1.5802618265151978, + "logps/chosen": -1.5029292106628418, + "logps/rejected": -2.018980026245117, + "loss": 1.6134, + "nll_loss": 1.5704439878463745, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15029294788837433, + "rewards/margins": 0.05160506069660187, + "rewards/rejected": -0.2018979787826538, + "step": 425 + }, + { + "epoch": 1.1739730755954436, + "grad_norm": 0.19747108221054077, + "learning_rate": 3.823287494809469e-06, + "log_odds_chosen": 0.7677461504936218, + "log_odds_ratio": -0.38692528009414673, + "logits/chosen": -0.38328537344932556, + "logits/rejected": -1.6817901134490967, + "logps/chosen": -1.4731775522232056, + "logps/rejected": -2.1067628860473633, + "loss": 1.5885, + "nll_loss": 1.5498508214950562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14731775224208832, + "rewards/margins": 0.06335853040218353, + "rewards/rejected": -0.21067628264427185, + "step": 426 + }, + { + "epoch": 1.1767345529858475, + "grad_norm": 0.185506671667099, + "learning_rate": 3.816481199504171e-06, + "log_odds_chosen": 0.6367388963699341, + "log_odds_ratio": -0.4264675974845886, + "logits/chosen": -0.3647797703742981, + "logits/rejected": -1.5863122940063477, + "logps/chosen": -1.5056264400482178, + "logps/rejected": -2.031801462173462, + "loss": 1.6257, + "nll_loss": 1.5830763578414917, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15056264400482178, + "rewards/margins": 0.052617497742176056, + "rewards/rejected": -0.20318013429641724, + "step": 427 + }, + { + "epoch": 1.1794960303762512, + "grad_norm": 0.1755794882774353, + "learning_rate": 3.8096613753176635e-06, + "log_odds_chosen": 0.584825873374939, + "log_odds_ratio": -0.4493084251880646, + "logits/chosen": -0.27612411975860596, + "logits/rejected": -1.4378138780593872, + "logps/chosen": -1.3924418687820435, + "logps/rejected": -1.8575388193130493, + "loss": 1.5104, + "nll_loss": 1.4654783010482788, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1392441838979721, + "rewards/margins": 0.046509698033332825, + "rewards/rejected": -0.18575388193130493, + "step": 428 + }, + { + "epoch": 1.1822575077666553, + "grad_norm": 0.19249552488327026, + "learning_rate": 3.8028280923341927e-06, + "log_odds_chosen": 0.6473675966262817, + "log_odds_ratio": -0.4286215901374817, + "logits/chosen": -0.384939968585968, + "logits/rejected": -1.5598773956298828, + "logps/chosen": -1.4334121942520142, + "logps/rejected": -1.9639126062393188, + "loss": 1.5432, + "nll_loss": 1.500383973121643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14334121346473694, + "rewards/margins": 0.05305003747344017, + "rewards/rejected": -0.1963912546634674, + "step": 429 + }, + { + "epoch": 1.185018985157059, + "grad_norm": 0.19077420234680176, + "learning_rate": 3.7959814207763134e-06, + "log_odds_chosen": 0.7534008026123047, + "log_odds_ratio": -0.3935543894767761, + "logits/chosen": -0.33575281500816345, + "logits/rejected": -1.7560871839523315, + "logps/chosen": -1.5460617542266846, + "logps/rejected": -2.17948579788208, + "loss": 1.6361, + "nll_loss": 1.5967589616775513, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1546061784029007, + "rewards/margins": 0.0633423924446106, + "rewards/rejected": -0.2179485559463501, + "step": 430 + }, + { + "epoch": 1.1877804625474628, + "grad_norm": 0.18546007573604584, + "learning_rate": 3.789121431004168e-06, + "log_odds_chosen": 0.6419503092765808, + "log_odds_ratio": -0.42676353454589844, + "logits/chosen": -0.36785295605659485, + "logits/rejected": -1.6887683868408203, + "logps/chosen": -1.5084519386291504, + "logps/rejected": -2.036573886871338, + "loss": 1.5997, + "nll_loss": 1.5570096969604492, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15084518492221832, + "rewards/margins": 0.05281219631433487, + "rewards/rejected": -0.2036573737859726, + "step": 431 + }, + { + "epoch": 1.1905419399378667, + "grad_norm": 0.194391131401062, + "learning_rate": 3.782248193514766e-06, + "log_odds_chosen": 0.6094563007354736, + "log_odds_ratio": -0.43929269909858704, + "logits/chosen": -0.29947635531425476, + "logits/rejected": -1.4955629110336304, + "logps/chosen": -1.4138904809951782, + "logps/rejected": -1.906821846961975, + "loss": 1.5403, + "nll_loss": 1.4963343143463135, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14138904213905334, + "rewards/margins": 0.049293152987957, + "rewards/rejected": -0.19068220257759094, + "step": 432 + }, + { + "epoch": 1.1933034173282706, + "grad_norm": 0.19166447222232819, + "learning_rate": 3.775361778941257e-06, + "log_odds_chosen": 0.7113713026046753, + "log_odds_ratio": -0.403501957654953, + "logits/chosen": -0.308038592338562, + "logits/rejected": -1.6283587217330933, + "logps/chosen": -1.501065731048584, + "logps/rejected": -2.0941624641418457, + "loss": 1.6079, + "nll_loss": 1.567560076713562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15010657906532288, + "rewards/margins": 0.05930966138839722, + "rewards/rejected": -0.2094162404537201, + "step": 433 + }, + { + "epoch": 1.1960648947186745, + "grad_norm": 0.18622690439224243, + "learning_rate": 3.7684622580522057e-06, + "log_odds_chosen": 0.42840951681137085, + "log_odds_ratio": -0.5051460862159729, + "logits/chosen": -0.2970636785030365, + "logits/rejected": -1.6902334690093994, + "logps/chosen": -1.4522030353546143, + "logps/rejected": -1.7967474460601807, + "loss": 1.5557, + "nll_loss": 1.5051684379577637, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1452203243970871, + "rewards/margins": 0.03445442020893097, + "rewards/rejected": -0.17967472970485687, + "step": 434 + }, + { + "epoch": 1.1988263721090784, + "grad_norm": 0.17388908565044403, + "learning_rate": 3.761549701750865e-06, + "log_odds_chosen": 0.7077789902687073, + "log_odds_ratio": -0.41333314776420593, + "logits/chosen": -0.3090924024581909, + "logits/rejected": -1.843423843383789, + "logps/chosen": -1.4652737379074097, + "logps/rejected": -2.052661418914795, + "loss": 1.5629, + "nll_loss": 1.5216063261032104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14652739465236664, + "rewards/margins": 0.058738768100738525, + "rewards/rejected": -0.20526614785194397, + "step": 435 + }, + { + "epoch": 1.2015878494994823, + "grad_norm": 0.17293590307235718, + "learning_rate": 3.7546241810744444e-06, + "log_odds_chosen": 0.5752092003822327, + "log_odds_ratio": -0.4525831639766693, + "logits/chosen": -0.32182741165161133, + "logits/rejected": -1.3567001819610596, + "logps/chosen": -1.4578838348388672, + "logps/rejected": -1.9286342859268188, + "loss": 1.5493, + "nll_loss": 1.50407075881958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14578840136528015, + "rewards/margins": 0.047075025737285614, + "rewards/rejected": -0.19286341965198517, + "step": 436 + }, + { + "epoch": 1.2043493268898862, + "grad_norm": 0.19718696177005768, + "learning_rate": 3.747685767193385e-06, + "log_odds_chosen": 0.5642775297164917, + "log_odds_ratio": -0.4569299817085266, + "logits/chosen": -0.3862457871437073, + "logits/rejected": -1.4825284481048584, + "logps/chosen": -1.5323418378829956, + "logps/rejected": -1.9992895126342773, + "loss": 1.6529, + "nll_loss": 1.6072373390197754, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15323419868946075, + "rewards/margins": 0.046694785356521606, + "rewards/rejected": -0.19992896914482117, + "step": 437 + }, + { + "epoch": 1.20711080428029, + "grad_norm": 0.17915678024291992, + "learning_rate": 3.740734531410626e-06, + "log_odds_chosen": 0.7052585482597351, + "log_odds_ratio": -0.4100540578365326, + "logits/chosen": -0.3775138258934021, + "logits/rejected": -1.859032154083252, + "logps/chosen": -1.448523998260498, + "logps/rejected": -2.021770477294922, + "loss": 1.5652, + "nll_loss": 1.5241587162017822, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1448523849248886, + "rewards/margins": 0.057324644178152084, + "rewards/rejected": -0.202177032828331, + "step": 438 + }, + { + "epoch": 1.2098722816706937, + "grad_norm": 0.19827017188072205, + "learning_rate": 3.7337705451608676e-06, + "log_odds_chosen": 0.859776496887207, + "log_odds_ratio": -0.3666639029979706, + "logits/chosen": -0.4492262601852417, + "logits/rejected": -1.819311261177063, + "logps/chosen": -1.3853495121002197, + "logps/rejected": -2.082892894744873, + "loss": 1.4954, + "nll_loss": 1.4586937427520752, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13853494822978973, + "rewards/margins": 0.06975432485342026, + "rewards/rejected": -0.2082892656326294, + "step": 439 + }, + { + "epoch": 1.2126337590610976, + "grad_norm": 0.19302192330360413, + "learning_rate": 3.7267938800098454e-06, + "log_odds_chosen": 0.8500153422355652, + "log_odds_ratio": -0.36537328362464905, + "logits/chosen": -0.5428920984268188, + "logits/rejected": -1.821568489074707, + "logps/chosen": -1.3568768501281738, + "logps/rejected": -2.052211046218872, + "loss": 1.4693, + "nll_loss": 1.4327150583267212, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1356876939535141, + "rewards/margins": 0.06953340768814087, + "rewards/rejected": -0.20522108674049377, + "step": 440 + }, + { + "epoch": 1.2153952364515015, + "grad_norm": 0.19974324107170105, + "learning_rate": 3.7198046076535865e-06, + "log_odds_chosen": 0.7102779746055603, + "log_odds_ratio": -0.40437954664230347, + "logits/chosen": -0.3492871820926666, + "logits/rejected": -1.5553520917892456, + "logps/chosen": -1.4951152801513672, + "logps/rejected": -2.0831298828125, + "loss": 1.6058, + "nll_loss": 1.565347671508789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14951153099536896, + "rewards/margins": 0.05880144238471985, + "rewards/rejected": -0.20831298828125, + "step": 441 + }, + { + "epoch": 1.2181567138419054, + "grad_norm": 0.18230760097503662, + "learning_rate": 3.71280279991768e-06, + "log_odds_chosen": 0.7506513595581055, + "log_odds_ratio": -0.3923317492008209, + "logits/chosen": -0.4152137339115143, + "logits/rejected": -1.5510238409042358, + "logps/chosen": -1.4440295696258545, + "logps/rejected": -2.0645270347595215, + "loss": 1.5654, + "nll_loss": 1.5261952877044678, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14440295100212097, + "rewards/margins": 0.06204976886510849, + "rewards/rejected": -0.20645272731781006, + "step": 442 + }, + { + "epoch": 1.2209181912323093, + "grad_norm": 0.17220567166805267, + "learning_rate": 3.705788528756533e-06, + "log_odds_chosen": 0.7942907810211182, + "log_odds_ratio": -0.37784573435783386, + "logits/chosen": -0.43315792083740234, + "logits/rejected": -1.8588192462921143, + "logps/chosen": -1.4370851516723633, + "logps/rejected": -2.0935044288635254, + "loss": 1.5386, + "nll_loss": 1.500786304473877, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14370852708816528, + "rewards/margins": 0.06564193964004517, + "rewards/rejected": -0.20935045182704926, + "step": 443 + }, + { + "epoch": 1.2236796686227132, + "grad_norm": 0.34615159034729004, + "learning_rate": 3.698761866252635e-06, + "log_odds_chosen": 0.6159027218818665, + "log_odds_ratio": -0.44577932357788086, + "logits/chosen": -0.4024880826473236, + "logits/rejected": -1.7161859273910522, + "logps/chosen": -1.533597707748413, + "logps/rejected": -2.050471067428589, + "loss": 1.6393, + "nll_loss": 1.5947318077087402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1533597856760025, + "rewards/margins": 0.0516873374581337, + "rewards/rejected": -0.2050471305847168, + "step": 444 + }, + { + "epoch": 1.226441146013117, + "grad_norm": 0.19309046864509583, + "learning_rate": 3.691722884615814e-06, + "log_odds_chosen": 0.6141983270645142, + "log_odds_ratio": -0.43802952766418457, + "logits/chosen": -0.2951904237270355, + "logits/rejected": -1.1809048652648926, + "logps/chosen": -1.4468181133270264, + "logps/rejected": -1.9419705867767334, + "loss": 1.5744, + "nll_loss": 1.5305941104888916, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14468181133270264, + "rewards/margins": 0.04951523244380951, + "rewards/rejected": -0.19419705867767334, + "step": 445 + }, + { + "epoch": 1.229202623403521, + "grad_norm": 0.19239898025989532, + "learning_rate": 3.684671656182497e-06, + "log_odds_chosen": 0.7434544563293457, + "log_odds_ratio": -0.39474281668663025, + "logits/chosen": -0.4174625277519226, + "logits/rejected": -1.7456690073013306, + "logps/chosen": -1.4840799570083618, + "logps/rejected": -2.0977742671966553, + "loss": 1.5985, + "nll_loss": 1.5590325593948364, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14840799570083618, + "rewards/margins": 0.06136942654848099, + "rewards/rejected": -0.20977741479873657, + "step": 446 + }, + { + "epoch": 1.2319641007939248, + "grad_norm": 0.19208678603172302, + "learning_rate": 3.6776082534149664e-06, + "log_odds_chosen": 0.7092225551605225, + "log_odds_ratio": -0.40430110692977905, + "logits/chosen": -0.4059605598449707, + "logits/rejected": -1.8414146900177002, + "logps/chosen": -1.5050946474075317, + "logps/rejected": -2.0934860706329346, + "loss": 1.5881, + "nll_loss": 1.5477027893066406, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15050947666168213, + "rewards/margins": 0.05883914604783058, + "rewards/rejected": -0.2093486189842224, + "step": 447 + }, + { + "epoch": 1.2347255781843287, + "grad_norm": 0.18131853640079498, + "learning_rate": 3.670532748900615e-06, + "log_odds_chosen": 0.7535479664802551, + "log_odds_ratio": -0.3983326554298401, + "logits/chosen": -0.3855303227901459, + "logits/rejected": -1.6423949003219604, + "logps/chosen": -1.4195005893707275, + "logps/rejected": -2.0328564643859863, + "loss": 1.5252, + "nll_loss": 1.4853616952896118, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14195004105567932, + "rewards/margins": 0.06133558601140976, + "rewards/rejected": -0.20328564941883087, + "step": 448 + }, + { + "epoch": 1.2374870555747326, + "grad_norm": 0.1773749440908432, + "learning_rate": 3.663445215351198e-06, + "log_odds_chosen": 0.794275164604187, + "log_odds_ratio": -0.38321515917778015, + "logits/chosen": -0.4386705160140991, + "logits/rejected": -1.5580754280090332, + "logps/chosen": -1.4094889163970947, + "logps/rejected": -2.067706823348999, + "loss": 1.5152, + "nll_loss": 1.476863145828247, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14094889163970947, + "rewards/margins": 0.06582178175449371, + "rewards/rejected": -0.20677067339420319, + "step": 449 + }, + { + "epoch": 1.2402485329651363, + "grad_norm": 0.18508228659629822, + "learning_rate": 3.656345725602089e-06, + "log_odds_chosen": 0.8697155714035034, + "log_odds_ratio": -0.35649657249450684, + "logits/chosen": -0.36322081089019775, + "logits/rejected": -1.8790079355239868, + "logps/chosen": -1.5029995441436768, + "logps/rejected": -2.235288143157959, + "loss": 1.5982, + "nll_loss": 1.5625407695770264, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15029993653297424, + "rewards/margins": 0.0732288807630539, + "rewards/rejected": -0.22352883219718933, + "step": 450 + }, + { + "epoch": 1.2430100103555402, + "grad_norm": 0.19414876401424408, + "learning_rate": 3.6492343526115292e-06, + "log_odds_chosen": 0.6894505023956299, + "log_odds_ratio": -0.4081804156303406, + "logits/chosen": -0.26924291253089905, + "logits/rejected": -1.8292944431304932, + "logps/chosen": -1.5266273021697998, + "logps/rejected": -2.100613594055176, + "loss": 1.612, + "nll_loss": 1.5711458921432495, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1526627391576767, + "rewards/margins": 0.05739862471818924, + "rewards/rejected": -0.21006137132644653, + "step": 451 + }, + { + "epoch": 1.245771487745944, + "grad_norm": 0.18495413661003113, + "learning_rate": 3.642111169459879e-06, + "log_odds_chosen": 0.5452620983123779, + "log_odds_ratio": -0.4608571529388428, + "logits/chosen": -0.3355942666530609, + "logits/rejected": -1.535355806350708, + "logps/chosen": -1.489123821258545, + "logps/rejected": -1.9351115226745605, + "loss": 1.5796, + "nll_loss": 1.533546805381775, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14891238510608673, + "rewards/margins": 0.044598765671253204, + "rewards/rejected": -0.19351115822792053, + "step": 452 + }, + { + "epoch": 1.248532965136348, + "grad_norm": 0.18209105730056763, + "learning_rate": 3.634976249348867e-06, + "log_odds_chosen": 0.6647239923477173, + "log_odds_ratio": -0.42399975657463074, + "logits/chosen": -0.3849475681781769, + "logits/rejected": -1.8075506687164307, + "logps/chosen": -1.4811009168624878, + "logps/rejected": -2.028481960296631, + "loss": 1.5873, + "nll_loss": 1.5449340343475342, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14811009168624878, + "rewards/margins": 0.05473810061812401, + "rewards/rejected": -0.2028481811285019, + "step": 453 + }, + { + "epoch": 1.2512944425267518, + "grad_norm": 0.20314469933509827, + "learning_rate": 3.6278296656008366e-06, + "log_odds_chosen": 0.8694459199905396, + "log_odds_ratio": -0.371795654296875, + "logits/chosen": -0.4006415903568268, + "logits/rejected": -1.7445772886276245, + "logps/chosen": -1.3579434156417847, + "logps/rejected": -2.065506935119629, + "loss": 1.4782, + "nll_loss": 1.4409888982772827, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13579432666301727, + "rewards/margins": 0.07075636833906174, + "rewards/rejected": -0.2065506875514984, + "step": 454 + }, + { + "epoch": 1.2540559199171557, + "grad_norm": 0.1716334968805313, + "learning_rate": 3.6206714916579925e-06, + "log_odds_chosen": 0.7703570127487183, + "log_odds_ratio": -0.3888989984989166, + "logits/chosen": -0.38477808237075806, + "logits/rejected": -1.7767083644866943, + "logps/chosen": -1.398410439491272, + "logps/rejected": -2.027221441268921, + "loss": 1.4987, + "nll_loss": 1.4598515033721924, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13984103500843048, + "rewards/margins": 0.06288108229637146, + "rewards/rejected": -0.20272211730480194, + "step": 455 + }, + { + "epoch": 1.2568173973075596, + "grad_norm": 0.18591812252998352, + "learning_rate": 3.613501801081648e-06, + "log_odds_chosen": 0.617323637008667, + "log_odds_ratio": -0.4372296929359436, + "logits/chosen": -0.36923906207084656, + "logits/rejected": -1.4284781217575073, + "logps/chosen": -1.4494431018829346, + "logps/rejected": -1.9530479907989502, + "loss": 1.5618, + "nll_loss": 1.5181188583374023, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14494431018829346, + "rewards/margins": 0.05036048963665962, + "rewards/rejected": -0.19530481100082397, + "step": 456 + }, + { + "epoch": 1.2595788746979635, + "grad_norm": 0.1984872668981552, + "learning_rate": 3.606320667551466e-06, + "log_odds_chosen": 0.7281745672225952, + "log_odds_ratio": -0.4030551612377167, + "logits/chosen": -0.36235833168029785, + "logits/rejected": -1.7550745010375977, + "logps/chosen": -1.49772310256958, + "logps/rejected": -2.1043448448181152, + "loss": 1.6083, + "nll_loss": 1.5679997205734253, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1497723013162613, + "rewards/margins": 0.06066218018531799, + "rewards/rejected": -0.21043448150157928, + "step": 457 + }, + { + "epoch": 1.2623403520883674, + "grad_norm": 0.1827681064605713, + "learning_rate": 3.599128164864706e-06, + "log_odds_chosen": 0.8096928596496582, + "log_odds_ratio": -0.37837105989456177, + "logits/chosen": -0.45445504784584045, + "logits/rejected": -1.770817756652832, + "logps/chosen": -1.4516648054122925, + "logps/rejected": -2.1250061988830566, + "loss": 1.5451, + "nll_loss": 1.5072699785232544, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14516648650169373, + "rewards/margins": 0.0673341453075409, + "rewards/rejected": -0.21250061690807343, + "step": 458 + }, + { + "epoch": 1.265101829478771, + "grad_norm": 0.19308075308799744, + "learning_rate": 3.5919243669354585e-06, + "log_odds_chosen": 0.6114473342895508, + "log_odds_ratio": -0.4423988461494446, + "logits/chosen": -0.405925452709198, + "logits/rejected": -1.2670363187789917, + "logps/chosen": -1.5176535844802856, + "logps/rejected": -2.0253612995147705, + "loss": 1.623, + "nll_loss": 1.5787138938903809, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1517653614282608, + "rewards/margins": 0.05077076703310013, + "rewards/rejected": -0.20253612101078033, + "step": 459 + }, + { + "epoch": 1.2678633068691751, + "grad_norm": 0.2073579728603363, + "learning_rate": 3.5847093477938955e-06, + "log_odds_chosen": 0.6397342681884766, + "log_odds_ratio": -0.4318113625049591, + "logits/chosen": -0.4677776098251343, + "logits/rejected": -1.5898178815841675, + "logps/chosen": -1.5188252925872803, + "logps/rejected": -2.048469066619873, + "loss": 1.6166, + "nll_loss": 1.5734236240386963, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15188252925872803, + "rewards/margins": 0.05296438932418823, + "rewards/rejected": -0.20484691858291626, + "step": 460 + }, + { + "epoch": 1.2706247842595788, + "grad_norm": 0.18931885063648224, + "learning_rate": 3.5774831815855017e-06, + "log_odds_chosen": 0.8001710772514343, + "log_odds_ratio": -0.3749019205570221, + "logits/chosen": -0.4591723084449768, + "logits/rejected": -1.6894080638885498, + "logps/chosen": -1.4038267135620117, + "logps/rejected": -2.0569498538970947, + "loss": 1.5196, + "nll_loss": 1.4821062088012695, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14038267731666565, + "rewards/margins": 0.06531231105327606, + "rewards/rejected": -0.2056949883699417, + "step": 461 + }, + { + "epoch": 1.2733862616499827, + "grad_norm": 0.17583203315734863, + "learning_rate": 3.5702459425703146e-06, + "log_odds_chosen": 0.8868230581283569, + "log_odds_ratio": -0.3547811210155487, + "logits/chosen": -0.43302029371261597, + "logits/rejected": -1.9412630796432495, + "logps/chosen": -1.508098840713501, + "logps/rejected": -2.258622884750366, + "loss": 1.6074, + "nll_loss": 1.5718796253204346, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1508098840713501, + "rewards/margins": 0.0750524029135704, + "rewards/rejected": -0.2258622944355011, + "step": 462 + }, + { + "epoch": 1.2761477390403866, + "grad_norm": 0.18731163442134857, + "learning_rate": 3.562997705122162e-06, + "log_odds_chosen": 0.7847945094108582, + "log_odds_ratio": -0.37928110361099243, + "logits/chosen": -0.38437145948410034, + "logits/rejected": -1.6156420707702637, + "logps/chosen": -1.4260213375091553, + "logps/rejected": -2.070751428604126, + "loss": 1.5279, + "nll_loss": 1.489925503730774, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14260214567184448, + "rewards/margins": 0.0644729733467102, + "rewards/rejected": -0.20707513391971588, + "step": 463 + }, + { + "epoch": 1.2789092164307905, + "grad_norm": 0.17475299537181854, + "learning_rate": 3.5557385437279e-06, + "log_odds_chosen": 0.838699221611023, + "log_odds_ratio": -0.36731863021850586, + "logits/chosen": -0.3592332601547241, + "logits/rejected": -1.4209747314453125, + "logps/chosen": -1.4949580430984497, + "logps/rejected": -2.196079730987549, + "loss": 1.5793, + "nll_loss": 1.5425434112548828, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14949579536914825, + "rewards/margins": 0.07011217623949051, + "rewards/rejected": -0.21960797905921936, + "step": 464 + }, + { + "epoch": 1.2816706938211944, + "grad_norm": 0.19598983228206635, + "learning_rate": 3.5484685329866424e-06, + "log_odds_chosen": 0.5413783192634583, + "log_odds_ratio": -0.4634351134300232, + "logits/chosen": -0.3708513081073761, + "logits/rejected": -1.4379222393035889, + "logps/chosen": -1.5153756141662598, + "logps/rejected": -1.9605016708374023, + "loss": 1.6065, + "nll_loss": 1.5601321458816528, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15153755247592926, + "rewards/margins": 0.044512614607810974, + "rewards/rejected": -0.19605018198490143, + "step": 465 + }, + { + "epoch": 1.2844321712115983, + "grad_norm": 0.18149885535240173, + "learning_rate": 3.541187747608998e-06, + "log_odds_chosen": 0.9340339303016663, + "log_odds_ratio": -0.34465107321739197, + "logits/chosen": -0.48461854457855225, + "logits/rejected": -1.7797619104385376, + "logps/chosen": -1.5739936828613281, + "logps/rejected": -2.3727049827575684, + "loss": 1.6472, + "nll_loss": 1.6127065420150757, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15739935636520386, + "rewards/margins": 0.07987111061811447, + "rewards/rejected": -0.23727048933506012, + "step": 466 + }, + { + "epoch": 1.2871936486020021, + "grad_norm": 0.19978304207324982, + "learning_rate": 3.533896262416302e-06, + "log_odds_chosen": 0.7202765941619873, + "log_odds_ratio": -0.4014014005661011, + "logits/chosen": -0.4147559404373169, + "logits/rejected": -1.5541059970855713, + "logps/chosen": -1.4875589609146118, + "logps/rejected": -2.0849876403808594, + "loss": 1.5897, + "nll_loss": 1.5495574474334717, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14875589311122894, + "rewards/margins": 0.05974285677075386, + "rewards/rejected": -0.2084987461566925, + "step": 467 + }, + { + "epoch": 1.2899551259924058, + "grad_norm": 0.21298463642597198, + "learning_rate": 3.5265941523398455e-06, + "log_odds_chosen": 0.8375830054283142, + "log_odds_ratio": -0.36393651366233826, + "logits/chosen": -0.42123696208000183, + "logits/rejected": -1.5165983438491821, + "logps/chosen": -1.5110690593719482, + "logps/rejected": -2.2138826847076416, + "loss": 1.6014, + "nll_loss": 1.5650498867034912, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15110690891742706, + "rewards/margins": 0.07028134167194366, + "rewards/rejected": -0.22138825058937073, + "step": 468 + }, + { + "epoch": 1.29271660338281, + "grad_norm": 0.1818659007549286, + "learning_rate": 3.519281492420108e-06, + "log_odds_chosen": 0.5199939012527466, + "log_odds_ratio": -0.47489961981773376, + "logits/chosen": -0.4219110906124115, + "logits/rejected": -1.4790453910827637, + "logps/chosen": -1.5349076986312866, + "logps/rejected": -1.9625968933105469, + "loss": 1.6236, + "nll_loss": 1.5760908126831055, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15349076688289642, + "rewards/margins": 0.042768917977809906, + "rewards/rejected": -0.19625969231128693, + "step": 469 + }, + { + "epoch": 1.2954780807732136, + "grad_norm": 0.19590230286121368, + "learning_rate": 3.5119583578059845e-06, + "log_odds_chosen": 0.6841270327568054, + "log_odds_ratio": -0.43147027492523193, + "logits/chosen": -0.4008702337741852, + "logits/rejected": -1.6819429397583008, + "logps/chosen": -1.6086608171463013, + "logps/rejected": -2.1926984786987305, + "loss": 1.6814, + "nll_loss": 1.6382827758789062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16086608171463013, + "rewards/margins": 0.05840376764535904, + "rewards/rejected": -0.21926987171173096, + "step": 470 + }, + { + "epoch": 1.2982395581636175, + "grad_norm": 0.17711324989795685, + "learning_rate": 3.504624823754014e-06, + "log_odds_chosen": 0.8761448860168457, + "log_odds_ratio": -0.3517749607563019, + "logits/chosen": -0.41212159395217896, + "logits/rejected": -1.957320213317871, + "logps/chosen": -1.4138109683990479, + "logps/rejected": -2.131588935852051, + "loss": 1.508, + "nll_loss": 1.4727998971939087, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14138111472129822, + "rewards/margins": 0.07177779823541641, + "rewards/rejected": -0.21315890550613403, + "step": 471 + }, + { + "epoch": 1.3010010355540214, + "grad_norm": 0.19494958221912384, + "learning_rate": 3.4972809656276047e-06, + "log_odds_chosen": 0.8046627640724182, + "log_odds_ratio": -0.3767206370830536, + "logits/chosen": -0.36885958909988403, + "logits/rejected": -1.8168095350265503, + "logps/chosen": -1.5026159286499023, + "logps/rejected": -2.176860809326172, + "loss": 1.6015, + "nll_loss": 1.5638599395751953, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15026158094406128, + "rewards/margins": 0.06742450594902039, + "rewards/rejected": -0.21768608689308167, + "step": 472 + }, + { + "epoch": 1.3037625129444252, + "grad_norm": 0.1766563355922699, + "learning_rate": 3.4899268588962613e-06, + "log_odds_chosen": 0.8576854467391968, + "log_odds_ratio": -0.3584885597229004, + "logits/chosen": -0.436603844165802, + "logits/rejected": -1.6729352474212646, + "logps/chosen": -1.3645590543746948, + "logps/rejected": -2.0580813884735107, + "loss": 1.4773, + "nll_loss": 1.4414961338043213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13645590841770172, + "rewards/margins": 0.06935223937034607, + "rewards/rejected": -0.20580816268920898, + "step": 473 + }, + { + "epoch": 1.3065239903348291, + "grad_norm": 0.18817074596881866, + "learning_rate": 3.4825625791348093e-06, + "log_odds_chosen": 0.8350617289543152, + "log_odds_ratio": -0.3708168864250183, + "logits/chosen": -0.3747791051864624, + "logits/rejected": -1.553459882736206, + "logps/chosen": -1.4124037027359009, + "logps/rejected": -2.1011507511138916, + "loss": 1.5145, + "nll_loss": 1.477430820465088, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14124037325382233, + "rewards/margins": 0.06887470930814743, + "rewards/rejected": -0.21011507511138916, + "step": 474 + }, + { + "epoch": 1.309285467725233, + "grad_norm": 0.1850864142179489, + "learning_rate": 3.4751882020226174e-06, + "log_odds_chosen": 0.7951568365097046, + "log_odds_ratio": -0.38151904940605164, + "logits/chosen": -0.41285592317581177, + "logits/rejected": -1.676164984703064, + "logps/chosen": -1.519218921661377, + "logps/rejected": -2.190464973449707, + "loss": 1.6106, + "nll_loss": 1.5724141597747803, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15192189812660217, + "rewards/margins": 0.06712460517883301, + "rewards/rejected": -0.21904650330543518, + "step": 475 + }, + { + "epoch": 1.312046945115637, + "grad_norm": 0.19581526517868042, + "learning_rate": 3.467803803342821e-06, + "log_odds_chosen": 0.7770808339118958, + "log_odds_ratio": -0.38217130303382874, + "logits/chosen": -0.3372090756893158, + "logits/rejected": -1.6060659885406494, + "logps/chosen": -1.4900453090667725, + "logps/rejected": -2.1375834941864014, + "loss": 1.5763, + "nll_loss": 1.5380480289459229, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14900454878807068, + "rewards/margins": 0.06475377827882767, + "rewards/rejected": -0.21375833451747894, + "step": 476 + }, + { + "epoch": 1.3148084225060408, + "grad_norm": 0.19793154299259186, + "learning_rate": 3.4604094589815402e-06, + "log_odds_chosen": 0.6940678358078003, + "log_odds_ratio": -0.40892940759658813, + "logits/chosen": -0.42878496646881104, + "logits/rejected": -1.5839377641677856, + "logps/chosen": -1.4118523597717285, + "logps/rejected": -1.9752473831176758, + "loss": 1.5164, + "nll_loss": 1.4755195379257202, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1411852389574051, + "rewards/margins": 0.056339483708143234, + "rewards/rejected": -0.19752474129199982, + "step": 477 + }, + { + "epoch": 1.3175698998964447, + "grad_norm": 0.18727529048919678, + "learning_rate": 3.4530052449271044e-06, + "log_odds_chosen": 0.8616414666175842, + "log_odds_ratio": -0.36362677812576294, + "logits/chosen": -0.4634791910648346, + "logits/rejected": -1.5600810050964355, + "logps/chosen": -1.4240953922271729, + "logps/rejected": -2.1343352794647217, + "loss": 1.521, + "nll_loss": 1.4846872091293335, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.142409548163414, + "rewards/margins": 0.07102398574352264, + "rewards/rejected": -0.21343351900577545, + "step": 478 + }, + { + "epoch": 1.3203313772868484, + "grad_norm": 0.19879932701587677, + "learning_rate": 3.4455912372692696e-06, + "log_odds_chosen": 0.5470461845397949, + "log_odds_ratio": -0.46493563055992126, + "logits/chosen": -0.3689558207988739, + "logits/rejected": -1.5988726615905762, + "logps/chosen": -1.4872140884399414, + "logps/rejected": -1.9357192516326904, + "loss": 1.5782, + "nll_loss": 1.5316959619522095, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14872139692306519, + "rewards/margins": 0.04485052451491356, + "rewards/rejected": -0.19357194006443024, + "step": 479 + }, + { + "epoch": 1.3230928546772525, + "grad_norm": 0.19459912180900574, + "learning_rate": 3.438167512198436e-06, + "log_odds_chosen": 0.5693470239639282, + "log_odds_ratio": -0.4518805146217346, + "logits/chosen": -0.41612759232521057, + "logits/rejected": -1.5004498958587646, + "logps/chosen": -1.5251268148422241, + "logps/rejected": -1.994539499282837, + "loss": 1.6246, + "nll_loss": 1.5794237852096558, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15251268446445465, + "rewards/margins": 0.046941258013248444, + "rewards/rejected": -0.1994539499282837, + "step": 480 + }, + { + "epoch": 1.3258543320676561, + "grad_norm": 0.1880672574043274, + "learning_rate": 3.4307341460048633e-06, + "log_odds_chosen": 0.7143114805221558, + "log_odds_ratio": -0.40364253520965576, + "logits/chosen": -0.4024220108985901, + "logits/rejected": -1.5502820014953613, + "logps/chosen": -1.5507985353469849, + "logps/rejected": -2.1468398571014404, + "loss": 1.6399, + "nll_loss": 1.5995802879333496, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15507985651493073, + "rewards/margins": 0.059604134410619736, + "rewards/rejected": -0.21468399465084076, + "step": 481 + }, + { + "epoch": 1.32861580945806, + "grad_norm": 0.19784514605998993, + "learning_rate": 3.4232912150778914e-06, + "log_odds_chosen": 0.7461116909980774, + "log_odds_ratio": -0.3941587507724762, + "logits/chosen": -0.43998983502388, + "logits/rejected": -1.7154864072799683, + "logps/chosen": -1.4538161754608154, + "logps/rejected": -2.0666849613189697, + "loss": 1.5731, + "nll_loss": 1.5336874723434448, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1453816145658493, + "rewards/margins": 0.06128688156604767, + "rewards/rejected": -0.20666849613189697, + "step": 482 + }, + { + "epoch": 1.331377286848464, + "grad_norm": 0.18405503034591675, + "learning_rate": 3.415838795905151e-06, + "log_odds_chosen": 0.9300730228424072, + "log_odds_ratio": -0.34047600626945496, + "logits/chosen": -0.5350120067596436, + "logits/rejected": -1.9728281497955322, + "logps/chosen": -1.4228546619415283, + "logps/rejected": -2.1867175102233887, + "loss": 1.5168, + "nll_loss": 1.4828009605407715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14228546619415283, + "rewards/margins": 0.07638627290725708, + "rewards/rejected": -0.2186717540025711, + "step": 483 + }, + { + "epoch": 1.3341387642388678, + "grad_norm": 0.19047723710536957, + "learning_rate": 3.408376965071779e-06, + "log_odds_chosen": 0.6742444634437561, + "log_odds_ratio": -0.41640961170196533, + "logits/chosen": -0.37510067224502563, + "logits/rejected": -1.5923813581466675, + "logps/chosen": -1.5178049802780151, + "logps/rejected": -2.0749661922454834, + "loss": 1.6055, + "nll_loss": 1.5638338327407837, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15178050100803375, + "rewards/margins": 0.05571611970663071, + "rewards/rejected": -0.20749662816524506, + "step": 484 + }, + { + "epoch": 1.3369002416292717, + "grad_norm": 0.1780618578195572, + "learning_rate": 3.400905799259634e-06, + "log_odds_chosen": 0.653221607208252, + "log_odds_ratio": -0.4260196387767792, + "logits/chosen": -0.42139092087745667, + "logits/rejected": -1.648254632949829, + "logps/chosen": -1.4374747276306152, + "logps/rejected": -1.9686298370361328, + "loss": 1.5329, + "nll_loss": 1.490256667137146, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.143747478723526, + "rewards/margins": 0.05311552435159683, + "rewards/rejected": -0.19686299562454224, + "step": 485 + }, + { + "epoch": 1.3396617190196756, + "grad_norm": 0.1804792881011963, + "learning_rate": 3.393425375246503e-06, + "log_odds_chosen": 0.6369755268096924, + "log_odds_ratio": -0.4262813925743103, + "logits/chosen": -0.4290909469127655, + "logits/rejected": -1.7290550470352173, + "logps/chosen": -1.4642930030822754, + "logps/rejected": -1.98381507396698, + "loss": 1.5519, + "nll_loss": 1.509236454963684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14642930030822754, + "rewards/margins": 0.05195220559835434, + "rewards/rejected": -0.1983814835548401, + "step": 486 + }, + { + "epoch": 1.3424231964100795, + "grad_norm": 0.19493341445922852, + "learning_rate": 3.3859357699053165e-06, + "log_odds_chosen": 0.7284951210021973, + "log_odds_ratio": -0.40332838892936707, + "logits/chosen": -0.40109413862228394, + "logits/rejected": -1.4352595806121826, + "logps/chosen": -1.402358055114746, + "logps/rejected": -1.9987090826034546, + "loss": 1.4967, + "nll_loss": 1.456404447555542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1402358114719391, + "rewards/margins": 0.059635113924741745, + "rewards/rejected": -0.19987091422080994, + "step": 487 + }, + { + "epoch": 1.3451846738004831, + "grad_norm": 0.19050319492816925, + "learning_rate": 3.3784370602033572e-06, + "log_odds_chosen": 0.8729457259178162, + "log_odds_ratio": -0.35851162672042847, + "logits/chosen": -0.4902225732803345, + "logits/rejected": -1.7542881965637207, + "logps/chosen": -1.4271174669265747, + "logps/rejected": -2.151813507080078, + "loss": 1.5277, + "nll_loss": 1.4917997121810913, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14271175861358643, + "rewards/margins": 0.07246959954500198, + "rewards/rejected": -0.2151813805103302, + "step": 488 + }, + { + "epoch": 1.3479461511908872, + "grad_norm": 0.18820421397686005, + "learning_rate": 3.3709293232014705e-06, + "log_odds_chosen": 0.73007732629776, + "log_odds_ratio": -0.40136751532554626, + "logits/chosen": -0.44850000739097595, + "logits/rejected": -2.0052490234375, + "logps/chosen": -1.4974483251571655, + "logps/rejected": -2.1092348098754883, + "loss": 1.5884, + "nll_loss": 1.5482368469238281, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14974482357501984, + "rewards/margins": 0.06117865815758705, + "rewards/rejected": -0.21092349290847778, + "step": 489 + }, + { + "epoch": 1.350707628581291, + "grad_norm": 0.18799753487110138, + "learning_rate": 3.3634126360532694e-06, + "log_odds_chosen": 0.8482156991958618, + "log_odds_ratio": -0.36712414026260376, + "logits/chosen": -0.32226884365081787, + "logits/rejected": -1.778417944908142, + "logps/chosen": -1.577460765838623, + "logps/rejected": -2.302645206451416, + "loss": 1.6644, + "nll_loss": 1.6277295351028442, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1577460616827011, + "rewards/margins": 0.07251846790313721, + "rewards/rejected": -0.23026452958583832, + "step": 490 + }, + { + "epoch": 1.3534691059716948, + "grad_norm": 0.20365531742572784, + "learning_rate": 3.355887076004345e-06, + "log_odds_chosen": 0.7709956765174866, + "log_odds_ratio": -0.383152574300766, + "logits/chosen": -0.5297287702560425, + "logits/rejected": -1.6290040016174316, + "logps/chosen": -1.4873484373092651, + "logps/rejected": -2.12845778465271, + "loss": 1.5899, + "nll_loss": 1.551561713218689, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14873485267162323, + "rewards/margins": 0.06411094218492508, + "rewards/rejected": -0.21284577250480652, + "step": 491 + }, + { + "epoch": 1.3562305833620987, + "grad_norm": 0.18781210482120514, + "learning_rate": 3.3483527203914694e-06, + "log_odds_chosen": 0.8746954798698425, + "log_odds_ratio": -0.3559949994087219, + "logits/chosen": -0.3871724307537079, + "logits/rejected": -1.6837961673736572, + "logps/chosen": -1.5181317329406738, + "logps/rejected": -2.254382610321045, + "loss": 1.613, + "nll_loss": 1.5773670673370361, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15181316435337067, + "rewards/margins": 0.0736251100897789, + "rewards/rejected": -0.22543828189373016, + "step": 492 + }, + { + "epoch": 1.3589920607525026, + "grad_norm": 0.18413974344730377, + "learning_rate": 3.340809646641805e-06, + "log_odds_chosen": 0.6966791749000549, + "log_odds_ratio": -0.4073706269264221, + "logits/chosen": -0.32745176553726196, + "logits/rejected": -1.734323501586914, + "logps/chosen": -1.5864430665969849, + "logps/rejected": -2.1757755279541016, + "loss": 1.6634, + "nll_loss": 1.6226191520690918, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15864431858062744, + "rewards/margins": 0.05893322825431824, + "rewards/rejected": -0.21757756173610687, + "step": 493 + }, + { + "epoch": 1.3617535381429065, + "grad_norm": 0.1795225441455841, + "learning_rate": 3.333257932272105e-06, + "log_odds_chosen": 0.7916382551193237, + "log_odds_ratio": -0.3854186236858368, + "logits/chosen": -0.3654117286205292, + "logits/rejected": -1.731317400932312, + "logps/chosen": -1.4387598037719727, + "logps/rejected": -2.0977869033813477, + "loss": 1.5401, + "nll_loss": 1.5015188455581665, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14387598633766174, + "rewards/margins": 0.0659027174115181, + "rewards/rejected": -0.20977871119976044, + "step": 494 + }, + { + "epoch": 1.3645150155333103, + "grad_norm": 0.19810736179351807, + "learning_rate": 3.3256976548879183e-06, + "log_odds_chosen": 0.7499178647994995, + "log_odds_ratio": -0.391695499420166, + "logits/chosen": -0.4586635231971741, + "logits/rejected": -1.5185637474060059, + "logps/chosen": -1.494145393371582, + "logps/rejected": -2.1175270080566406, + "loss": 1.591, + "nll_loss": 1.551873803138733, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1494145393371582, + "rewards/margins": 0.06233816593885422, + "rewards/rejected": -0.21175269782543182, + "step": 495 + }, + { + "epoch": 1.3672764929237142, + "grad_norm": 0.17777174711227417, + "learning_rate": 3.3181288921827925e-06, + "log_odds_chosen": 0.8323229551315308, + "log_odds_ratio": -0.37165793776512146, + "logits/chosen": -0.3486481308937073, + "logits/rejected": -1.4729896783828735, + "logps/chosen": -1.3969337940216064, + "logps/rejected": -2.0784289836883545, + "loss": 1.5068, + "nll_loss": 1.4696803092956543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13969337940216064, + "rewards/margins": 0.06814949959516525, + "rewards/rejected": -0.2078428864479065, + "step": 496 + }, + { + "epoch": 1.3700379703141181, + "grad_norm": 0.18033576011657715, + "learning_rate": 3.310551721937475e-06, + "log_odds_chosen": 0.7983404994010925, + "log_odds_ratio": -0.3745984435081482, + "logits/chosen": -0.3823201656341553, + "logits/rejected": -1.5644809007644653, + "logps/chosen": -1.4277198314666748, + "logps/rejected": -2.0840868949890137, + "loss": 1.5007, + "nll_loss": 1.4631915092468262, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14277197420597076, + "rewards/margins": 0.06563669443130493, + "rewards/rejected": -0.2084086835384369, + "step": 497 + }, + { + "epoch": 1.372799447704522, + "grad_norm": 0.19242247939109802, + "learning_rate": 3.3029662220191146e-06, + "log_odds_chosen": 0.8101353645324707, + "log_odds_ratio": -0.37228965759277344, + "logits/chosen": -0.4509270191192627, + "logits/rejected": -1.8056501150131226, + "logps/chosen": -1.3381811380386353, + "logps/rejected": -1.9927046298980713, + "loss": 1.4395, + "nll_loss": 1.4023054838180542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.133818119764328, + "rewards/margins": 0.06545236706733704, + "rewards/rejected": -0.19927047193050385, + "step": 498 + }, + { + "epoch": 1.3755609250949257, + "grad_norm": 0.1753472536802292, + "learning_rate": 3.2953724703804572e-06, + "log_odds_chosen": 0.892001211643219, + "log_odds_ratio": -0.35052934288978577, + "logits/chosen": -0.4044141173362732, + "logits/rejected": -1.8937467336654663, + "logps/chosen": -1.407699704170227, + "logps/rejected": -2.1476714611053467, + "loss": 1.4972, + "nll_loss": 1.4621860980987549, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14076997339725494, + "rewards/margins": 0.07399718463420868, + "rewards/rejected": -0.21476714313030243, + "step": 499 + }, + { + "epoch": 1.3783224024853298, + "grad_norm": 0.1958237588405609, + "learning_rate": 3.2877705450590525e-06, + "log_odds_chosen": 0.7319251298904419, + "log_odds_ratio": -0.396656334400177, + "logits/chosen": -0.3846544325351715, + "logits/rejected": -1.6542454957962036, + "logps/chosen": -1.4686310291290283, + "logps/rejected": -2.0715951919555664, + "loss": 1.5948, + "nll_loss": 1.5551836490631104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14686310291290283, + "rewards/margins": 0.06029640883207321, + "rewards/rejected": -0.20715951919555664, + "step": 500 + } + ], + "logging_steps": 1, + "max_steps": 1089, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}