diff --git "a/run_20251219_040313/checkpoint-1089/trainer_state.json" "b/run_20251219_040313/checkpoint-1089/trainer_state.json" new file mode 100644--- /dev/null +++ "b/run_20251219_040313/checkpoint-1089/trainer_state.json" @@ -0,0 +1,19636 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1089, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.002761477390403866, + "grad_norm": 0.5198726654052734, + "learning_rate": 0.0, + "log_odds_chosen": 0.4317269027233124, + "log_odds_ratio": -0.5042418837547302, + "logits/chosen": -0.5456271767616272, + "logits/rejected": -0.10779725015163422, + "logps/chosen": -1.998489260673523, + "logps/rejected": -2.3799774646759033, + "loss": 2.1834, + "nll_loss": 2.1329703330993652, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.199848935008049, + "rewards/margins": 0.03814878687262535, + "rewards/rejected": -0.23799772560596466, + "step": 1 + }, + { + "epoch": 0.005522954780807732, + "grad_norm": 0.7381364107131958, + "learning_rate": 4.587155963302753e-08, + "log_odds_chosen": 0.4411306381225586, + "log_odds_ratio": -0.4988963305950165, + "logits/chosen": -0.6594648361206055, + "logits/rejected": -0.07251911610364914, + "logps/chosen": -2.0794589519500732, + "logps/rejected": -2.473759889602661, + "loss": 2.2463, + "nll_loss": 2.196385383605957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20794589817523956, + "rewards/margins": 0.039430104196071625, + "rewards/rejected": -0.24737600982189178, + "step": 2 + }, + { + "epoch": 0.008284432171211598, + "grad_norm": 0.6377604603767395, + "learning_rate": 9.174311926605506e-08, + "log_odds_chosen": 0.5233978033065796, + "log_odds_ratio": -0.4696895480155945, + "logits/chosen": -0.7436237931251526, + "logits/rejected": -0.04083387181162834, + "logps/chosen": -2.0129284858703613, + "logps/rejected": -2.4808387756347656, + "loss": 2.1678, + "nll_loss": 2.120811700820923, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20129284262657166, + "rewards/margins": 0.04679100960493088, + "rewards/rejected": -0.24808385968208313, + "step": 3 + }, + { + "epoch": 0.011045909561615464, + "grad_norm": 0.5183601379394531, + "learning_rate": 1.376146788990826e-07, + "log_odds_chosen": 0.4738181531429291, + "log_odds_ratio": -0.4919201731681824, + "logits/chosen": -0.5707842111587524, + "logits/rejected": -0.06899966299533844, + "logps/chosen": -2.006629467010498, + "logps/rejected": -2.429466724395752, + "loss": 2.1541, + "nll_loss": 2.104935646057129, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20066295564174652, + "rewards/margins": 0.04228372871875763, + "rewards/rejected": -0.24294668436050415, + "step": 4 + }, + { + "epoch": 0.013807386952019331, + "grad_norm": 0.5362528562545776, + "learning_rate": 1.8348623853211012e-07, + "log_odds_chosen": 0.47211968898773193, + "log_odds_ratio": -0.49660325050354004, + "logits/chosen": -0.5831207633018494, + "logits/rejected": -0.13220791518688202, + "logps/chosen": -1.9650582075119019, + "logps/rejected": -2.3817272186279297, + "loss": 2.1187, + "nll_loss": 2.0690417289733887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1965058296918869, + "rewards/margins": 0.04166688770055771, + "rewards/rejected": -0.23817269504070282, + "step": 5 + }, + { + "epoch": 0.016568864342423197, + "grad_norm": 0.5392005443572998, + "learning_rate": 2.2935779816513764e-07, + "log_odds_chosen": 0.3942825496196747, + "log_odds_ratio": -0.518440306186676, + "logits/chosen": -0.5348646640777588, + "logits/rejected": -0.23154743015766144, + "logps/chosen": -2.012246608734131, + "logps/rejected": -2.361220359802246, + "loss": 2.178, + "nll_loss": 2.126155376434326, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2012246698141098, + "rewards/margins": 0.03489736467599869, + "rewards/rejected": -0.2361220419406891, + "step": 6 + }, + { + "epoch": 0.019330341732827064, + "grad_norm": 0.5474534630775452, + "learning_rate": 2.752293577981652e-07, + "log_odds_chosen": 0.2622841000556946, + "log_odds_ratio": -0.576733410358429, + "logits/chosen": -0.5426779985427856, + "logits/rejected": -0.2341059297323227, + "logps/chosen": -2.074805736541748, + "logps/rejected": -2.310166120529175, + "loss": 2.2374, + "nll_loss": 2.179717779159546, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2074805647134781, + "rewards/margins": 0.023536043241620064, + "rewards/rejected": -0.231016606092453, + "step": 7 + }, + { + "epoch": 0.022091819123230928, + "grad_norm": 0.5940824747085571, + "learning_rate": 3.211009174311927e-07, + "log_odds_chosen": 0.41454389691352844, + "log_odds_ratio": -0.5088675618171692, + "logits/chosen": -0.45266416668891907, + "logits/rejected": -0.03741999715566635, + "logps/chosen": -2.081907033920288, + "logps/rejected": -2.452606439590454, + "loss": 2.2474, + "nll_loss": 2.19650936126709, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2081906944513321, + "rewards/margins": 0.037069931626319885, + "rewards/rejected": -0.24526062607765198, + "step": 8 + }, + { + "epoch": 0.024853296513634795, + "grad_norm": 0.5987945795059204, + "learning_rate": 3.6697247706422023e-07, + "log_odds_chosen": 0.666429340839386, + "log_odds_ratio": -0.42349112033843994, + "logits/chosen": -0.7504861950874329, + "logits/rejected": -0.16368302702903748, + "logps/chosen": -1.9702867269515991, + "logps/rejected": -2.566361427307129, + "loss": 2.1456, + "nll_loss": 2.10321307182312, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19702866673469543, + "rewards/margins": 0.05960746854543686, + "rewards/rejected": -0.2566361427307129, + "step": 9 + }, + { + "epoch": 0.027614773904038662, + "grad_norm": 0.7403894662857056, + "learning_rate": 4.128440366972478e-07, + "log_odds_chosen": 0.44253280758857727, + "log_odds_ratio": -0.502151370048523, + "logits/chosen": -0.7380926012992859, + "logits/rejected": -0.23479697108268738, + "logps/chosen": -2.0925681591033936, + "logps/rejected": -2.4903366565704346, + "loss": 2.2572, + "nll_loss": 2.206953287124634, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20925681293010712, + "rewards/margins": 0.03977686166763306, + "rewards/rejected": -0.24903367459774017, + "step": 10 + }, + { + "epoch": 0.030376251294442526, + "grad_norm": 0.6613215208053589, + "learning_rate": 4.587155963302753e-07, + "log_odds_chosen": 0.5335031151771545, + "log_odds_ratio": -0.4685918688774109, + "logits/chosen": -0.6707795262336731, + "logits/rejected": -0.13829460740089417, + "logps/chosen": -2.0549967288970947, + "logps/rejected": -2.5354185104370117, + "loss": 2.2247, + "nll_loss": 2.1778128147125244, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20549967885017395, + "rewards/margins": 0.04804220795631409, + "rewards/rejected": -0.25354188680648804, + "step": 11 + }, + { + "epoch": 0.03313772868484639, + "grad_norm": 0.5580596327781677, + "learning_rate": 5.045871559633028e-07, + "log_odds_chosen": 0.4023962616920471, + "log_odds_ratio": -0.5220973491668701, + "logits/chosen": -0.5904384851455688, + "logits/rejected": 0.0007063774392008781, + "logps/chosen": -1.958735704421997, + "logps/rejected": -2.3157620429992676, + "loss": 2.1455, + "nll_loss": 2.09328293800354, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19587357342243195, + "rewards/margins": 0.03570263832807541, + "rewards/rejected": -0.23157618939876556, + "step": 12 + }, + { + "epoch": 0.03589920607525026, + "grad_norm": 0.4629175662994385, + "learning_rate": 5.504587155963304e-07, + "log_odds_chosen": 0.4922281503677368, + "log_odds_ratio": -0.4793952405452728, + "logits/chosen": -0.489984929561615, + "logits/rejected": -0.08446945250034332, + "logps/chosen": -1.8884625434875488, + "logps/rejected": -2.318817377090454, + "loss": 2.0661, + "nll_loss": 2.0181963443756104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18884626030921936, + "rewards/margins": 0.043035492300987244, + "rewards/rejected": -0.2318817526102066, + "step": 13 + }, + { + "epoch": 0.03866068346565413, + "grad_norm": 0.6970926523208618, + "learning_rate": 5.963302752293579e-07, + "log_odds_chosen": 0.46570533514022827, + "log_odds_ratio": -0.49952778220176697, + "logits/chosen": -0.6894131898880005, + "logits/rejected": -0.010050175711512566, + "logps/chosen": -2.0864779949188232, + "logps/rejected": -2.503685235977173, + "loss": 2.2329, + "nll_loss": 2.182929277420044, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20864778757095337, + "rewards/margins": 0.041720740497112274, + "rewards/rejected": -0.25036853551864624, + "step": 14 + }, + { + "epoch": 0.04142216085605799, + "grad_norm": 0.7035249471664429, + "learning_rate": 6.422018348623854e-07, + "log_odds_chosen": 0.5717816948890686, + "log_odds_ratio": -0.44831815361976624, + "logits/chosen": -0.7732019424438477, + "logits/rejected": 0.13750173151493073, + "logps/chosen": -2.0055058002471924, + "logps/rejected": -2.513883590698242, + "loss": 2.1667, + "nll_loss": 2.1218552589416504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20055057108402252, + "rewards/margins": 0.05083777755498886, + "rewards/rejected": -0.2513883411884308, + "step": 15 + }, + { + "epoch": 0.044183638246461855, + "grad_norm": 0.489059180021286, + "learning_rate": 6.880733944954129e-07, + "log_odds_chosen": 0.45178472995758057, + "log_odds_ratio": -0.501604437828064, + "logits/chosen": -0.6046928763389587, + "logits/rejected": 0.05104057490825653, + "logps/chosen": -1.8943061828613281, + "logps/rejected": -2.2864151000976562, + "loss": 2.0731, + "nll_loss": 2.022980213165283, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18943063914775848, + "rewards/margins": 0.03921087831258774, + "rewards/rejected": -0.22864152491092682, + "step": 16 + }, + { + "epoch": 0.04694511563686572, + "grad_norm": 0.5798972249031067, + "learning_rate": 7.339449541284405e-07, + "log_odds_chosen": 0.521186113357544, + "log_odds_ratio": -0.4737667441368103, + "logits/chosen": -0.5957604646682739, + "logits/rejected": 0.05874314904212952, + "logps/chosen": -1.9864083528518677, + "logps/rejected": -2.4512085914611816, + "loss": 2.1331, + "nll_loss": 2.085712194442749, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1986408233642578, + "rewards/margins": 0.04648003727197647, + "rewards/rejected": -0.24512089788913727, + "step": 17 + }, + { + "epoch": 0.04970659302726959, + "grad_norm": 0.602294921875, + "learning_rate": 7.79816513761468e-07, + "log_odds_chosen": 0.6343832612037659, + "log_odds_ratio": -0.42983177304267883, + "logits/chosen": -0.6467751264572144, + "logits/rejected": -0.0914345234632492, + "logps/chosen": -2.020120143890381, + "logps/rejected": -2.5913381576538086, + "loss": 2.1654, + "nll_loss": 2.1224629878997803, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2020120471715927, + "rewards/margins": 0.05712177976965904, + "rewards/rejected": -0.25913381576538086, + "step": 18 + }, + { + "epoch": 0.05246807041767346, + "grad_norm": 0.6167095303535461, + "learning_rate": 8.256880733944956e-07, + "log_odds_chosen": 0.23837895691394806, + "log_odds_ratio": -0.5851647853851318, + "logits/chosen": -0.6367707848548889, + "logits/rejected": -0.06072646751999855, + "logps/chosen": -2.055112838745117, + "logps/rejected": -2.2656140327453613, + "loss": 2.2217, + "nll_loss": 2.163205146789551, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20551128685474396, + "rewards/margins": 0.021050114184617996, + "rewards/rejected": -0.22656141221523285, + "step": 19 + }, + { + "epoch": 0.055229547808077324, + "grad_norm": 0.5270997285842896, + "learning_rate": 8.71559633027523e-07, + "log_odds_chosen": 0.4399060010910034, + "log_odds_ratio": -0.5041620135307312, + "logits/chosen": -0.5500829815864563, + "logits/rejected": 0.014720816165208817, + "logps/chosen": -1.9720858335494995, + "logps/rejected": -2.3600573539733887, + "loss": 2.1571, + "nll_loss": 2.1066458225250244, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19720861315727234, + "rewards/margins": 0.038797151297330856, + "rewards/rejected": -0.2360057532787323, + "step": 20 + }, + { + "epoch": 0.057991025198481184, + "grad_norm": 0.639258086681366, + "learning_rate": 9.174311926605506e-07, + "log_odds_chosen": 0.35250768065452576, + "log_odds_ratio": -0.5405789017677307, + "logits/chosen": -0.45625555515289307, + "logits/rejected": -0.03890611231327057, + "logps/chosen": -2.1359739303588867, + "logps/rejected": -2.454291820526123, + "loss": 2.2807, + "nll_loss": 2.2266433238983154, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.21359741687774658, + "rewards/margins": 0.0318317785859108, + "rewards/rejected": -0.2454291731119156, + "step": 21 + }, + { + "epoch": 0.06075250258888505, + "grad_norm": 0.5805932283401489, + "learning_rate": 9.633027522935782e-07, + "log_odds_chosen": 0.5011261105537415, + "log_odds_ratio": -0.4887810945510864, + "logits/chosen": -0.6073933839797974, + "logits/rejected": -0.18206848204135895, + "logps/chosen": -1.9807653427124023, + "logps/rejected": -2.4255237579345703, + "loss": 2.1351, + "nll_loss": 2.0862059593200684, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1980765163898468, + "rewards/margins": 0.044475845992565155, + "rewards/rejected": -0.24255238473415375, + "step": 22 + }, + { + "epoch": 0.06351397997928893, + "grad_norm": 0.5500138998031616, + "learning_rate": 1.0091743119266057e-06, + "log_odds_chosen": 0.3974594175815582, + "log_odds_ratio": -0.5229506492614746, + "logits/chosen": -0.5682663321495056, + "logits/rejected": -0.3609941005706787, + "logps/chosen": -1.963416576385498, + "logps/rejected": -2.318310499191284, + "loss": 2.1453, + "nll_loss": 2.0929956436157227, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19634166359901428, + "rewards/margins": 0.03548937663435936, + "rewards/rejected": -0.23183104395866394, + "step": 23 + }, + { + "epoch": 0.06627545736969279, + "grad_norm": 0.5702619552612305, + "learning_rate": 1.055045871559633e-06, + "log_odds_chosen": 0.522091805934906, + "log_odds_ratio": -0.4669547379016876, + "logits/chosen": -0.5776705741882324, + "logits/rejected": 0.03815801814198494, + "logps/chosen": -1.9660505056381226, + "logps/rejected": -2.429518461227417, + "loss": 2.1353, + "nll_loss": 2.0886194705963135, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19660505652427673, + "rewards/margins": 0.04634679853916168, + "rewards/rejected": -0.24295185506343842, + "step": 24 + }, + { + "epoch": 0.06903693476009665, + "grad_norm": 0.6908774375915527, + "learning_rate": 1.1009174311926608e-06, + "log_odds_chosen": 0.6555817723274231, + "log_odds_ratio": -0.42638978362083435, + "logits/chosen": -0.6369448304176331, + "logits/rejected": -0.0680353045463562, + "logps/chosen": -1.996623158454895, + "logps/rejected": -2.585740327835083, + "loss": 2.157, + "nll_loss": 2.1143739223480225, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19966234266757965, + "rewards/margins": 0.058911725878715515, + "rewards/rejected": -0.25857406854629517, + "step": 25 + }, + { + "epoch": 0.07179841215050052, + "grad_norm": 0.6247107982635498, + "learning_rate": 1.1467889908256882e-06, + "log_odds_chosen": 0.37967437505722046, + "log_odds_ratio": -0.528645396232605, + "logits/chosen": -0.5880488157272339, + "logits/rejected": -0.11361770331859589, + "logps/chosen": -2.0194289684295654, + "logps/rejected": -2.3577256202697754, + "loss": 2.1956, + "nll_loss": 2.142758369445801, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20194292068481445, + "rewards/margins": 0.03382965177297592, + "rewards/rejected": -0.23577255010604858, + "step": 26 + }, + { + "epoch": 0.07455988954090438, + "grad_norm": 0.44772788882255554, + "learning_rate": 1.1926605504587159e-06, + "log_odds_chosen": 0.4675738513469696, + "log_odds_ratio": -0.4964170753955841, + "logits/chosen": -0.434501588344574, + "logits/rejected": -0.17547550797462463, + "logps/chosen": -1.865759253501892, + "logps/rejected": -2.272789239883423, + "loss": 2.0462, + "nll_loss": 1.996600866317749, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18657593429088593, + "rewards/margins": 0.04070296883583069, + "rewards/rejected": -0.2272789031267166, + "step": 27 + }, + { + "epoch": 0.07732136693130826, + "grad_norm": 0.5303645730018616, + "learning_rate": 1.2385321100917433e-06, + "log_odds_chosen": 0.5267918109893799, + "log_odds_ratio": -0.46872708201408386, + "logits/chosen": -0.5293477773666382, + "logits/rejected": -0.20712637901306152, + "logps/chosen": -1.9557033777236938, + "logps/rejected": -2.422588586807251, + "loss": 2.123, + "nll_loss": 2.0761377811431885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19557033479213715, + "rewards/margins": 0.04668852686882019, + "rewards/rejected": -0.24225887656211853, + "step": 28 + }, + { + "epoch": 0.08008284432171212, + "grad_norm": 0.558870792388916, + "learning_rate": 1.2844036697247707e-06, + "log_odds_chosen": 0.4761933982372284, + "log_odds_ratio": -0.49172335863113403, + "logits/chosen": -0.5614610314369202, + "logits/rejected": -0.03958575427532196, + "logps/chosen": -2.044667959213257, + "logps/rejected": -2.4699392318725586, + "loss": 2.2024, + "nll_loss": 2.153231143951416, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2044668048620224, + "rewards/margins": 0.042527101933956146, + "rewards/rejected": -0.24699391424655914, + "step": 29 + }, + { + "epoch": 0.08284432171211598, + "grad_norm": 0.6042240262031555, + "learning_rate": 1.3302752293577984e-06, + "log_odds_chosen": 0.6456740498542786, + "log_odds_ratio": -0.4289007782936096, + "logits/chosen": -0.5478062629699707, + "logits/rejected": -0.0002663079649209976, + "logps/chosen": -2.02502703666687, + "logps/rejected": -2.60587477684021, + "loss": 2.1737, + "nll_loss": 2.1308515071868896, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20250271260738373, + "rewards/margins": 0.058084748685359955, + "rewards/rejected": -0.2605874538421631, + "step": 30 + }, + { + "epoch": 0.08560579910251985, + "grad_norm": 0.6082971692085266, + "learning_rate": 1.3761467889908258e-06, + "log_odds_chosen": 0.5572243928909302, + "log_odds_ratio": -0.45846420526504517, + "logits/chosen": -0.5786283016204834, + "logits/rejected": -0.06976839900016785, + "logps/chosen": -2.0147552490234375, + "logps/rejected": -2.5146052837371826, + "loss": 2.1834, + "nll_loss": 2.1375184059143066, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20147554576396942, + "rewards/margins": 0.049984999001026154, + "rewards/rejected": -0.2514605224132538, + "step": 31 + }, + { + "epoch": 0.08836727649292371, + "grad_norm": 0.4792933762073517, + "learning_rate": 1.4220183486238535e-06, + "log_odds_chosen": 0.43500036001205444, + "log_odds_ratio": -0.5046517252922058, + "logits/chosen": -0.4258999824523926, + "logits/rejected": 0.003258749842643738, + "logps/chosen": -2.011561870574951, + "logps/rejected": -2.398580312728882, + "loss": 2.1831, + "nll_loss": 2.132627010345459, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20115619897842407, + "rewards/margins": 0.03870181739330292, + "rewards/rejected": -0.239858016371727, + "step": 32 + }, + { + "epoch": 0.09112875388332758, + "grad_norm": 0.5385621786117554, + "learning_rate": 1.467889908256881e-06, + "log_odds_chosen": 0.573083758354187, + "log_odds_ratio": -0.4524250328540802, + "logits/chosen": -0.4552776515483856, + "logits/rejected": -0.06403280049562454, + "logps/chosen": -1.9241302013397217, + "logps/rejected": -2.4318630695343018, + "loss": 2.0919, + "nll_loss": 2.046658754348755, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19241301715373993, + "rewards/margins": 0.05077328532934189, + "rewards/rejected": -0.24318630993366241, + "step": 33 + }, + { + "epoch": 0.09389023127373144, + "grad_norm": 0.4870143234729767, + "learning_rate": 1.5137614678899084e-06, + "log_odds_chosen": 0.6046110391616821, + "log_odds_ratio": -0.4510130286216736, + "logits/chosen": -0.40763065218925476, + "logits/rejected": -0.08770006895065308, + "logps/chosen": -1.922775387763977, + "logps/rejected": -2.462465524673462, + "loss": 2.0741, + "nll_loss": 2.028975009918213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19227753579616547, + "rewards/margins": 0.053969040513038635, + "rewards/rejected": -0.2462465912103653, + "step": 34 + }, + { + "epoch": 0.09665170866413532, + "grad_norm": 0.6609561443328857, + "learning_rate": 1.559633027522936e-06, + "log_odds_chosen": 0.40986764430999756, + "log_odds_ratio": -0.5126968026161194, + "logits/chosen": -0.8081121444702148, + "logits/rejected": -0.20409950613975525, + "logps/chosen": -1.9795390367507935, + "logps/rejected": -2.3391332626342773, + "loss": 2.152, + "nll_loss": 2.1007797718048096, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19795390963554382, + "rewards/margins": 0.03595941513776779, + "rewards/rejected": -0.23391331732273102, + "step": 35 + }, + { + "epoch": 0.09941318605453918, + "grad_norm": 0.6118403673171997, + "learning_rate": 1.6055045871559635e-06, + "log_odds_chosen": 0.603810727596283, + "log_odds_ratio": -0.4452260434627533, + "logits/chosen": -0.6869708299636841, + "logits/rejected": 0.01393081247806549, + "logps/chosen": -1.9751290082931519, + "logps/rejected": -2.515267848968506, + "loss": 2.1348, + "nll_loss": 2.0902533531188965, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1975128948688507, + "rewards/margins": 0.05401391163468361, + "rewards/rejected": -0.251526802778244, + "step": 36 + }, + { + "epoch": 0.10217466344494304, + "grad_norm": 0.6636885404586792, + "learning_rate": 1.6513761467889911e-06, + "log_odds_chosen": 0.5960386991500854, + "log_odds_ratio": -0.44556429982185364, + "logits/chosen": -0.5324239134788513, + "logits/rejected": -0.1962709128856659, + "logps/chosen": -2.1164088249206543, + "logps/rejected": -2.6588618755340576, + "loss": 2.2904, + "nll_loss": 2.2458741664886475, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2116408795118332, + "rewards/margins": 0.054245319217443466, + "rewards/rejected": -0.26588618755340576, + "step": 37 + }, + { + "epoch": 0.10493614083534691, + "grad_norm": 0.5414693355560303, + "learning_rate": 1.6972477064220186e-06, + "log_odds_chosen": 0.4305243492126465, + "log_odds_ratio": -0.5040389895439148, + "logits/chosen": -0.5475431680679321, + "logits/rejected": -0.15733303129673004, + "logps/chosen": -1.9466073513031006, + "logps/rejected": -2.3275275230407715, + "loss": 2.1303, + "nll_loss": 2.079880475997925, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1946607381105423, + "rewards/margins": 0.03809202462434769, + "rewards/rejected": -0.2327527403831482, + "step": 38 + }, + { + "epoch": 0.10769761822575077, + "grad_norm": 0.7092136740684509, + "learning_rate": 1.743119266055046e-06, + "log_odds_chosen": 0.49458950757980347, + "log_odds_ratio": -0.499477356672287, + "logits/chosen": -0.47480347752571106, + "logits/rejected": 0.009076721966266632, + "logps/chosen": -2.077540159225464, + "logps/rejected": -2.527543067932129, + "loss": 2.2296, + "nll_loss": 2.1796536445617676, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20775403082370758, + "rewards/margins": 0.04500027373433113, + "rewards/rejected": -0.2527543008327484, + "step": 39 + }, + { + "epoch": 0.11045909561615465, + "grad_norm": 0.7840990424156189, + "learning_rate": 1.7889908256880737e-06, + "log_odds_chosen": 0.27720558643341064, + "log_odds_ratio": -0.5738192796707153, + "logits/chosen": -0.6754797101020813, + "logits/rejected": -0.1666824221611023, + "logps/chosen": -2.097449779510498, + "logps/rejected": -2.346257448196411, + "loss": 2.2706, + "nll_loss": 2.2131741046905518, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20974498987197876, + "rewards/margins": 0.02488076500594616, + "rewards/rejected": -0.23462577164173126, + "step": 40 + }, + { + "epoch": 0.11322057300655851, + "grad_norm": 0.49594300985336304, + "learning_rate": 1.8348623853211011e-06, + "log_odds_chosen": 0.4892931878566742, + "log_odds_ratio": -0.4875307083129883, + "logits/chosen": -0.5313754081726074, + "logits/rejected": -0.11813461780548096, + "logps/chosen": -1.9190640449523926, + "logps/rejected": -2.3516781330108643, + "loss": 2.0847, + "nll_loss": 2.035940647125244, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1919064074754715, + "rewards/margins": 0.043261416256427765, + "rewards/rejected": -0.23516784608364105, + "step": 41 + }, + { + "epoch": 0.11598205039696237, + "grad_norm": 0.6706519722938538, + "learning_rate": 1.8807339449541288e-06, + "log_odds_chosen": 0.3553638756275177, + "log_odds_ratio": -0.5406981706619263, + "logits/chosen": -0.6044270992279053, + "logits/rejected": -0.1877760887145996, + "logps/chosen": -2.110844850540161, + "logps/rejected": -2.4292314052581787, + "loss": 2.2811, + "nll_loss": 2.2269935607910156, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2110844999551773, + "rewards/margins": 0.031838610768318176, + "rewards/rejected": -0.24292311072349548, + "step": 42 + }, + { + "epoch": 0.11874352778736624, + "grad_norm": 0.569320023059845, + "learning_rate": 1.9266055045871564e-06, + "log_odds_chosen": 0.4219287633895874, + "log_odds_ratio": -0.5136047005653381, + "logits/chosen": -0.564371645450592, + "logits/rejected": -0.1906774342060089, + "logps/chosen": -1.9711859226226807, + "logps/rejected": -2.341031312942505, + "loss": 2.1533, + "nll_loss": 2.101966381072998, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1971185803413391, + "rewards/margins": 0.036984533071517944, + "rewards/rejected": -0.23410312831401825, + "step": 43 + }, + { + "epoch": 0.1215050051777701, + "grad_norm": 0.6019275188446045, + "learning_rate": 1.9724770642201837e-06, + "log_odds_chosen": 0.3504191040992737, + "log_odds_ratio": -0.5448323488235474, + "logits/chosen": -0.5221514105796814, + "logits/rejected": -0.2111213058233261, + "logps/chosen": -2.0630786418914795, + "logps/rejected": -2.3758115768432617, + "loss": 2.226, + "nll_loss": 2.1714861392974854, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20630787312984467, + "rewards/margins": 0.03127329424023628, + "rewards/rejected": -0.23758116364479065, + "step": 44 + }, + { + "epoch": 0.12426648256817398, + "grad_norm": 0.5997087955474854, + "learning_rate": 2.0183486238532113e-06, + "log_odds_chosen": 0.6440756916999817, + "log_odds_ratio": -0.42918679118156433, + "logits/chosen": -0.6537622213363647, + "logits/rejected": -0.3867274224758148, + "logps/chosen": -1.964981198310852, + "logps/rejected": -2.541144371032715, + "loss": 2.1164, + "nll_loss": 2.073434352874756, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1964981108903885, + "rewards/margins": 0.057616353034973145, + "rewards/rejected": -0.25411444902420044, + "step": 45 + }, + { + "epoch": 0.12702795995857785, + "grad_norm": 0.5181168913841248, + "learning_rate": 2.064220183486239e-06, + "log_odds_chosen": 0.564142107963562, + "log_odds_ratio": -0.4582359790802002, + "logits/chosen": -0.5351958870887756, + "logits/rejected": -0.1501241773366928, + "logps/chosen": -1.84321928024292, + "logps/rejected": -2.331892967224121, + "loss": 2.0016, + "nll_loss": 1.9557558298110962, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18432192504405975, + "rewards/margins": 0.04886738583445549, + "rewards/rejected": -0.23318932950496674, + "step": 46 + }, + { + "epoch": 0.1297894373489817, + "grad_norm": 0.5089595913887024, + "learning_rate": 2.110091743119266e-06, + "log_odds_chosen": 0.4004448652267456, + "log_odds_ratio": -0.5262866616249084, + "logits/chosen": -0.4425760507583618, + "logits/rejected": -0.17353124916553497, + "logps/chosen": -2.045093059539795, + "logps/rejected": -2.4056942462921143, + "loss": 2.2153, + "nll_loss": 2.162646770477295, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20450931787490845, + "rewards/margins": 0.03606009483337402, + "rewards/rejected": -0.24056941270828247, + "step": 47 + }, + { + "epoch": 0.13255091473938557, + "grad_norm": 0.6208946108818054, + "learning_rate": 2.155963302752294e-06, + "log_odds_chosen": 0.18348652124404907, + "log_odds_ratio": -0.6307563781738281, + "logits/chosen": -0.45777827501296997, + "logits/rejected": -0.11396709084510803, + "logps/chosen": -2.1183199882507324, + "logps/rejected": -2.2930657863616943, + "loss": 2.2819, + "nll_loss": 2.21886944770813, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.21183200180530548, + "rewards/margins": 0.01747458055615425, + "rewards/rejected": -0.22930656373500824, + "step": 48 + }, + { + "epoch": 0.13531239212978943, + "grad_norm": 0.6294072866439819, + "learning_rate": 2.2018348623853215e-06, + "log_odds_chosen": 0.6346040964126587, + "log_odds_ratio": -0.43739554286003113, + "logits/chosen": -0.542172908782959, + "logits/rejected": -0.039900042116642, + "logps/chosen": -2.052166223526001, + "logps/rejected": -2.628195285797119, + "loss": 2.2026, + "nll_loss": 2.1588120460510254, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.205216646194458, + "rewards/margins": 0.05760287865996361, + "rewards/rejected": -0.2628195285797119, + "step": 49 + }, + { + "epoch": 0.1380738695201933, + "grad_norm": 0.6679416298866272, + "learning_rate": 2.2477064220183487e-06, + "log_odds_chosen": 0.5639493465423584, + "log_odds_ratio": -0.47183454036712646, + "logits/chosen": -0.6221768856048584, + "logits/rejected": -0.0222525242716074, + "logps/chosen": -2.010540008544922, + "logps/rejected": -2.51875901222229, + "loss": 2.1569, + "nll_loss": 2.1097309589385986, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20105400681495667, + "rewards/margins": 0.05082192271947861, + "rewards/rejected": -0.25187593698501587, + "step": 50 + }, + { + "epoch": 0.14083534691059718, + "grad_norm": 0.6739494800567627, + "learning_rate": 2.2935779816513764e-06, + "log_odds_chosen": 0.4806385040283203, + "log_odds_ratio": -0.486289381980896, + "logits/chosen": -0.5340446829795837, + "logits/rejected": -0.3175644278526306, + "logps/chosen": -2.032139301300049, + "logps/rejected": -2.4611167907714844, + "loss": 2.1976, + "nll_loss": 2.1489548683166504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2032139152288437, + "rewards/margins": 0.04289776086807251, + "rewards/rejected": -0.2461116760969162, + "step": 51 + }, + { + "epoch": 0.14359682430100104, + "grad_norm": 0.5641347765922546, + "learning_rate": 2.339449541284404e-06, + "log_odds_chosen": 0.4302893579006195, + "log_odds_ratio": -0.5059407353401184, + "logits/chosen": -0.4652354121208191, + "logits/rejected": -0.14794279634952545, + "logps/chosen": -1.98146390914917, + "logps/rejected": -2.362124443054199, + "loss": 2.1395, + "nll_loss": 2.0888633728027344, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19814638793468475, + "rewards/margins": 0.0380660817027092, + "rewards/rejected": -0.23621246218681335, + "step": 52 + }, + { + "epoch": 0.1463583016914049, + "grad_norm": 0.6226781606674194, + "learning_rate": 2.3853211009174317e-06, + "log_odds_chosen": 0.38262999057769775, + "log_odds_ratio": -0.5264566540718079, + "logits/chosen": -0.601564884185791, + "logits/rejected": -0.23214676976203918, + "logps/chosen": -1.9446452856063843, + "logps/rejected": -2.2811543941497803, + "loss": 2.1236, + "nll_loss": 2.070988178253174, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1944645196199417, + "rewards/margins": 0.03365091234445572, + "rewards/rejected": -0.22811545431613922, + "step": 53 + }, + { + "epoch": 0.14911977908180876, + "grad_norm": 0.576812207698822, + "learning_rate": 2.431192660550459e-06, + "log_odds_chosen": 0.3935718238353729, + "log_odds_ratio": -0.5239484310150146, + "logits/chosen": -0.5574474930763245, + "logits/rejected": -0.1828337013721466, + "logps/chosen": -1.9322319030761719, + "logps/rejected": -2.280305862426758, + "loss": 2.1135, + "nll_loss": 2.061114549636841, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19322317838668823, + "rewards/margins": 0.03480737283825874, + "rewards/rejected": -0.22803056240081787, + "step": 54 + }, + { + "epoch": 0.15188125647221262, + "grad_norm": 0.5661758184432983, + "learning_rate": 2.4770642201834866e-06, + "log_odds_chosen": 0.39002981781959534, + "log_odds_ratio": -0.5207180380821228, + "logits/chosen": -0.6281000971794128, + "logits/rejected": -0.12741385400295258, + "logps/chosen": -1.9887436628341675, + "logps/rejected": -2.3331456184387207, + "loss": 2.1681, + "nll_loss": 2.115997314453125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.198874369263649, + "rewards/margins": 0.03444019705057144, + "rewards/rejected": -0.23331457376480103, + "step": 55 + }, + { + "epoch": 0.1546427338626165, + "grad_norm": 0.6655378937721252, + "learning_rate": 2.522935779816514e-06, + "log_odds_chosen": 0.6584038138389587, + "log_odds_ratio": -0.427642822265625, + "logits/chosen": -0.6279682517051697, + "logits/rejected": -0.12986338138580322, + "logps/chosen": -1.9852056503295898, + "logps/rejected": -2.5761494636535645, + "loss": 2.1584, + "nll_loss": 2.115626335144043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19852055609226227, + "rewards/margins": 0.059094399213790894, + "rewards/rejected": -0.25761497020721436, + "step": 56 + }, + { + "epoch": 0.15740421125302037, + "grad_norm": 0.48872971534729004, + "learning_rate": 2.5688073394495415e-06, + "log_odds_chosen": 0.42161181569099426, + "log_odds_ratio": -0.514995276927948, + "logits/chosen": -0.49338769912719727, + "logits/rejected": -0.07346588373184204, + "logps/chosen": -1.8429884910583496, + "logps/rejected": -2.2116591930389404, + "loss": 2.014, + "nll_loss": 1.9625122547149658, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18429884314537048, + "rewards/margins": 0.036867089569568634, + "rewards/rejected": -0.22116592526435852, + "step": 57 + }, + { + "epoch": 0.16016568864342423, + "grad_norm": 0.6582737565040588, + "learning_rate": 2.6146788990825687e-06, + "log_odds_chosen": 0.5138990879058838, + "log_odds_ratio": -0.4847075939178467, + "logits/chosen": -0.5234618186950684, + "logits/rejected": -0.12736433744430542, + "logps/chosen": -1.9835774898529053, + "logps/rejected": -2.4381330013275146, + "loss": 2.1515, + "nll_loss": 2.1030287742614746, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19835779070854187, + "rewards/margins": 0.045455530285835266, + "rewards/rejected": -0.24381330609321594, + "step": 58 + }, + { + "epoch": 0.1629271660338281, + "grad_norm": 0.5795738697052002, + "learning_rate": 2.6605504587155968e-06, + "log_odds_chosen": 0.2905232906341553, + "log_odds_ratio": -0.5645147562026978, + "logits/chosen": -0.47340482473373413, + "logits/rejected": -0.21826684474945068, + "logps/chosen": -1.9716663360595703, + "logps/rejected": -2.227067470550537, + "loss": 2.1514, + "nll_loss": 2.0949792861938477, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19716663658618927, + "rewards/margins": 0.025540094822645187, + "rewards/rejected": -0.22270673513412476, + "step": 59 + }, + { + "epoch": 0.16568864342423195, + "grad_norm": 0.5678736567497253, + "learning_rate": 2.706422018348624e-06, + "log_odds_chosen": 0.3714035749435425, + "log_odds_ratio": -0.5299574136734009, + "logits/chosen": -0.45862269401550293, + "logits/rejected": 0.08986371755599976, + "logps/chosen": -2.004216432571411, + "logps/rejected": -2.333292245864868, + "loss": 2.1781, + "nll_loss": 2.125133514404297, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20042164623737335, + "rewards/margins": 0.032907579094171524, + "rewards/rejected": -0.23332923650741577, + "step": 60 + }, + { + "epoch": 0.16845012081463584, + "grad_norm": 0.5007438063621521, + "learning_rate": 2.7522935779816517e-06, + "log_odds_chosen": 0.667113721370697, + "log_odds_ratio": -0.4193596839904785, + "logits/chosen": -0.5191382765769958, + "logits/rejected": -0.19058826565742493, + "logps/chosen": -1.917590856552124, + "logps/rejected": -2.5098061561584473, + "loss": 2.0632, + "nll_loss": 2.021275281906128, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1917591094970703, + "rewards/margins": 0.0592215433716774, + "rewards/rejected": -0.2509806454181671, + "step": 61 + }, + { + "epoch": 0.1712115982050397, + "grad_norm": 0.6429389119148254, + "learning_rate": 2.798165137614679e-06, + "log_odds_chosen": 0.41257914900779724, + "log_odds_ratio": -0.5154306888580322, + "logits/chosen": -0.45402857661247253, + "logits/rejected": 0.11148527264595032, + "logps/chosen": -2.081867218017578, + "logps/rejected": -2.45068359375, + "loss": 2.2481, + "nll_loss": 2.1965107917785645, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20818671584129333, + "rewards/margins": 0.03688164800405502, + "rewards/rejected": -0.24506837129592896, + "step": 62 + }, + { + "epoch": 0.17397307559544356, + "grad_norm": 0.5388195514678955, + "learning_rate": 2.844036697247707e-06, + "log_odds_chosen": 0.5105969905853271, + "log_odds_ratio": -0.4755265414714813, + "logits/chosen": -0.46547558903694153, + "logits/rejected": -0.1431473195552826, + "logps/chosen": -1.9948811531066895, + "logps/rejected": -2.448573589324951, + "loss": 2.1454, + "nll_loss": 2.0978972911834717, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19948811829090118, + "rewards/margins": 0.0453692302107811, + "rewards/rejected": -0.24485734105110168, + "step": 63 + }, + { + "epoch": 0.17673455298584742, + "grad_norm": 0.4911787211894989, + "learning_rate": 2.8899082568807342e-06, + "log_odds_chosen": 0.5747222304344177, + "log_odds_ratio": -0.4552631676197052, + "logits/chosen": -0.5521761178970337, + "logits/rejected": -0.294252872467041, + "logps/chosen": -1.9650574922561646, + "logps/rejected": -2.4793307781219482, + "loss": 2.1135, + "nll_loss": 2.067988634109497, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19650575518608093, + "rewards/margins": 0.051427341997623444, + "rewards/rejected": -0.24793308973312378, + "step": 64 + }, + { + "epoch": 0.17949603037625128, + "grad_norm": 0.533399760723114, + "learning_rate": 2.935779816513762e-06, + "log_odds_chosen": 0.3648831844329834, + "log_odds_ratio": -0.5328130722045898, + "logits/chosen": -0.4136759638786316, + "logits/rejected": -0.046070147305727005, + "logps/chosen": -1.8867706060409546, + "logps/rejected": -2.202749729156494, + "loss": 2.0708, + "nll_loss": 2.0175259113311768, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18867707252502441, + "rewards/margins": 0.031597916036844254, + "rewards/rejected": -0.22027498483657837, + "step": 65 + }, + { + "epoch": 0.18225750776665517, + "grad_norm": 0.5182326436042786, + "learning_rate": 2.981651376146789e-06, + "log_odds_chosen": 0.5400457382202148, + "log_odds_ratio": -0.48501351475715637, + "logits/chosen": -0.6035399436950684, + "logits/rejected": -0.1548141986131668, + "logps/chosen": -1.8529176712036133, + "logps/rejected": -2.325399398803711, + "loss": 2.022, + "nll_loss": 1.973459243774414, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18529179692268372, + "rewards/margins": 0.04724816977977753, + "rewards/rejected": -0.23253995180130005, + "step": 66 + }, + { + "epoch": 0.18501898515705903, + "grad_norm": 0.5337976813316345, + "learning_rate": 3.0275229357798168e-06, + "log_odds_chosen": 0.33741873502731323, + "log_odds_ratio": -0.5426305532455444, + "logits/chosen": -0.39523714780807495, + "logits/rejected": 0.0008284337818622589, + "logps/chosen": -1.997262716293335, + "logps/rejected": -2.2943520545959473, + "loss": 2.1433, + "nll_loss": 2.0890326499938965, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19972628355026245, + "rewards/margins": 0.029708925634622574, + "rewards/rejected": -0.22943520545959473, + "step": 67 + }, + { + "epoch": 0.1877804625474629, + "grad_norm": 0.5520625114440918, + "learning_rate": 3.073394495412844e-06, + "log_odds_chosen": 0.4403454065322876, + "log_odds_ratio": -0.5091642141342163, + "logits/chosen": -0.24377387762069702, + "logits/rejected": 0.08138424903154373, + "logps/chosen": -2.0106263160705566, + "logps/rejected": -2.402209758758545, + "loss": 2.2053, + "nll_loss": 2.1543631553649902, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2010626494884491, + "rewards/margins": 0.039158351719379425, + "rewards/rejected": -0.24022099375724792, + "step": 68 + }, + { + "epoch": 0.19054193993786675, + "grad_norm": 0.650439441204071, + "learning_rate": 3.119266055045872e-06, + "log_odds_chosen": 0.5253904461860657, + "log_odds_ratio": -0.4741411507129669, + "logits/chosen": -0.4677438735961914, + "logits/rejected": -0.013416798785328865, + "logps/chosen": -2.077601909637451, + "logps/rejected": -2.5524415969848633, + "loss": 2.2423, + "nll_loss": 2.194887161254883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20776019990444183, + "rewards/margins": 0.04748394340276718, + "rewards/rejected": -0.2552441358566284, + "step": 69 + }, + { + "epoch": 0.19330341732827064, + "grad_norm": 0.48958319425582886, + "learning_rate": 3.1651376146788993e-06, + "log_odds_chosen": 0.5917057991027832, + "log_odds_ratio": -0.45470088720321655, + "logits/chosen": -0.36797553300857544, + "logits/rejected": -0.3181126117706299, + "logps/chosen": -2.022463321685791, + "logps/rejected": -2.5544497966766357, + "loss": 2.1697, + "nll_loss": 2.124249219894409, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20224635303020477, + "rewards/margins": 0.05319864675402641, + "rewards/rejected": -0.2554450035095215, + "step": 70 + }, + { + "epoch": 0.1960648947186745, + "grad_norm": 0.6293954253196716, + "learning_rate": 3.211009174311927e-06, + "log_odds_chosen": 0.44158488512039185, + "log_odds_ratio": -0.5029429793357849, + "logits/chosen": -0.4253910481929779, + "logits/rejected": -0.14389574527740479, + "logps/chosen": -1.9857234954833984, + "logps/rejected": -2.379268169403076, + "loss": 2.17, + "nll_loss": 2.119678497314453, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19857235252857208, + "rewards/margins": 0.03935447335243225, + "rewards/rejected": -0.23792681097984314, + "step": 71 + }, + { + "epoch": 0.19882637210907836, + "grad_norm": 0.6024268269538879, + "learning_rate": 3.256880733944954e-06, + "log_odds_chosen": 0.5992636680603027, + "log_odds_ratio": -0.4583205580711365, + "logits/chosen": -0.3624473810195923, + "logits/rejected": -0.16807770729064941, + "logps/chosen": -1.8936195373535156, + "logps/rejected": -2.403137683868408, + "loss": 2.0566, + "nll_loss": 2.0107438564300537, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18936194479465485, + "rewards/margins": 0.05095181241631508, + "rewards/rejected": -0.24031376838684082, + "step": 72 + }, + { + "epoch": 0.20158784949948222, + "grad_norm": 0.537467360496521, + "learning_rate": 3.3027522935779823e-06, + "log_odds_chosen": 0.38986673951148987, + "log_odds_ratio": -0.518619179725647, + "logits/chosen": -0.28752538561820984, + "logits/rejected": -0.08893117308616638, + "logps/chosen": -2.056885242462158, + "logps/rejected": -2.4047675132751465, + "loss": 2.2167, + "nll_loss": 2.1648688316345215, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2056885063648224, + "rewards/margins": 0.03478822112083435, + "rewards/rejected": -0.24047674238681793, + "step": 73 + }, + { + "epoch": 0.20434932688988608, + "grad_norm": 0.48359447717666626, + "learning_rate": 3.3486238532110095e-06, + "log_odds_chosen": 0.39413708448410034, + "log_odds_ratio": -0.5191097259521484, + "logits/chosen": -0.28818562626838684, + "logits/rejected": -0.17117023468017578, + "logps/chosen": -1.9585490226745605, + "logps/rejected": -2.3046271800994873, + "loss": 2.1453, + "nll_loss": 2.093356132507324, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19585487246513367, + "rewards/margins": 0.03460782766342163, + "rewards/rejected": -0.2304627150297165, + "step": 74 + }, + { + "epoch": 0.20711080428028997, + "grad_norm": 0.6197869777679443, + "learning_rate": 3.394495412844037e-06, + "log_odds_chosen": 0.4853059649467468, + "log_odds_ratio": -0.487282931804657, + "logits/chosen": -0.4553055763244629, + "logits/rejected": -0.07368504256010056, + "logps/chosen": -1.9564940929412842, + "logps/rejected": -2.38771653175354, + "loss": 2.1426, + "nll_loss": 2.093902826309204, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1956494152545929, + "rewards/margins": 0.04312223941087723, + "rewards/rejected": -0.23877164721488953, + "step": 75 + }, + { + "epoch": 0.20987228167069383, + "grad_norm": 0.5478652119636536, + "learning_rate": 3.4403669724770644e-06, + "log_odds_chosen": 0.4639507532119751, + "log_odds_ratio": -0.4932023882865906, + "logits/chosen": -0.30065810680389404, + "logits/rejected": -0.25181248784065247, + "logps/chosen": -2.044691562652588, + "logps/rejected": -2.460970401763916, + "loss": 2.1933, + "nll_loss": 2.1440277099609375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20446917414665222, + "rewards/margins": 0.04162788391113281, + "rewards/rejected": -0.24609704315662384, + "step": 76 + }, + { + "epoch": 0.2126337590610977, + "grad_norm": 0.5865225195884705, + "learning_rate": 3.486238532110092e-06, + "log_odds_chosen": 0.40864938497543335, + "log_odds_ratio": -0.5256964564323425, + "logits/chosen": -0.414636492729187, + "logits/rejected": -0.1841173619031906, + "logps/chosen": -2.0234270095825195, + "logps/rejected": -2.3900880813598633, + "loss": 2.2079, + "nll_loss": 2.155355215072632, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2023427039384842, + "rewards/margins": 0.03666612505912781, + "rewards/rejected": -0.239008828997612, + "step": 77 + }, + { + "epoch": 0.21539523645150155, + "grad_norm": 0.5245475769042969, + "learning_rate": 3.5321100917431193e-06, + "log_odds_chosen": 0.3867124617099762, + "log_odds_ratio": -0.5356163382530212, + "logits/chosen": -0.22849449515342712, + "logits/rejected": 0.04786547273397446, + "logps/chosen": -2.047503709793091, + "logps/rejected": -2.391859531402588, + "loss": 2.2124, + "nll_loss": 2.1587889194488525, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.20475035905838013, + "rewards/margins": 0.03443560749292374, + "rewards/rejected": -0.23918597400188446, + "step": 78 + }, + { + "epoch": 0.2181567138419054, + "grad_norm": 0.4416508972644806, + "learning_rate": 3.5779816513761473e-06, + "log_odds_chosen": 0.457964152097702, + "log_odds_ratio": -0.49229007959365845, + "logits/chosen": -0.24249830842018127, + "logits/rejected": -0.1156221255660057, + "logps/chosen": -1.9371097087860107, + "logps/rejected": -2.3405327796936035, + "loss": 2.1264, + "nll_loss": 2.077136516571045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19371098279953003, + "rewards/margins": 0.0403422936797142, + "rewards/rejected": -0.23405326902866364, + "step": 79 + }, + { + "epoch": 0.2209181912323093, + "grad_norm": 0.511163055896759, + "learning_rate": 3.6238532110091746e-06, + "log_odds_chosen": 0.3918084502220154, + "log_odds_ratio": -0.527028501033783, + "logits/chosen": -0.3217644989490509, + "logits/rejected": -0.12379680573940277, + "logps/chosen": -1.9086521863937378, + "logps/rejected": -2.250434160232544, + "loss": 2.0875, + "nll_loss": 2.034804105758667, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19086521863937378, + "rewards/margins": 0.03417817875742912, + "rewards/rejected": -0.2250434011220932, + "step": 80 + }, + { + "epoch": 0.22367966862271316, + "grad_norm": 0.48532167077064514, + "learning_rate": 3.6697247706422022e-06, + "log_odds_chosen": 0.38070449233055115, + "log_odds_ratio": -0.5242205858230591, + "logits/chosen": -0.24639271199703217, + "logits/rejected": -0.02295789122581482, + "logps/chosen": -1.9235130548477173, + "logps/rejected": -2.256152868270874, + "loss": 2.0966, + "nll_loss": 2.044215440750122, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.192351296544075, + "rewards/margins": 0.03326397389173508, + "rewards/rejected": -0.22561527788639069, + "step": 81 + }, + { + "epoch": 0.22644114601311702, + "grad_norm": 0.4871878921985626, + "learning_rate": 3.7155963302752295e-06, + "log_odds_chosen": 0.3706399202346802, + "log_odds_ratio": -0.5302938222885132, + "logits/chosen": -0.32461822032928467, + "logits/rejected": -0.07178197801113129, + "logps/chosen": -2.002980947494507, + "logps/rejected": -2.3308701515197754, + "loss": 2.1902, + "nll_loss": 2.1371781826019287, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20029808580875397, + "rewards/margins": 0.03278890624642372, + "rewards/rejected": -0.2330869883298874, + "step": 82 + }, + { + "epoch": 0.22920262340352088, + "grad_norm": 0.5049198865890503, + "learning_rate": 3.7614678899082575e-06, + "log_odds_chosen": 0.6436396241188049, + "log_odds_ratio": -0.4284180700778961, + "logits/chosen": -0.3081129789352417, + "logits/rejected": -0.1997162252664566, + "logps/chosen": -1.9448068141937256, + "logps/rejected": -2.5192015171051025, + "loss": 2.1169, + "nll_loss": 2.0740115642547607, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19448068737983704, + "rewards/margins": 0.057439468801021576, + "rewards/rejected": -0.2519201636314392, + "step": 83 + }, + { + "epoch": 0.23196410079392474, + "grad_norm": 0.5246726870536804, + "learning_rate": 3.8073394495412848e-06, + "log_odds_chosen": 0.43793147802352905, + "log_odds_ratio": -0.5015705823898315, + "logits/chosen": -0.29296931624412537, + "logits/rejected": -0.13174469769001007, + "logps/chosen": -1.941349744796753, + "logps/rejected": -2.3253884315490723, + "loss": 2.1333, + "nll_loss": 2.08317232131958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19413498044013977, + "rewards/margins": 0.03840385749936104, + "rewards/rejected": -0.2325388491153717, + "step": 84 + }, + { + "epoch": 0.23472557818432863, + "grad_norm": 0.41432732343673706, + "learning_rate": 3.853211009174313e-06, + "log_odds_chosen": 0.5745496153831482, + "log_odds_ratio": -0.4579155147075653, + "logits/chosen": -0.3449418544769287, + "logits/rejected": -0.19380588829517365, + "logps/chosen": -1.8625164031982422, + "logps/rejected": -2.3706510066986084, + "loss": 2.0375, + "nll_loss": 1.9917008876800537, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1862516552209854, + "rewards/margins": 0.050813477486371994, + "rewards/rejected": -0.2370651215314865, + "step": 85 + }, + { + "epoch": 0.2374870555747325, + "grad_norm": 0.438418984413147, + "learning_rate": 3.89908256880734e-06, + "log_odds_chosen": 0.580481231212616, + "log_odds_ratio": -0.4620014429092407, + "logits/chosen": -0.14796528220176697, + "logits/rejected": -0.32091909646987915, + "logps/chosen": -1.9756391048431396, + "logps/rejected": -2.4986729621887207, + "loss": 2.1484, + "nll_loss": 2.102167844772339, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19756391644477844, + "rewards/margins": 0.052303388714790344, + "rewards/rejected": -0.2498673051595688, + "step": 86 + }, + { + "epoch": 0.24024853296513635, + "grad_norm": 0.4355674684047699, + "learning_rate": 3.944954128440367e-06, + "log_odds_chosen": 0.5157158970832825, + "log_odds_ratio": -0.4764195382595062, + "logits/chosen": -0.16625456511974335, + "logits/rejected": -0.22959333658218384, + "logps/chosen": -1.8700374364852905, + "logps/rejected": -2.3208043575286865, + "loss": 2.0401, + "nll_loss": 1.9924601316452026, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18700376152992249, + "rewards/margins": 0.04507668316364288, + "rewards/rejected": -0.23208042979240417, + "step": 87 + }, + { + "epoch": 0.2430100103555402, + "grad_norm": 0.4138505756855011, + "learning_rate": 3.9908256880733945e-06, + "log_odds_chosen": 0.5811448693275452, + "log_odds_ratio": -0.48126012086868286, + "logits/chosen": -0.3125801980495453, + "logits/rejected": -0.16787581145763397, + "logps/chosen": -1.79574453830719, + "logps/rejected": -2.3057985305786133, + "loss": 1.9834, + "nll_loss": 1.9353020191192627, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17957444489002228, + "rewards/margins": 0.051005423069000244, + "rewards/rejected": -0.23057986795902252, + "step": 88 + }, + { + "epoch": 0.24577148774594407, + "grad_norm": 0.3883982002735138, + "learning_rate": 4.036697247706423e-06, + "log_odds_chosen": 0.6218655705451965, + "log_odds_ratio": -0.43868428468704224, + "logits/chosen": -0.16164137423038483, + "logits/rejected": -0.2850674092769623, + "logps/chosen": -1.8971762657165527, + "logps/rejected": -2.4469919204711914, + "loss": 2.0667, + "nll_loss": 2.0228123664855957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.189717635512352, + "rewards/margins": 0.05498155951499939, + "rewards/rejected": -0.24469918012619019, + "step": 89 + }, + { + "epoch": 0.24853296513634796, + "grad_norm": 0.441839337348938, + "learning_rate": 4.08256880733945e-06, + "log_odds_chosen": 0.5872446894645691, + "log_odds_ratio": -0.44401073455810547, + "logits/chosen": -0.2670595347881317, + "logits/rejected": -0.07647659629583359, + "logps/chosen": -1.8952282667160034, + "logps/rejected": -2.412400245666504, + "loss": 2.066, + "nll_loss": 2.021595001220703, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18952281773090363, + "rewards/margins": 0.051717206835746765, + "rewards/rejected": -0.24124003946781158, + "step": 90 + }, + { + "epoch": 0.2512944425267518, + "grad_norm": 0.4077044427394867, + "learning_rate": 4.128440366972478e-06, + "log_odds_chosen": 0.4384881556034088, + "log_odds_ratio": -0.5076763033866882, + "logits/chosen": -0.15786704421043396, + "logits/rejected": -0.090408094227314, + "logps/chosen": -1.878911018371582, + "logps/rejected": -2.262026786804199, + "loss": 2.047, + "nll_loss": 1.9962477684020996, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18789111077785492, + "rewards/margins": 0.03831159323453903, + "rewards/rejected": -0.22620268166065216, + "step": 91 + }, + { + "epoch": 0.2540559199171557, + "grad_norm": 0.3839576244354248, + "learning_rate": 4.174311926605505e-06, + "log_odds_chosen": 0.42869681119918823, + "log_odds_ratio": -0.5070521235466003, + "logits/chosen": -0.08416657149791718, + "logits/rejected": -0.4555465579032898, + "logps/chosen": -1.9147799015045166, + "logps/rejected": -2.292423725128174, + "loss": 2.0814, + "nll_loss": 2.0306570529937744, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19147798418998718, + "rewards/margins": 0.03776439279317856, + "rewards/rejected": -0.22924238443374634, + "step": 92 + }, + { + "epoch": 0.25681739730755954, + "grad_norm": 0.42368394136428833, + "learning_rate": 4.220183486238532e-06, + "log_odds_chosen": 0.5149534344673157, + "log_odds_ratio": -0.47629106044769287, + "logits/chosen": -0.13718552887439728, + "logits/rejected": -0.20033138990402222, + "logps/chosen": -1.9013878107070923, + "logps/rejected": -2.3537254333496094, + "loss": 2.0789, + "nll_loss": 2.0312626361846924, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1901387870311737, + "rewards/margins": 0.04523376002907753, + "rewards/rejected": -0.23537255823612213, + "step": 93 + }, + { + "epoch": 0.2595788746979634, + "grad_norm": 0.38014668226242065, + "learning_rate": 4.26605504587156e-06, + "log_odds_chosen": 0.4552799165248871, + "log_odds_ratio": -0.49941790103912354, + "logits/chosen": -0.2649000883102417, + "logits/rejected": -0.20182102918624878, + "logps/chosen": -1.8007447719573975, + "logps/rejected": -2.1948485374450684, + "loss": 1.9637, + "nll_loss": 1.9138000011444092, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18007448315620422, + "rewards/margins": 0.03941037505865097, + "rewards/rejected": -0.2194848358631134, + "step": 94 + }, + { + "epoch": 0.26234035208836726, + "grad_norm": 0.3550948202610016, + "learning_rate": 4.311926605504588e-06, + "log_odds_chosen": 0.36030256748199463, + "log_odds_ratio": -0.533312976360321, + "logits/chosen": -0.0772688165307045, + "logits/rejected": -0.16052654385566711, + "logps/chosen": -1.8254213333129883, + "logps/rejected": -2.136554002761841, + "loss": 2.0005, + "nll_loss": 1.9472124576568604, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1825421154499054, + "rewards/margins": 0.03111325576901436, + "rewards/rejected": -0.21365538239479065, + "step": 95 + }, + { + "epoch": 0.26510182947877114, + "grad_norm": 0.40300413966178894, + "learning_rate": 4.357798165137615e-06, + "log_odds_chosen": 0.2818768620491028, + "log_odds_ratio": -0.5676061511039734, + "logits/chosen": -0.10504347085952759, + "logits/rejected": -0.05843639373779297, + "logps/chosen": -1.9888579845428467, + "logps/rejected": -2.2374351024627686, + "loss": 2.1718, + "nll_loss": 2.11505126953125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19888579845428467, + "rewards/margins": 0.02485768496990204, + "rewards/rejected": -0.2237434834241867, + "step": 96 + }, + { + "epoch": 0.26786330686917503, + "grad_norm": 0.398783802986145, + "learning_rate": 4.403669724770643e-06, + "log_odds_chosen": 0.5546217560768127, + "log_odds_ratio": -0.46683651208877563, + "logits/chosen": -0.008073419332504272, + "logits/rejected": -0.08634719252586365, + "logps/chosen": -1.8501688241958618, + "logps/rejected": -2.342288017272949, + "loss": 2.0318, + "nll_loss": 1.9851207733154297, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18501687049865723, + "rewards/margins": 0.04921191930770874, + "rewards/rejected": -0.23422878980636597, + "step": 97 + }, + { + "epoch": 0.27062478425957887, + "grad_norm": 0.39449602365493774, + "learning_rate": 4.44954128440367e-06, + "log_odds_chosen": 0.4133303165435791, + "log_odds_ratio": -0.5122652649879456, + "logits/chosen": -0.05717798322439194, + "logits/rejected": -0.1295921504497528, + "logps/chosen": -1.9054274559020996, + "logps/rejected": -2.266583204269409, + "loss": 2.0813, + "nll_loss": 2.030045509338379, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19054275751113892, + "rewards/margins": 0.03611557558178902, + "rewards/rejected": -0.22665831446647644, + "step": 98 + }, + { + "epoch": 0.27338626164998275, + "grad_norm": 0.39879634976387024, + "learning_rate": 4.4954128440366975e-06, + "log_odds_chosen": 0.547784149646759, + "log_odds_ratio": -0.46036428213119507, + "logits/chosen": -0.09109216928482056, + "logits/rejected": -0.3699495494365692, + "logps/chosen": -1.9557167291641235, + "logps/rejected": -2.4426751136779785, + "loss": 2.1161, + "nll_loss": 2.070013999938965, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1955716609954834, + "rewards/margins": 0.04869585484266281, + "rewards/rejected": -0.2442675083875656, + "step": 99 + }, + { + "epoch": 0.2761477390403866, + "grad_norm": 0.3873852491378784, + "learning_rate": 4.541284403669725e-06, + "log_odds_chosen": 0.3448026478290558, + "log_odds_ratio": -0.5393580198287964, + "logits/chosen": -0.1656271517276764, + "logits/rejected": -0.07856383174657822, + "logps/chosen": -1.8394955396652222, + "logps/rejected": -2.1358819007873535, + "loss": 2.0365, + "nll_loss": 1.9825859069824219, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1839495450258255, + "rewards/margins": 0.029638633131980896, + "rewards/rejected": -0.2135881781578064, + "step": 100 + }, + { + "epoch": 0.2789092164307905, + "grad_norm": 0.3506630063056946, + "learning_rate": 4.587155963302753e-06, + "log_odds_chosen": 0.33486518263816833, + "log_odds_ratio": -0.5475890636444092, + "logits/chosen": 0.009370687417685986, + "logits/rejected": -0.18966984748840332, + "logps/chosen": -1.9532427787780762, + "logps/rejected": -2.2507667541503906, + "loss": 2.132, + "nll_loss": 2.077195405960083, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19532427191734314, + "rewards/margins": 0.02975238859653473, + "rewards/rejected": -0.22507666051387787, + "step": 101 + }, + { + "epoch": 0.28167069382119436, + "grad_norm": 0.33624136447906494, + "learning_rate": 4.63302752293578e-06, + "log_odds_chosen": 0.5504237413406372, + "log_odds_ratio": -0.4613422155380249, + "logits/chosen": -0.06238182261586189, + "logits/rejected": -0.22859227657318115, + "logps/chosen": -1.791447401046753, + "logps/rejected": -2.2658450603485107, + "loss": 1.9877, + "nll_loss": 1.9415662288665771, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1791447550058365, + "rewards/margins": 0.04743976891040802, + "rewards/rejected": -0.2265845388174057, + "step": 102 + }, + { + "epoch": 0.2844321712115982, + "grad_norm": 0.3664388656616211, + "learning_rate": 4.678899082568808e-06, + "log_odds_chosen": 0.3437744081020355, + "log_odds_ratio": -0.5439410209655762, + "logits/chosen": -0.1386101245880127, + "logits/rejected": -0.17609171569347382, + "logps/chosen": -1.8379578590393066, + "logps/rejected": -2.1358203887939453, + "loss": 2.022, + "nll_loss": 1.9676285982131958, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1837957799434662, + "rewards/margins": 0.0297862458974123, + "rewards/rejected": -0.21358203887939453, + "step": 103 + }, + { + "epoch": 0.2871936486020021, + "grad_norm": 0.3293968439102173, + "learning_rate": 4.724770642201835e-06, + "log_odds_chosen": 0.3870212137699127, + "log_odds_ratio": -0.5231497883796692, + "logits/chosen": -0.07520203292369843, + "logits/rejected": -0.15402644872665405, + "logps/chosen": -1.756553053855896, + "logps/rejected": -2.086921453475952, + "loss": 1.9488, + "nll_loss": 1.8965202569961548, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1756553202867508, + "rewards/margins": 0.033036813139915466, + "rewards/rejected": -0.20869213342666626, + "step": 104 + }, + { + "epoch": 0.2899551259924059, + "grad_norm": 0.33918339014053345, + "learning_rate": 4.770642201834863e-06, + "log_odds_chosen": 0.4852727949619293, + "log_odds_ratio": -0.4833363890647888, + "logits/chosen": -0.004435461014509201, + "logits/rejected": -0.1534399837255478, + "logps/chosen": -1.7944328784942627, + "logps/rejected": -2.2146499156951904, + "loss": 1.9627, + "nll_loss": 1.9144006967544556, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17944329977035522, + "rewards/margins": 0.042021699249744415, + "rewards/rejected": -0.22146499156951904, + "step": 105 + }, + { + "epoch": 0.2927166033828098, + "grad_norm": 0.3346327841281891, + "learning_rate": 4.816513761467891e-06, + "log_odds_chosen": 0.5177164673805237, + "log_odds_ratio": -0.47983041405677795, + "logits/chosen": -0.00399226788431406, + "logits/rejected": -0.10182341933250427, + "logps/chosen": -1.7794350385665894, + "logps/rejected": -2.226297616958618, + "loss": 1.9511, + "nll_loss": 1.9030946493148804, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17794349789619446, + "rewards/margins": 0.044686250388622284, + "rewards/rejected": -0.22262977063655853, + "step": 106 + }, + { + "epoch": 0.2954780807732137, + "grad_norm": 0.3454085886478424, + "learning_rate": 4.862385321100918e-06, + "log_odds_chosen": 0.3980882465839386, + "log_odds_ratio": -0.5184234380722046, + "logits/chosen": 0.01898978091776371, + "logits/rejected": -0.21249954402446747, + "logps/chosen": -1.8531229496002197, + "logps/rejected": -2.197577953338623, + "loss": 2.0242, + "nll_loss": 1.9724009037017822, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18531231582164764, + "rewards/margins": 0.03444547951221466, + "rewards/rejected": -0.2197577804327011, + "step": 107 + }, + { + "epoch": 0.2982395581636175, + "grad_norm": 0.3301767408847809, + "learning_rate": 4.908256880733945e-06, + "log_odds_chosen": 0.44768860936164856, + "log_odds_ratio": -0.5000944137573242, + "logits/chosen": 0.06930336356163025, + "logits/rejected": -0.24058011174201965, + "logps/chosen": -1.8849480152130127, + "logps/rejected": -2.278297185897827, + "loss": 2.0446, + "nll_loss": 1.9945893287658691, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18849481642246246, + "rewards/margins": 0.039334915578365326, + "rewards/rejected": -0.2278297394514084, + "step": 108 + }, + { + "epoch": 0.3010010355540214, + "grad_norm": 0.3244481086730957, + "learning_rate": 4.954128440366973e-06, + "log_odds_chosen": 0.6234428286552429, + "log_odds_ratio": -0.44249945878982544, + "logits/chosen": 0.045590970665216446, + "logits/rejected": -0.3307313919067383, + "logps/chosen": -1.8896514177322388, + "logps/rejected": -2.4449257850646973, + "loss": 2.0431, + "nll_loss": 1.9988850355148315, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18896515667438507, + "rewards/margins": 0.0555274523794651, + "rewards/rejected": -0.24449260532855988, + "step": 109 + }, + { + "epoch": 0.30376251294442524, + "grad_norm": 0.31317761540412903, + "learning_rate": 5e-06, + "log_odds_chosen": 0.2916356325149536, + "log_odds_ratio": -0.5743635892868042, + "logits/chosen": 0.144297257065773, + "logits/rejected": -0.26908552646636963, + "logps/chosen": -1.8418867588043213, + "logps/rejected": -2.0925679206848145, + "loss": 2.0441, + "nll_loss": 1.9866299629211426, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18418867886066437, + "rewards/margins": 0.02506813406944275, + "rewards/rejected": -0.20925679802894592, + "step": 110 + }, + { + "epoch": 0.30652399033482913, + "grad_norm": 0.29466620087623596, + "learning_rate": 4.999987154315977e-06, + "log_odds_chosen": 0.42353084683418274, + "log_odds_ratio": -0.5111951231956482, + "logits/chosen": 0.05306434631347656, + "logits/rejected": -0.15503177046775818, + "logps/chosen": -1.7407968044281006, + "logps/rejected": -2.0989816188812256, + "loss": 1.924, + "nll_loss": 1.87285315990448, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17407968640327454, + "rewards/margins": 0.03581848740577698, + "rewards/rejected": -0.2098981738090515, + "step": 111 + }, + { + "epoch": 0.309285467725233, + "grad_norm": 0.3188883364200592, + "learning_rate": 4.999948617395916e-06, + "log_odds_chosen": 0.4043401777744293, + "log_odds_ratio": -0.5218645930290222, + "logits/chosen": 0.16884373128414154, + "logits/rejected": -0.07719510793685913, + "logps/chosen": -1.8007270097732544, + "logps/rejected": -2.151782274246216, + "loss": 1.9777, + "nll_loss": 1.9254791736602783, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18007270991802216, + "rewards/margins": 0.035105518996715546, + "rewards/rejected": -0.2151782363653183, + "step": 112 + }, + { + "epoch": 0.31204694511563685, + "grad_norm": 0.3535216450691223, + "learning_rate": 4.999884389635843e-06, + "log_odds_chosen": 0.41415858268737793, + "log_odds_ratio": -0.5124688744544983, + "logits/chosen": 0.0558352991938591, + "logits/rejected": -0.09803693741559982, + "logps/chosen": -1.8450539112091064, + "logps/rejected": -2.2039334774017334, + "loss": 2.0336, + "nll_loss": 1.9823946952819824, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1845053881406784, + "rewards/margins": 0.035887960344552994, + "rewards/rejected": -0.2203933596611023, + "step": 113 + }, + { + "epoch": 0.31480842250604074, + "grad_norm": 0.2991940379142761, + "learning_rate": 4.9997944716957985e-06, + "log_odds_chosen": 0.514029324054718, + "log_odds_ratio": -0.47918182611465454, + "logits/chosen": 0.16882193088531494, + "logits/rejected": -0.24593651294708252, + "logps/chosen": -1.8493305444717407, + "logps/rejected": -2.298590898513794, + "loss": 2.0272, + "nll_loss": 1.9792673587799072, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18493306636810303, + "rewards/margins": 0.044926032423973083, + "rewards/rejected": -0.22985908389091492, + "step": 114 + }, + { + "epoch": 0.3175698998964446, + "grad_norm": 0.28921976685523987, + "learning_rate": 4.999678864499828e-06, + "log_odds_chosen": 0.38468483090400696, + "log_odds_ratio": -0.5222321152687073, + "logits/chosen": 0.2794942557811737, + "logits/rejected": -0.21004724502563477, + "logps/chosen": -1.8734655380249023, + "logps/rejected": -2.2063117027282715, + "loss": 2.0344, + "nll_loss": 1.9822089672088623, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18734657764434814, + "rewards/margins": 0.03328459709882736, + "rewards/rejected": -0.2206311821937561, + "step": 115 + }, + { + "epoch": 0.32033137728684846, + "grad_norm": 0.30199047923088074, + "learning_rate": 4.999537569235975e-06, + "log_odds_chosen": 0.35607457160949707, + "log_odds_ratio": -0.5390717387199402, + "logits/chosen": 0.17685924470424652, + "logits/rejected": -0.16482022404670715, + "logps/chosen": -1.8530224561691284, + "logps/rejected": -2.1583948135375977, + "loss": 2.0285, + "nll_loss": 1.9746413230895996, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1853022426366806, + "rewards/margins": 0.030537229031324387, + "rewards/rejected": -0.21583949029445648, + "step": 116 + }, + { + "epoch": 0.32309285467725235, + "grad_norm": 0.305818647146225, + "learning_rate": 4.999370587356267e-06, + "log_odds_chosen": 0.3153924345970154, + "log_odds_ratio": -0.5565903782844543, + "logits/chosen": 0.17289672791957855, + "logits/rejected": -0.3006405234336853, + "logps/chosen": -1.9371519088745117, + "logps/rejected": -2.214411497116089, + "loss": 2.1096, + "nll_loss": 2.0539159774780273, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1937151849269867, + "rewards/margins": 0.027725949883461, + "rewards/rejected": -0.2214411348104477, + "step": 117 + }, + { + "epoch": 0.3258543320676562, + "grad_norm": 0.324216365814209, + "learning_rate": 4.9991779205767e-06, + "log_odds_chosen": 0.29256871342658997, + "log_odds_ratio": -0.5621715784072876, + "logits/chosen": 0.055469777435064316, + "logits/rejected": -0.5593094229698181, + "logps/chosen": -1.808518886566162, + "logps/rejected": -2.060049295425415, + "loss": 1.9916, + "nll_loss": 1.9353588819503784, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18085187673568726, + "rewards/margins": 0.02515305206179619, + "rewards/rejected": -0.20600494742393494, + "step": 118 + }, + { + "epoch": 0.32861580945806007, + "grad_norm": 0.316292941570282, + "learning_rate": 4.998959570877224e-06, + "log_odds_chosen": 0.391570121049881, + "log_odds_ratio": -0.519210159778595, + "logits/chosen": 0.29701903462409973, + "logits/rejected": -0.248467817902565, + "logps/chosen": -1.950477123260498, + "logps/rejected": -2.2915000915527344, + "loss": 2.1349, + "nll_loss": 2.0829989910125732, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.19504770636558533, + "rewards/margins": 0.034102290868759155, + "rewards/rejected": -0.22914999723434448, + "step": 119 + }, + { + "epoch": 0.3313772868484639, + "grad_norm": 0.29707539081573486, + "learning_rate": 4.99871554050172e-06, + "log_odds_chosen": 0.49879857897758484, + "log_odds_ratio": -0.4770265221595764, + "logits/chosen": 0.1394173949956894, + "logits/rejected": -0.11445163935422897, + "logps/chosen": -1.7406611442565918, + "logps/rejected": -2.167613983154297, + "loss": 1.9065, + "nll_loss": 1.8588043451309204, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17406611144542694, + "rewards/margins": 0.042695268988609314, + "rewards/rejected": -0.21676141023635864, + "step": 120 + }, + { + "epoch": 0.3341387642388678, + "grad_norm": 0.3011818528175354, + "learning_rate": 4.9984458319579775e-06, + "log_odds_chosen": 0.4734145998954773, + "log_odds_ratio": -0.48969942331314087, + "logits/chosen": 0.10353265702724457, + "logits/rejected": -0.22807854413986206, + "logps/chosen": -1.7799421548843384, + "logps/rejected": -2.1871492862701416, + "loss": 1.9363, + "nll_loss": 1.8872833251953125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17799422144889832, + "rewards/margins": 0.04072071984410286, + "rewards/rejected": -0.21871493756771088, + "step": 121 + }, + { + "epoch": 0.3369002416292717, + "grad_norm": 0.28915688395500183, + "learning_rate": 4.99815044801767e-06, + "log_odds_chosen": 0.3440546989440918, + "log_odds_ratio": -0.5443906784057617, + "logits/chosen": 0.22890803217887878, + "logits/rejected": -0.20038765668869019, + "logps/chosen": -1.8269233703613281, + "logps/rejected": -2.1230876445770264, + "loss": 2.0144, + "nll_loss": 1.959984540939331, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18269234895706177, + "rewards/margins": 0.02961641550064087, + "rewards/rejected": -0.21230876445770264, + "step": 122 + }, + { + "epoch": 0.3396617190196755, + "grad_norm": 0.31248003244400024, + "learning_rate": 4.9978293917163225e-06, + "log_odds_chosen": 0.21560871601104736, + "log_odds_ratio": -0.5958309769630432, + "logits/chosen": 0.2199111431837082, + "logits/rejected": -0.15359394252300262, + "logps/chosen": -1.9724359512329102, + "logps/rejected": -2.162384033203125, + "loss": 2.1306, + "nll_loss": 2.0710067749023438, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1972435861825943, + "rewards/margins": 0.01899481564760208, + "rewards/rejected": -0.21623840928077698, + "step": 123 + }, + { + "epoch": 0.3424231964100794, + "grad_norm": 0.3086913824081421, + "learning_rate": 4.997482666353287e-06, + "log_odds_chosen": 0.36193954944610596, + "log_odds_ratio": -0.5371627807617188, + "logits/chosen": 0.07905742526054382, + "logits/rejected": -0.047993652522563934, + "logps/chosen": -1.860518217086792, + "logps/rejected": -2.175654649734497, + "loss": 2.0362, + "nll_loss": 1.982520341873169, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18605183064937592, + "rewards/margins": 0.03151364251971245, + "rewards/rejected": -0.21756546199321747, + "step": 124 + }, + { + "epoch": 0.34518467380048323, + "grad_norm": 0.2956501543521881, + "learning_rate": 4.997110275491702e-06, + "log_odds_chosen": 0.5367758274078369, + "log_odds_ratio": -0.4794245958328247, + "logits/chosen": 0.17313790321350098, + "logits/rejected": -0.5377508401870728, + "logps/chosen": -1.7679404020309448, + "logps/rejected": -2.2323243618011475, + "loss": 1.9515, + "nll_loss": 1.9035319089889526, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17679405212402344, + "rewards/margins": 0.04643838852643967, + "rewards/rejected": -0.2232324331998825, + "step": 125 + }, + { + "epoch": 0.3479461511908871, + "grad_norm": 0.2930620014667511, + "learning_rate": 4.9967122229584614e-06, + "log_odds_chosen": 0.5086734890937805, + "log_odds_ratio": -0.47367364168167114, + "logits/chosen": 0.20121866464614868, + "logits/rejected": -0.3580709397792816, + "logps/chosen": -1.772174596786499, + "logps/rejected": -2.211575746536255, + "loss": 1.9502, + "nll_loss": 1.9028148651123047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17721746861934662, + "rewards/margins": 0.04394011199474335, + "rewards/rejected": -0.22115758061408997, + "step": 126 + }, + { + "epoch": 0.350707628581291, + "grad_norm": 0.313006728887558, + "learning_rate": 4.996288512844169e-06, + "log_odds_chosen": 0.26984095573425293, + "log_odds_ratio": -0.5770066380500793, + "logits/chosen": 0.30613642930984497, + "logits/rejected": -0.0801864042878151, + "logps/chosen": -1.9204916954040527, + "logps/rejected": -2.153423547744751, + "loss": 2.1089, + "nll_loss": 2.051222324371338, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19204919040203094, + "rewards/margins": 0.023293154314160347, + "rewards/rejected": -0.21534234285354614, + "step": 127 + }, + { + "epoch": 0.35346910597169484, + "grad_norm": 0.26883664727211, + "learning_rate": 4.995839149503103e-06, + "log_odds_chosen": 0.4242061972618103, + "log_odds_ratio": -0.5166581273078918, + "logits/chosen": 0.287309467792511, + "logits/rejected": -0.28880101442337036, + "logps/chosen": -1.7962003946304321, + "logps/rejected": -2.1639606952667236, + "loss": 1.9745, + "nll_loss": 1.9228155612945557, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17962004244327545, + "rewards/margins": 0.036776017397642136, + "rewards/rejected": -0.2163960486650467, + "step": 128 + }, + { + "epoch": 0.35623058336209873, + "grad_norm": 0.3000660240650177, + "learning_rate": 4.995364137553166e-06, + "log_odds_chosen": 0.2731889486312866, + "log_odds_ratio": -0.5769234299659729, + "logits/chosen": 0.17702779173851013, + "logits/rejected": -0.16427305340766907, + "logps/chosen": -1.8508391380310059, + "logps/rejected": -2.0855119228363037, + "loss": 2.0272, + "nll_loss": 1.9694753885269165, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18508392572402954, + "rewards/margins": 0.023467278108000755, + "rewards/rejected": -0.20855121314525604, + "step": 129 + }, + { + "epoch": 0.35899206075250256, + "grad_norm": 0.27979782223701477, + "learning_rate": 4.994863481875842e-06, + "log_odds_chosen": 0.539387047290802, + "log_odds_ratio": -0.4670449495315552, + "logits/chosen": 0.19368596374988556, + "logits/rejected": -0.4078698754310608, + "logps/chosen": -1.7874817848205566, + "logps/rejected": -2.2576088905334473, + "loss": 1.9686, + "nll_loss": 1.9218828678131104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17874819040298462, + "rewards/margins": 0.04701270908117294, + "rewards/rejected": -0.22576089203357697, + "step": 130 + }, + { + "epoch": 0.36175353814290645, + "grad_norm": 0.2961781322956085, + "learning_rate": 4.99433718761614e-06, + "log_odds_chosen": 0.482613205909729, + "log_odds_ratio": -0.48607659339904785, + "logits/chosen": 0.3117329776287079, + "logits/rejected": -0.19324824213981628, + "logps/chosen": -1.8348233699798584, + "logps/rejected": -2.2560348510742188, + "loss": 1.9992, + "nll_loss": 1.9505534172058105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1834823340177536, + "rewards/margins": 0.042121127247810364, + "rewards/rejected": -0.22560347616672516, + "step": 131 + }, + { + "epoch": 0.36451501553331034, + "grad_norm": 0.27247706055641174, + "learning_rate": 4.993785260182552e-06, + "log_odds_chosen": 0.5040154457092285, + "log_odds_ratio": -0.47964078187942505, + "logits/chosen": 0.26086652278900146, + "logits/rejected": -0.39971601963043213, + "logps/chosen": -1.7693819999694824, + "logps/rejected": -2.206263780593872, + "loss": 1.9461, + "nll_loss": 1.898113489151001, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17693820595741272, + "rewards/margins": 0.043688178062438965, + "rewards/rejected": -0.2206263691186905, + "step": 132 + }, + { + "epoch": 0.36727649292371417, + "grad_norm": 0.2559458613395691, + "learning_rate": 4.993207705246983e-06, + "log_odds_chosen": 0.37582507729530334, + "log_odds_ratio": -0.529410719871521, + "logits/chosen": 0.24356283247470856, + "logits/rejected": -0.23510105907917023, + "logps/chosen": -1.8250073194503784, + "logps/rejected": -2.1527535915374756, + "loss": 1.9998, + "nll_loss": 1.9468413591384888, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18250073492527008, + "rewards/margins": 0.03277461230754852, + "rewards/rejected": -0.2152753621339798, + "step": 133 + }, + { + "epoch": 0.37003797031411806, + "grad_norm": 0.2661672532558441, + "learning_rate": 4.992604528744705e-06, + "log_odds_chosen": 0.4147174656391144, + "log_odds_ratio": -0.5111517906188965, + "logits/chosen": 0.23473379015922546, + "logits/rejected": -0.5313011407852173, + "logps/chosen": -1.8088786602020264, + "logps/rejected": -2.1657345294952393, + "loss": 1.985, + "nll_loss": 1.9338966608047485, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1808878630399704, + "rewards/margins": 0.035685598850250244, + "rewards/rejected": -0.21657346189022064, + "step": 134 + }, + { + "epoch": 0.37279944770452195, + "grad_norm": 0.26244473457336426, + "learning_rate": 4.9919757368742895e-06, + "log_odds_chosen": 0.4870528280735016, + "log_odds_ratio": -0.4892481565475464, + "logits/chosen": 0.2686365842819214, + "logits/rejected": -0.3376864194869995, + "logps/chosen": -1.7298060655593872, + "logps/rejected": -2.145728349685669, + "loss": 1.9046, + "nll_loss": 1.8556550741195679, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17298059165477753, + "rewards/margins": 0.04159224033355713, + "rewards/rejected": -0.21457283198833466, + "step": 135 + }, + { + "epoch": 0.3755609250949258, + "grad_norm": 0.26679527759552, + "learning_rate": 4.991321336097546e-06, + "log_odds_chosen": 0.4111718535423279, + "log_odds_ratio": -0.512808084487915, + "logits/chosen": 0.22426243126392365, + "logits/rejected": -0.2763628363609314, + "logps/chosen": -1.8473613262176514, + "logps/rejected": -2.2056474685668945, + "loss": 1.9977, + "nll_loss": 1.9464530944824219, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18473613262176514, + "rewards/margins": 0.03582862764596939, + "rewards/rejected": -0.22056476771831512, + "step": 136 + }, + { + "epoch": 0.37832240248532967, + "grad_norm": 0.25576797127723694, + "learning_rate": 4.990641333139455e-06, + "log_odds_chosen": 0.435663640499115, + "log_odds_ratio": -0.5039057731628418, + "logits/chosen": 0.29907894134521484, + "logits/rejected": -0.34149447083473206, + "logps/chosen": -1.7228469848632812, + "logps/rejected": -2.093015670776367, + "loss": 1.8956, + "nll_loss": 1.8452366590499878, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17228470742702484, + "rewards/margins": 0.037016861140728, + "rewards/rejected": -0.20930157601833344, + "step": 137 + }, + { + "epoch": 0.3810838798757335, + "grad_norm": 0.2576519250869751, + "learning_rate": 4.989935734988098e-06, + "log_odds_chosen": 0.32478412985801697, + "log_odds_ratio": -0.5477995872497559, + "logits/chosen": 0.3516974151134491, + "logits/rejected": -0.4254434406757355, + "logps/chosen": -1.8142131567001343, + "logps/rejected": -2.0922889709472656, + "loss": 1.9776, + "nll_loss": 1.922855257987976, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18142130970954895, + "rewards/margins": 0.02780758962035179, + "rewards/rejected": -0.20922890305519104, + "step": 138 + }, + { + "epoch": 0.3838453572661374, + "grad_norm": 0.2815987765789032, + "learning_rate": 4.989204548894589e-06, + "log_odds_chosen": 0.32539641857147217, + "log_odds_ratio": -0.5531569719314575, + "logits/chosen": 0.4096101224422455, + "logits/rejected": -0.19687263667583466, + "logps/chosen": -1.8913159370422363, + "logps/rejected": -2.1725118160247803, + "loss": 2.0515, + "nll_loss": 1.99615478515625, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18913161754608154, + "rewards/margins": 0.028119584545493126, + "rewards/rejected": -0.21725118160247803, + "step": 139 + }, + { + "epoch": 0.3866068346565413, + "grad_norm": 0.25471100211143494, + "learning_rate": 4.988447782372996e-06, + "log_odds_chosen": 0.3162972331047058, + "log_odds_ratio": -0.553735077381134, + "logits/chosen": 0.2287934273481369, + "logits/rejected": -0.36916857957839966, + "logps/chosen": -1.717573642730713, + "logps/rejected": -1.986279010772705, + "loss": 1.8883, + "nll_loss": 1.8329135179519653, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17175735533237457, + "rewards/margins": 0.026870528236031532, + "rewards/rejected": -0.19862788915634155, + "step": 140 + }, + { + "epoch": 0.3893683120469451, + "grad_norm": 0.23676469922065735, + "learning_rate": 4.9876654432002655e-06, + "log_odds_chosen": 0.41759181022644043, + "log_odds_ratio": -0.5150130391120911, + "logits/chosen": 0.3617481589317322, + "logits/rejected": -0.5005238056182861, + "logps/chosen": -1.7590631246566772, + "logps/rejected": -2.120802640914917, + "loss": 1.9118, + "nll_loss": 1.8603452444076538, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17590634524822235, + "rewards/margins": 0.03617396205663681, + "rewards/rejected": -0.21208029985427856, + "step": 141 + }, + { + "epoch": 0.392129789437349, + "grad_norm": 0.23819445073604584, + "learning_rate": 4.986857539416144e-06, + "log_odds_chosen": 0.514154851436615, + "log_odds_ratio": -0.47449907660484314, + "logits/chosen": 0.2892986834049225, + "logits/rejected": -0.5338828563690186, + "logps/chosen": -1.732762336730957, + "logps/rejected": -2.1715476512908936, + "loss": 1.8923, + "nll_loss": 1.8448883295059204, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17327623069286346, + "rewards/margins": 0.04387851804494858, + "rewards/rejected": -0.21715475618839264, + "step": 142 + }, + { + "epoch": 0.39489126682775283, + "grad_norm": 0.2597619891166687, + "learning_rate": 4.986024079323092e-06, + "log_odds_chosen": 0.21202370524406433, + "log_odds_ratio": -0.5989823341369629, + "logits/chosen": 0.31219571828842163, + "logits/rejected": -0.2251828908920288, + "logps/chosen": -1.7001993656158447, + "logps/rejected": -1.8785269260406494, + "loss": 1.9001, + "nll_loss": 1.840247392654419, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17001992464065552, + "rewards/margins": 0.017832759767770767, + "rewards/rejected": -0.18785269558429718, + "step": 143 + }, + { + "epoch": 0.3976527442181567, + "grad_norm": 0.24414244294166565, + "learning_rate": 4.985165071486201e-06, + "log_odds_chosen": 0.42103180289268494, + "log_odds_ratio": -0.506747305393219, + "logits/chosen": 0.24320174753665924, + "logits/rejected": -0.47630739212036133, + "logps/chosen": -1.813495397567749, + "logps/rejected": -2.177159309387207, + "loss": 1.9628, + "nll_loss": 1.9121257066726685, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18134953081607819, + "rewards/margins": 0.03636639565229416, + "rewards/rejected": -0.21771591901779175, + "step": 144 + }, + { + "epoch": 0.4004142216085606, + "grad_norm": 0.25120964646339417, + "learning_rate": 4.984280524733107e-06, + "log_odds_chosen": 0.4766540825366974, + "log_odds_ratio": -0.4867539405822754, + "logits/chosen": 0.23771557211875916, + "logits/rejected": -0.3588354289531708, + "logps/chosen": -1.7543095350265503, + "logps/rejected": -2.1605966091156006, + "loss": 1.92, + "nll_loss": 1.8713701963424683, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17543095350265503, + "rewards/margins": 0.040628716349601746, + "rewards/rejected": -0.21605966985225677, + "step": 145 + }, + { + "epoch": 0.40317569899896444, + "grad_norm": 0.23762984573841095, + "learning_rate": 4.983370448153896e-06, + "log_odds_chosen": 0.3954075574874878, + "log_odds_ratio": -0.5178768038749695, + "logits/chosen": 0.2783900201320648, + "logits/rejected": -0.3363473117351532, + "logps/chosen": -1.7586162090301514, + "logps/rejected": -2.097499370574951, + "loss": 1.9169, + "nll_loss": 1.8651440143585205, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1758616417646408, + "rewards/margins": 0.033888280391693115, + "rewards/rejected": -0.20974992215633392, + "step": 146 + }, + { + "epoch": 0.4059371763893683, + "grad_norm": 0.250926673412323, + "learning_rate": 4.9824348511010115e-06, + "log_odds_chosen": 0.37487757205963135, + "log_odds_ratio": -0.5328046083450317, + "logits/chosen": 0.3899734616279602, + "logits/rejected": -0.3820725679397583, + "logps/chosen": -1.8538525104522705, + "logps/rejected": -2.1806483268737793, + "loss": 2.0058, + "nll_loss": 1.9525387287139893, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18538527190685272, + "rewards/margins": 0.03267957270145416, + "rewards/rejected": -0.2180648297071457, + "step": 147 + }, + { + "epoch": 0.40869865377977216, + "grad_norm": 0.24456721544265747, + "learning_rate": 4.981473743189163e-06, + "log_odds_chosen": 0.6152381300926208, + "log_odds_ratio": -0.4415377974510193, + "logits/chosen": 0.15124794840812683, + "logits/rejected": -0.6728506684303284, + "logps/chosen": -1.630881905555725, + "logps/rejected": -2.1548542976379395, + "loss": 1.8038, + "nll_loss": 1.7596325874328613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16308818757534027, + "rewards/margins": 0.05239725112915039, + "rewards/rejected": -0.21548543870449066, + "step": 148 + }, + { + "epoch": 0.41146013117017605, + "grad_norm": 0.2503897547721863, + "learning_rate": 4.98048713429522e-06, + "log_odds_chosen": 0.4051974415779114, + "log_odds_ratio": -0.5133127570152283, + "logits/chosen": 0.3494684100151062, + "logits/rejected": -0.11723777651786804, + "logps/chosen": -1.7437934875488281, + "logps/rejected": -2.086435079574585, + "loss": 1.9057, + "nll_loss": 1.8543728590011597, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1743793487548828, + "rewards/margins": 0.034264158457517624, + "rewards/rejected": -0.20864351093769073, + "step": 149 + }, + { + "epoch": 0.41422160856057993, + "grad_norm": 0.26401370763778687, + "learning_rate": 4.979475034558115e-06, + "log_odds_chosen": 0.35506629943847656, + "log_odds_ratio": -0.534768283367157, + "logits/chosen": 0.4498825669288635, + "logits/rejected": -0.32746273279190063, + "logps/chosen": -1.8957396745681763, + "logps/rejected": -2.205925464630127, + "loss": 2.042, + "nll_loss": 1.988513469696045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1895739734172821, + "rewards/margins": 0.0310185756534338, + "rewards/rejected": -0.22059254348278046, + "step": 150 + }, + { + "epoch": 0.41698308595098377, + "grad_norm": 0.23593436181545258, + "learning_rate": 4.978437454378741e-06, + "log_odds_chosen": 0.36985495686531067, + "log_odds_ratio": -0.5328302979469299, + "logits/chosen": 0.4250433146953583, + "logits/rejected": -0.38277071714401245, + "logps/chosen": -1.7761732339859009, + "logps/rejected": -2.0909667015075684, + "loss": 1.942, + "nll_loss": 1.8887526988983154, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17761734127998352, + "rewards/margins": 0.03147934749722481, + "rewards/rejected": -0.20909668505191803, + "step": 151 + }, + { + "epoch": 0.41974456334138766, + "grad_norm": 0.26158878207206726, + "learning_rate": 4.977374404419838e-06, + "log_odds_chosen": 0.39601171016693115, + "log_odds_ratio": -0.5228374004364014, + "logits/chosen": 0.42059236764907837, + "logits/rejected": -0.3891626000404358, + "logps/chosen": -1.7959572076797485, + "logps/rejected": -2.140069007873535, + "loss": 1.9499, + "nll_loss": 1.8975740671157837, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1795957237482071, + "rewards/margins": 0.03441117703914642, + "rewards/rejected": -0.21400688588619232, + "step": 152 + }, + { + "epoch": 0.4225060407317915, + "grad_norm": 0.2561565339565277, + "learning_rate": 4.976285895605888e-06, + "log_odds_chosen": 0.4899553656578064, + "log_odds_ratio": -0.4833296239376068, + "logits/chosen": 0.37404921650886536, + "logits/rejected": -0.5243082046508789, + "logps/chosen": -1.7399574518203735, + "logps/rejected": -2.1531152725219727, + "loss": 1.8971, + "nll_loss": 1.8487193584442139, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1739957481622696, + "rewards/margins": 0.04131579399108887, + "rewards/rejected": -0.21531155705451965, + "step": 153 + }, + { + "epoch": 0.4252675181221954, + "grad_norm": 0.2550884485244751, + "learning_rate": 4.9751719391230055e-06, + "log_odds_chosen": 0.25457680225372314, + "log_odds_ratio": -0.5827968120574951, + "logits/chosen": 0.426510751247406, + "logits/rejected": -0.38418132066726685, + "logps/chosen": -1.710974931716919, + "logps/rejected": -1.9265732765197754, + "loss": 1.8985, + "nll_loss": 1.840250015258789, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17109748721122742, + "rewards/margins": 0.02155984938144684, + "rewards/rejected": -0.19265732169151306, + "step": 154 + }, + { + "epoch": 0.42802899551259926, + "grad_norm": 0.25417274236679077, + "learning_rate": 4.974032546418816e-06, + "log_odds_chosen": 0.47524771094322205, + "log_odds_ratio": -0.49181026220321655, + "logits/chosen": 0.43296438455581665, + "logits/rejected": -0.41207337379455566, + "logps/chosen": -1.756546139717102, + "logps/rejected": -2.1634457111358643, + "loss": 1.9365, + "nll_loss": 1.8873004913330078, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17565463483333588, + "rewards/margins": 0.04068994149565697, + "rewards/rejected": -0.21634456515312195, + "step": 155 + }, + { + "epoch": 0.4307904729030031, + "grad_norm": 0.24467967450618744, + "learning_rate": 4.9728677292023405e-06, + "log_odds_chosen": 0.19028525054454803, + "log_odds_ratio": -0.6092196702957153, + "logits/chosen": 0.5236613750457764, + "logits/rejected": -0.3672065734863281, + "logps/chosen": -1.851803183555603, + "logps/rejected": -2.0145559310913086, + "loss": 2.0125, + "nll_loss": 1.9516232013702393, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18518033623695374, + "rewards/margins": 0.016275260597467422, + "rewards/rejected": -0.20145559310913086, + "step": 156 + }, + { + "epoch": 0.433551950293407, + "grad_norm": 0.24561214447021484, + "learning_rate": 4.971677499443882e-06, + "log_odds_chosen": 0.34714600443840027, + "log_odds_ratio": -0.538092315196991, + "logits/chosen": 0.42429813742637634, + "logits/rejected": -0.4320365786552429, + "logps/chosen": -1.7550561428070068, + "logps/rejected": -2.0503358840942383, + "loss": 1.9273, + "nll_loss": 1.873533010482788, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1755056083202362, + "rewards/margins": 0.029527965933084488, + "rewards/rejected": -0.2050335705280304, + "step": 157 + }, + { + "epoch": 0.4363134276838108, + "grad_norm": 0.23480936884880066, + "learning_rate": 4.97046186937489e-06, + "log_odds_chosen": 0.3259715735912323, + "log_odds_ratio": -0.5574356913566589, + "logits/chosen": 0.33684611320495605, + "logits/rejected": -0.48997414112091064, + "logps/chosen": -1.733799934387207, + "logps/rejected": -2.011017084121704, + "loss": 1.8941, + "nll_loss": 1.8383519649505615, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17338000237941742, + "rewards/margins": 0.027721701189875603, + "rewards/rejected": -0.20110172033309937, + "step": 158 + }, + { + "epoch": 0.4390749050742147, + "grad_norm": 0.23975060880184174, + "learning_rate": 4.9692208514878445e-06, + "log_odds_chosen": 0.2344578355550766, + "log_odds_ratio": -0.5858049392700195, + "logits/chosen": 0.4837522804737091, + "logits/rejected": -0.2647075951099396, + "logps/chosen": -1.8640694618225098, + "logps/rejected": -2.064138412475586, + "loss": 2.0361, + "nll_loss": 1.9775654077529907, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1864069551229477, + "rewards/margins": 0.02000689134001732, + "rewards/rejected": -0.2064138650894165, + "step": 159 + }, + { + "epoch": 0.4418363824646186, + "grad_norm": 0.25472894310951233, + "learning_rate": 4.967954458536126e-06, + "log_odds_chosen": 0.34545353055000305, + "log_odds_ratio": -0.5416699051856995, + "logits/chosen": 0.4849855303764343, + "logits/rejected": -0.14890551567077637, + "logps/chosen": -1.6954622268676758, + "logps/rejected": -1.9883880615234375, + "loss": 1.8756, + "nll_loss": 1.821388602256775, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1695462316274643, + "rewards/margins": 0.029292574152350426, + "rewards/rejected": -0.19883880019187927, + "step": 160 + }, + { + "epoch": 0.4445978598550224, + "grad_norm": 0.2551233172416687, + "learning_rate": 4.96666270353388e-06, + "log_odds_chosen": 0.34612518548965454, + "log_odds_ratio": -0.5391160845756531, + "logits/chosen": 0.4446476995944977, + "logits/rejected": -0.4060593545436859, + "logps/chosen": -1.7577486038208008, + "logps/rejected": -2.0534539222717285, + "loss": 1.9068, + "nll_loss": 1.8529114723205566, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17577485740184784, + "rewards/margins": 0.029570531100034714, + "rewards/rejected": -0.20534539222717285, + "step": 161 + }, + { + "epoch": 0.4473593372454263, + "grad_norm": 0.2522425651550293, + "learning_rate": 4.965345599755888e-06, + "log_odds_chosen": 0.3961338400840759, + "log_odds_ratio": -0.520170271396637, + "logits/chosen": 0.38032418489456177, + "logits/rejected": -0.48995065689086914, + "logps/chosen": -1.815320372581482, + "logps/rejected": -2.156792640686035, + "loss": 1.9657, + "nll_loss": 1.9137252569198608, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18153204023838043, + "rewards/margins": 0.0341472253203392, + "rewards/rejected": -0.21567925810813904, + "step": 162 + }, + { + "epoch": 0.45012081463583015, + "grad_norm": 0.23575294017791748, + "learning_rate": 4.964003160737429e-06, + "log_odds_chosen": 0.42496663331985474, + "log_odds_ratio": -0.5144410133361816, + "logits/chosen": 0.4265897274017334, + "logits/rejected": -0.6324371695518494, + "logps/chosen": -1.7239199876785278, + "logps/rejected": -2.0822784900665283, + "loss": 1.8878, + "nll_loss": 1.8363168239593506, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17239199578762054, + "rewards/margins": 0.03583585098385811, + "rewards/rejected": -0.20822784304618835, + "step": 163 + }, + { + "epoch": 0.45288229202623403, + "grad_norm": 0.22830626368522644, + "learning_rate": 4.9626354002741424e-06, + "log_odds_chosen": 0.39489883184432983, + "log_odds_ratio": -0.519772469997406, + "logits/chosen": 0.43731188774108887, + "logits/rejected": -0.5182772874832153, + "logps/chosen": -1.6721280813217163, + "logps/rejected": -2.0018677711486816, + "loss": 1.8343, + "nll_loss": 1.782306432723999, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1672128140926361, + "rewards/margins": 0.032973967492580414, + "rewards/rejected": -0.20018678903579712, + "step": 164 + }, + { + "epoch": 0.4556437694166379, + "grad_norm": 0.25069254636764526, + "learning_rate": 4.9612423324218816e-06, + "log_odds_chosen": 0.4574624300003052, + "log_odds_ratio": -0.4939710795879364, + "logits/chosen": 0.4768182933330536, + "logits/rejected": -0.6430546045303345, + "logps/chosen": -1.8401563167572021, + "logps/rejected": -2.23877215385437, + "loss": 1.9978, + "nll_loss": 1.9484000205993652, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18401561677455902, + "rewards/margins": 0.03986157849431038, + "rewards/rejected": -0.2238771915435791, + "step": 165 + }, + { + "epoch": 0.45840524680704176, + "grad_norm": 0.25569233298301697, + "learning_rate": 4.959823971496575e-06, + "log_odds_chosen": 0.31097978353500366, + "log_odds_ratio": -0.5631955862045288, + "logits/chosen": 0.39253953099250793, + "logits/rejected": -0.3464045226573944, + "logps/chosen": -1.7822060585021973, + "logps/rejected": -2.0535507202148438, + "loss": 1.9515, + "nll_loss": 1.8951623439788818, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17822058498859406, + "rewards/margins": 0.027134478092193604, + "rewards/rejected": -0.20535509288311005, + "step": 166 + }, + { + "epoch": 0.46116672419744564, + "grad_norm": 0.2754010260105133, + "learning_rate": 4.958380332074074e-06, + "log_odds_chosen": 0.38330915570259094, + "log_odds_ratio": -0.5226565003395081, + "logits/chosen": 0.5406702756881714, + "logits/rejected": -0.2918284833431244, + "logps/chosen": -1.744588851928711, + "logps/rejected": -2.0728600025177, + "loss": 1.9256, + "nll_loss": 1.873305320739746, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17445889115333557, + "rewards/margins": 0.03282713145017624, + "rewards/rejected": -0.2072860151529312, + "step": 167 + }, + { + "epoch": 0.4639282015878495, + "grad_norm": 0.24440248310565948, + "learning_rate": 4.95691142899001e-06, + "log_odds_chosen": 0.4813528060913086, + "log_odds_ratio": -0.49559223651885986, + "logits/chosen": 0.42939212918281555, + "logits/rejected": -0.5100030303001404, + "logps/chosen": -1.7333720922470093, + "logps/rejected": -2.148804187774658, + "loss": 1.8988, + "nll_loss": 1.8492889404296875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17333722114562988, + "rewards/margins": 0.04154320806264877, + "rewards/rejected": -0.21488040685653687, + "step": 168 + }, + { + "epoch": 0.46668967897825336, + "grad_norm": 0.24897295236587524, + "learning_rate": 4.955417277339633e-06, + "log_odds_chosen": 0.3721994459629059, + "log_odds_ratio": -0.5288943648338318, + "logits/chosen": 0.4496070444583893, + "logits/rejected": -0.5069053173065186, + "logps/chosen": -1.7618987560272217, + "logps/rejected": -2.0793538093566895, + "loss": 1.9112, + "nll_loss": 1.858320713043213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17618988454341888, + "rewards/margins": 0.03174550086259842, + "rewards/rejected": -0.2079353779554367, + "step": 169 + }, + { + "epoch": 0.46945115636865725, + "grad_norm": 0.24506047368049622, + "learning_rate": 4.953897892477664e-06, + "log_odds_chosen": 0.4158302843570709, + "log_odds_ratio": -0.524972677230835, + "logits/chosen": 0.39215707778930664, + "logits/rejected": -0.6964855790138245, + "logps/chosen": -1.6938952207565308, + "logps/rejected": -2.0564043521881104, + "loss": 1.8573, + "nll_loss": 1.8047559261322021, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1693895161151886, + "rewards/margins": 0.03625092655420303, + "rewards/rejected": -0.20564045011997223, + "step": 170 + }, + { + "epoch": 0.4722126337590611, + "grad_norm": 0.25454598665237427, + "learning_rate": 4.952353290018132e-06, + "log_odds_chosen": 0.39419156312942505, + "log_odds_ratio": -0.5228413939476013, + "logits/chosen": 0.4652084410190582, + "logits/rejected": -0.09862995892763138, + "logps/chosen": -1.7650055885314941, + "logps/rejected": -2.102964162826538, + "loss": 1.9365, + "nll_loss": 1.8842390775680542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17650054395198822, + "rewards/margins": 0.033795878291130066, + "rewards/rejected": -0.2102964073419571, + "step": 171 + }, + { + "epoch": 0.474974111149465, + "grad_norm": 0.23166699707508087, + "learning_rate": 4.950783485834218e-06, + "log_odds_chosen": 0.45087021589279175, + "log_odds_ratio": -0.4937146306037903, + "logits/chosen": 0.43644362688064575, + "logits/rejected": -0.45175108313560486, + "logps/chosen": -1.6984238624572754, + "logps/rejected": -2.0791146755218506, + "loss": 1.8668, + "nll_loss": 1.817420482635498, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16984236240386963, + "rewards/margins": 0.03806909918785095, + "rewards/rejected": -0.20791146159172058, + "step": 172 + }, + { + "epoch": 0.4777355885398688, + "grad_norm": 0.22430026531219482, + "learning_rate": 4.949188496058089e-06, + "log_odds_chosen": 0.3452419340610504, + "log_odds_ratio": -0.5469062328338623, + "logits/chosen": 0.42078280448913574, + "logits/rejected": -0.4557611346244812, + "logps/chosen": -1.6979789733886719, + "logps/rejected": -1.9911997318267822, + "loss": 1.8648, + "nll_loss": 1.8101435899734497, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1697978973388672, + "rewards/margins": 0.029322080314159393, + "rewards/rejected": -0.19911997020244598, + "step": 173 + }, + { + "epoch": 0.4804970659302727, + "grad_norm": 0.25257107615470886, + "learning_rate": 4.947568337080733e-06, + "log_odds_chosen": 0.1399567574262619, + "log_odds_ratio": -0.6351712942123413, + "logits/chosen": 0.3786250352859497, + "logits/rejected": -0.433152973651886, + "logps/chosen": -1.8261394500732422, + "logps/rejected": -1.9481170177459717, + "loss": 1.9906, + "nll_loss": 1.9270497560501099, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.18261395394802094, + "rewards/margins": 0.012197760865092278, + "rewards/rejected": -0.19481170177459717, + "step": 174 + }, + { + "epoch": 0.4832585433206766, + "grad_norm": 0.25380203127861023, + "learning_rate": 4.945923025551789e-06, + "log_odds_chosen": 0.37491294741630554, + "log_odds_ratio": -0.5253455638885498, + "logits/chosen": 0.44007402658462524, + "logits/rejected": -0.5158473253250122, + "logps/chosen": -1.7649321556091309, + "logps/rejected": -2.0859415531158447, + "loss": 1.9061, + "nll_loss": 1.8535852432250977, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17649321258068085, + "rewards/margins": 0.032100923359394073, + "rewards/rejected": -0.20859414339065552, + "step": 175 + }, + { + "epoch": 0.4860200207110804, + "grad_norm": 0.23105277121067047, + "learning_rate": 4.944252578379379e-06, + "log_odds_chosen": 0.2923380434513092, + "log_odds_ratio": -0.5638449788093567, + "logits/chosen": 0.3620964288711548, + "logits/rejected": -0.7222499847412109, + "logps/chosen": -1.6879165172576904, + "logps/rejected": -1.9339208602905273, + "loss": 1.8481, + "nll_loss": 1.7917577028274536, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16879163682460785, + "rewards/margins": 0.024600449949502945, + "rewards/rejected": -0.1933920979499817, + "step": 176 + }, + { + "epoch": 0.4887814981014843, + "grad_norm": 0.2548852264881134, + "learning_rate": 4.942557012729933e-06, + "log_odds_chosen": 0.37703031301498413, + "log_odds_ratio": -0.5374451875686646, + "logits/chosen": 0.451770544052124, + "logits/rejected": -0.6359795331954956, + "logps/chosen": -1.7573479413986206, + "logps/rejected": -2.0781285762786865, + "loss": 1.9276, + "nll_loss": 1.873888373374939, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17573478817939758, + "rewards/margins": 0.03207805007696152, + "rewards/rejected": -0.2078128606081009, + "step": 177 + }, + { + "epoch": 0.49154297549188813, + "grad_norm": 0.24361710250377655, + "learning_rate": 4.940836346028011e-06, + "log_odds_chosen": 0.564251184463501, + "log_odds_ratio": -0.4567793309688568, + "logits/chosen": 0.36331912875175476, + "logits/rejected": -0.6997017860412598, + "logps/chosen": -1.7021052837371826, + "logps/rejected": -2.1804330348968506, + "loss": 1.8632, + "nll_loss": 1.8174842596054077, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17021054029464722, + "rewards/margins": 0.047832753509283066, + "rewards/rejected": -0.2180432826280594, + "step": 178 + }, + { + "epoch": 0.494304452882292, + "grad_norm": 0.25140658020973206, + "learning_rate": 4.9390905959561254e-06, + "log_odds_chosen": 0.42371851205825806, + "log_odds_ratio": -0.5053101778030396, + "logits/chosen": 0.4960322678089142, + "logits/rejected": -0.7143914699554443, + "logps/chosen": -1.8060599565505981, + "logps/rejected": -2.168437957763672, + "loss": 1.9361, + "nll_loss": 1.8855350017547607, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18060600757598877, + "rewards/margins": 0.03623779118061066, + "rewards/rejected": -0.21684379875659943, + "step": 179 + }, + { + "epoch": 0.4970659302726959, + "grad_norm": 0.25267475843429565, + "learning_rate": 4.937319780454559e-06, + "log_odds_chosen": 0.24488888680934906, + "log_odds_ratio": -0.5824177861213684, + "logits/chosen": 0.43725982308387756, + "logits/rejected": -0.5809499621391296, + "logps/chosen": -1.7686806917190552, + "logps/rejected": -1.9778132438659668, + "loss": 1.9319, + "nll_loss": 1.873632550239563, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17686808109283447, + "rewards/margins": 0.020913248881697655, + "rewards/rejected": -0.19778132438659668, + "step": 180 + }, + { + "epoch": 0.49982740766309974, + "grad_norm": 0.2517687976360321, + "learning_rate": 4.935523917721182e-06, + "log_odds_chosen": 0.3661497235298157, + "log_odds_ratio": -0.5325534343719482, + "logits/chosen": 0.4761643409729004, + "logits/rejected": -0.4684900641441345, + "logps/chosen": -1.7682512998580933, + "logps/rejected": -2.079028844833374, + "loss": 1.9513, + "nll_loss": 1.8980205059051514, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1768251359462738, + "rewards/margins": 0.031077751889824867, + "rewards/rejected": -0.20790287852287292, + "step": 181 + }, + { + "epoch": 0.5025888850535036, + "grad_norm": 0.23882247507572174, + "learning_rate": 4.933703026211262e-06, + "log_odds_chosen": 0.40052229166030884, + "log_odds_ratio": -0.5199273824691772, + "logits/chosen": 0.5116904377937317, + "logits/rejected": -0.8737332820892334, + "logps/chosen": -1.7753740549087524, + "logps/rejected": -2.1211981773376465, + "loss": 1.9275, + "nll_loss": 1.8754714727401733, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17753739655017853, + "rewards/margins": 0.03458239883184433, + "rewards/rejected": -0.21211980283260345, + "step": 182 + }, + { + "epoch": 0.5053503624439075, + "grad_norm": 0.23901337385177612, + "learning_rate": 4.931857124637276e-06, + "log_odds_chosen": 0.3831828832626343, + "log_odds_ratio": -0.5294877886772156, + "logits/chosen": 0.44559329748153687, + "logits/rejected": -0.512153685092926, + "logps/chosen": -1.660269021987915, + "logps/rejected": -1.9844969511032104, + "loss": 1.8228, + "nll_loss": 1.7698148488998413, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16602689027786255, + "rewards/margins": 0.032422810792922974, + "rewards/rejected": -0.19844970107078552, + "step": 183 + }, + { + "epoch": 0.5081118398343114, + "grad_norm": 0.26273974776268005, + "learning_rate": 4.92998623196872e-06, + "log_odds_chosen": 0.40838003158569336, + "log_odds_ratio": -0.5120916366577148, + "logits/chosen": 0.36793801188468933, + "logits/rejected": -0.883270263671875, + "logps/chosen": -1.7183860540390015, + "logps/rejected": -2.0646042823791504, + "loss": 1.8679, + "nll_loss": 1.8166720867156982, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17183861136436462, + "rewards/margins": 0.034621842205524445, + "rewards/rejected": -0.20646044611930847, + "step": 184 + }, + { + "epoch": 0.5108733172247152, + "grad_norm": 0.24954968690872192, + "learning_rate": 4.92809036743191e-06, + "log_odds_chosen": 0.44134509563446045, + "log_odds_ratio": -0.5009733438491821, + "logits/chosen": 0.38025662302970886, + "logits/rejected": -0.6959909200668335, + "logps/chosen": -1.6237648725509644, + "logps/rejected": -1.991347312927246, + "loss": 1.7726, + "nll_loss": 1.7225314378738403, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1623764932155609, + "rewards/margins": 0.03675824776291847, + "rewards/rejected": -0.19913475215435028, + "step": 185 + }, + { + "epoch": 0.5136347946151191, + "grad_norm": 0.24785040318965912, + "learning_rate": 4.926169550509787e-06, + "log_odds_chosen": 0.304913192987442, + "log_odds_ratio": -0.5551115274429321, + "logits/chosen": 0.49904897809028625, + "logits/rejected": -0.5060792565345764, + "logps/chosen": -1.752536416053772, + "logps/rejected": -2.009927749633789, + "loss": 1.8926, + "nll_loss": 1.837074875831604, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17525365948677063, + "rewards/margins": 0.0257391519844532, + "rewards/rejected": -0.20099279284477234, + "step": 186 + }, + { + "epoch": 0.516396272005523, + "grad_norm": 0.25260496139526367, + "learning_rate": 4.924223800941718e-06, + "log_odds_chosen": 0.18274395167827606, + "log_odds_ratio": -0.6112073659896851, + "logits/chosen": 0.4272249937057495, + "logits/rejected": -0.5964298248291016, + "logps/chosen": -1.7245608568191528, + "logps/rejected": -1.87729012966156, + "loss": 1.8858, + "nll_loss": 1.824722170829773, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17245608568191528, + "rewards/margins": 0.015272947028279305, + "rewards/rejected": -0.18772903084754944, + "step": 187 + }, + { + "epoch": 0.5191577493959268, + "grad_norm": 0.25974321365356445, + "learning_rate": 4.9222531387232885e-06, + "log_odds_chosen": 0.2772579491138458, + "log_odds_ratio": -0.5700020790100098, + "logits/chosen": 0.5035889148712158, + "logits/rejected": -0.6021788120269775, + "logps/chosen": -1.810874581336975, + "logps/rejected": -2.048560380935669, + "loss": 1.9601, + "nll_loss": 1.903147578239441, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.181087464094162, + "rewards/margins": 0.023768583312630653, + "rewards/rejected": -0.2048560529947281, + "step": 188 + }, + { + "epoch": 0.5219192267863307, + "grad_norm": 0.2509472668170929, + "learning_rate": 4.920257584106104e-06, + "log_odds_chosen": 0.2922664284706116, + "log_odds_ratio": -0.5610611438751221, + "logits/chosen": 0.4223700761795044, + "logits/rejected": -0.8161606192588806, + "logps/chosen": -1.7685585021972656, + "logps/rejected": -2.017383575439453, + "loss": 1.9162, + "nll_loss": 1.8600515127182007, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17685584723949432, + "rewards/margins": 0.02488252893090248, + "rewards/rejected": -0.2017383873462677, + "step": 189 + }, + { + "epoch": 0.5246807041767345, + "grad_norm": 0.26333701610565186, + "learning_rate": 4.918237157597574e-06, + "log_odds_chosen": 0.367781400680542, + "log_odds_ratio": -0.5288804769515991, + "logits/chosen": 0.34821969270706177, + "logits/rejected": -0.5903292894363403, + "logps/chosen": -1.7087079286575317, + "logps/rejected": -2.0187838077545166, + "loss": 1.8792, + "nll_loss": 1.8263163566589355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1708707958459854, + "rewards/margins": 0.03100760281085968, + "rewards/rejected": -0.2018783837556839, + "step": 190 + }, + { + "epoch": 0.5274421815671384, + "grad_norm": 0.2659012973308563, + "learning_rate": 4.916191879960708e-06, + "log_odds_chosen": 0.41825154423713684, + "log_odds_ratio": -0.5096725225448608, + "logits/chosen": 0.3918278217315674, + "logits/rejected": -0.7869745492935181, + "logps/chosen": -1.819606900215149, + "logps/rejected": -2.178121328353882, + "loss": 1.9644, + "nll_loss": 1.9134495258331299, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18196068704128265, + "rewards/margins": 0.03585144132375717, + "rewards/rejected": -0.21781213581562042, + "step": 191 + }, + { + "epoch": 0.5302036589575423, + "grad_norm": 0.2279636263847351, + "learning_rate": 4.914121772213898e-06, + "log_odds_chosen": 0.3906615972518921, + "log_odds_ratio": -0.5185546875, + "logits/chosen": 0.4513569176197052, + "logits/rejected": -0.5819852948188782, + "logps/chosen": -1.6937000751495361, + "logps/rejected": -2.0224616527557373, + "loss": 1.8302, + "nll_loss": 1.778322458267212, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16937001049518585, + "rewards/margins": 0.03287617862224579, + "rewards/rejected": -0.20224617421627045, + "step": 192 + }, + { + "epoch": 0.5329651363479462, + "grad_norm": 0.25850534439086914, + "learning_rate": 4.912026855630703e-06, + "log_odds_chosen": 0.4198724031448364, + "log_odds_ratio": -0.5083851218223572, + "logits/chosen": 0.44823789596557617, + "logits/rejected": -0.45316439867019653, + "logps/chosen": -1.7062734365463257, + "logps/rejected": -2.061528205871582, + "loss": 1.8534, + "nll_loss": 1.8025155067443848, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17062735557556152, + "rewards/margins": 0.035525478422641754, + "rewards/rejected": -0.20615282654762268, + "step": 193 + }, + { + "epoch": 0.5357266137383501, + "grad_norm": 0.2361551970243454, + "learning_rate": 4.909907151739634e-06, + "log_odds_chosen": 0.3865026831626892, + "log_odds_ratio": -0.527399480342865, + "logits/chosen": 0.3759016692638397, + "logits/rejected": -0.6874773502349854, + "logps/chosen": -1.6837282180786133, + "logps/rejected": -2.00811505317688, + "loss": 1.8406, + "nll_loss": 1.7878108024597168, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16837282478809357, + "rewards/margins": 0.03243869170546532, + "rewards/rejected": -0.200811505317688, + "step": 194 + }, + { + "epoch": 0.5384880911287538, + "grad_norm": 0.25059905648231506, + "learning_rate": 4.907762682323926e-06, + "log_odds_chosen": 0.5617655515670776, + "log_odds_ratio": -0.4947778284549713, + "logits/chosen": 0.4162059426307678, + "logits/rejected": -0.6656520366668701, + "logps/chosen": -1.6942963600158691, + "logps/rejected": -2.1570441722869873, + "loss": 1.8694, + "nll_loss": 1.8199554681777954, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16942965984344482, + "rewards/margins": 0.046274758875370026, + "rewards/rejected": -0.21570439636707306, + "step": 195 + }, + { + "epoch": 0.5412495685191577, + "grad_norm": 0.24844138324260712, + "learning_rate": 4.905593469421323e-06, + "log_odds_chosen": 0.2864172160625458, + "log_odds_ratio": -0.5658568143844604, + "logits/chosen": 0.40823209285736084, + "logits/rejected": -1.0584533214569092, + "logps/chosen": -1.6995760202407837, + "logps/rejected": -1.9419208765029907, + "loss": 1.8573, + "nll_loss": 1.8007633686065674, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16995760798454285, + "rewards/margins": 0.024234486743807793, + "rewards/rejected": -0.1941920816898346, + "step": 196 + }, + { + "epoch": 0.5440110459095616, + "grad_norm": 0.25183677673339844, + "learning_rate": 4.90339953532384e-06, + "log_odds_chosen": 0.32565394043922424, + "log_odds_ratio": -0.5478559732437134, + "logits/chosen": 0.4504649341106415, + "logits/rejected": -0.6915105581283569, + "logps/chosen": -1.723501443862915, + "logps/rejected": -1.9977482557296753, + "loss": 1.8793, + "nll_loss": 1.8244800567626953, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17235015332698822, + "rewards/margins": 0.027424685657024384, + "rewards/rejected": -0.199774831533432, + "step": 197 + }, + { + "epoch": 0.5467725232999655, + "grad_norm": 0.24120980501174927, + "learning_rate": 4.901180902577549e-06, + "log_odds_chosen": 0.36884185671806335, + "log_odds_ratio": -0.5313685536384583, + "logits/chosen": 0.47393080592155457, + "logits/rejected": -0.8454681038856506, + "logps/chosen": -1.7091484069824219, + "logps/rejected": -2.021761655807495, + "loss": 1.8562, + "nll_loss": 1.80307936668396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17091482877731323, + "rewards/margins": 0.03126133233308792, + "rewards/rejected": -0.20217616856098175, + "step": 198 + }, + { + "epoch": 0.5495340006903694, + "grad_norm": 0.26695704460144043, + "learning_rate": 4.8989375939823305e-06, + "log_odds_chosen": 0.2716186046600342, + "log_odds_ratio": -0.5816208124160767, + "logits/chosen": 0.43514788150787354, + "logits/rejected": -0.9399389028549194, + "logps/chosen": -1.6752939224243164, + "logps/rejected": -1.9062312841415405, + "loss": 1.829, + "nll_loss": 1.770880937576294, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1675294041633606, + "rewards/margins": 0.023093728348612785, + "rewards/rejected": -0.19062313437461853, + "step": 199 + }, + { + "epoch": 0.5522954780807732, + "grad_norm": 0.248373344540596, + "learning_rate": 4.896669632591652e-06, + "log_odds_chosen": 0.43290525674819946, + "log_odds_ratio": -0.5045643448829651, + "logits/chosen": 0.41075634956359863, + "logits/rejected": -0.6819908618927002, + "logps/chosen": -1.6926066875457764, + "logps/rejected": -2.0592057704925537, + "loss": 1.8492, + "nll_loss": 1.7987439632415771, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1692606657743454, + "rewards/margins": 0.03665991127490997, + "rewards/rejected": -0.20592057704925537, + "step": 200 + }, + { + "epoch": 0.5550569554711771, + "grad_norm": 0.2426389902830124, + "learning_rate": 4.894377041712327e-06, + "log_odds_chosen": 0.3046472668647766, + "log_odds_ratio": -0.5560302138328552, + "logits/chosen": 0.4586237370967865, + "logits/rejected": -0.889277458190918, + "logps/chosen": -1.7483210563659668, + "logps/rejected": -2.0069689750671387, + "loss": 1.8995, + "nll_loss": 1.8438801765441895, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17483210563659668, + "rewards/margins": 0.025864800438284874, + "rewards/rejected": -0.2006969153881073, + "step": 201 + }, + { + "epoch": 0.557818432861581, + "grad_norm": 0.2384442389011383, + "learning_rate": 4.892059844904273e-06, + "log_odds_chosen": 0.35407793521881104, + "log_odds_ratio": -0.5373241901397705, + "logits/chosen": 0.3391227722167969, + "logits/rejected": -0.8238649964332581, + "logps/chosen": -1.6727503538131714, + "logps/rejected": -1.971817135810852, + "loss": 1.8314, + "nll_loss": 1.7776691913604736, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16727504134178162, + "rewards/margins": 0.02990666963160038, + "rewards/rejected": -0.19718170166015625, + "step": 202 + }, + { + "epoch": 0.5605799102519848, + "grad_norm": 0.2237497717142105, + "learning_rate": 4.889718065980272e-06, + "log_odds_chosen": 0.42560863494873047, + "log_odds_ratio": -0.5102251768112183, + "logits/chosen": 0.38399800658226013, + "logits/rejected": -0.940986156463623, + "logps/chosen": -1.7369155883789062, + "logps/rejected": -2.0991053581237793, + "loss": 1.8724, + "nll_loss": 1.821379542350769, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1736915558576584, + "rewards/margins": 0.03621895611286163, + "rewards/rejected": -0.2099105268716812, + "step": 203 + }, + { + "epoch": 0.5633413876423887, + "grad_norm": 0.23220422863960266, + "learning_rate": 4.8873517290057265e-06, + "log_odds_chosen": 0.5465662479400635, + "log_odds_ratio": -0.45910075306892395, + "logits/chosen": 0.33986854553222656, + "logits/rejected": -0.9626595973968506, + "logps/chosen": -1.7113467454910278, + "logps/rejected": -2.1808066368103027, + "loss": 1.8594, + "nll_loss": 1.8135318756103516, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17113468050956726, + "rewards/margins": 0.046945974230766296, + "rewards/rejected": -0.21808065474033356, + "step": 204 + }, + { + "epoch": 0.5661028650327925, + "grad_norm": 0.22727084159851074, + "learning_rate": 4.88496085829841e-06, + "log_odds_chosen": 0.3183567523956299, + "log_odds_ratio": -0.5485174059867859, + "logits/chosen": 0.38077837228775024, + "logits/rejected": -0.3718608617782593, + "logps/chosen": -1.682438611984253, + "logps/rejected": -1.9494574069976807, + "loss": 1.8312, + "nll_loss": 1.7763221263885498, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.168243870139122, + "rewards/margins": 0.026701876893639565, + "rewards/rejected": -0.19494575262069702, + "step": 205 + }, + { + "epoch": 0.5688643424231964, + "grad_norm": 0.2435847967863083, + "learning_rate": 4.882545478428219e-06, + "log_odds_chosen": 0.4757111072540283, + "log_odds_ratio": -0.5007758140563965, + "logits/chosen": 0.4316224753856659, + "logits/rejected": -0.9769002199172974, + "logps/chosen": -1.6722346544265747, + "logps/rejected": -2.073765277862549, + "loss": 1.8362, + "nll_loss": 1.7861530780792236, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1672234833240509, + "rewards/margins": 0.04015304520726204, + "rewards/rejected": -0.20737652480602264, + "step": 206 + }, + { + "epoch": 0.5716258198136003, + "grad_norm": 0.28022557497024536, + "learning_rate": 4.880105614216917e-06, + "log_odds_chosen": 0.42549797892570496, + "log_odds_ratio": -0.507975161075592, + "logits/chosen": 0.42804068326950073, + "logits/rejected": -0.8541020750999451, + "logps/chosen": -1.8415427207946777, + "logps/rejected": -2.209498405456543, + "loss": 1.9733, + "nll_loss": 1.9224536418914795, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18415425717830658, + "rewards/margins": 0.03679555654525757, + "rewards/rejected": -0.22094982862472534, + "step": 207 + }, + { + "epoch": 0.5743872972040042, + "grad_norm": 0.26431816816329956, + "learning_rate": 4.8776412907378845e-06, + "log_odds_chosen": 0.3722533583641052, + "log_odds_ratio": -0.5265185832977295, + "logits/chosen": 0.46087294816970825, + "logits/rejected": -0.8973901271820068, + "logps/chosen": -1.7529377937316895, + "logps/rejected": -2.0698721408843994, + "loss": 1.8939, + "nll_loss": 1.8412883281707764, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17529378831386566, + "rewards/margins": 0.03169342502951622, + "rewards/rejected": -0.206987202167511, + "step": 208 + }, + { + "epoch": 0.577148774594408, + "grad_norm": 0.2286507785320282, + "learning_rate": 4.875152533315859e-06, + "log_odds_chosen": 0.42754417657852173, + "log_odds_ratio": -0.5049228668212891, + "logits/chosen": 0.3295610547065735, + "logits/rejected": -0.821418285369873, + "logps/chosen": -1.6234990358352661, + "logps/rejected": -1.9790537357330322, + "loss": 1.7729, + "nll_loss": 1.7223597764968872, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1623499095439911, + "rewards/margins": 0.035555459558963776, + "rewards/rejected": -0.19790537655353546, + "step": 209 + }, + { + "epoch": 0.5799102519848118, + "grad_norm": 0.23942524194717407, + "learning_rate": 4.872639367526672e-06, + "log_odds_chosen": 0.4114229083061218, + "log_odds_ratio": -0.5152523517608643, + "logits/chosen": 0.3920520842075348, + "logits/rejected": -0.7173675298690796, + "logps/chosen": -1.6646122932434082, + "logps/rejected": -2.0126848220825195, + "loss": 1.8132, + "nll_loss": 1.7616626024246216, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16646124422550201, + "rewards/margins": 0.0348072350025177, + "rewards/rejected": -0.20126846432685852, + "step": 210 + }, + { + "epoch": 0.5826717293752157, + "grad_norm": 0.259086012840271, + "learning_rate": 4.870101819196992e-06, + "log_odds_chosen": 0.37038204073905945, + "log_odds_ratio": -0.5485732555389404, + "logits/chosen": 0.33591514825820923, + "logits/rejected": -0.7169915437698364, + "logps/chosen": -1.7096450328826904, + "logps/rejected": -2.028204917907715, + "loss": 1.863, + "nll_loss": 1.8081302642822266, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17096450924873352, + "rewards/margins": 0.031855978071689606, + "rewards/rejected": -0.20282049477100372, + "step": 211 + }, + { + "epoch": 0.5854332067656196, + "grad_norm": 0.23236605525016785, + "learning_rate": 4.8675399144040535e-06, + "log_odds_chosen": 0.48778051137924194, + "log_odds_ratio": -0.4863608777523041, + "logits/chosen": 0.3406648337841034, + "logits/rejected": -1.1020050048828125, + "logps/chosen": -1.6164040565490723, + "logps/rejected": -2.0246694087982178, + "loss": 1.7551, + "nll_loss": 1.7064510583877563, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16164040565490723, + "rewards/margins": 0.04082653671503067, + "rewards/rejected": -0.2024669200181961, + "step": 212 + }, + { + "epoch": 0.5881946841560235, + "grad_norm": 0.2472049593925476, + "learning_rate": 4.864953679475392e-06, + "log_odds_chosen": 0.4090927541255951, + "log_odds_ratio": -0.5185278654098511, + "logits/chosen": 0.3806914687156677, + "logits/rejected": -1.1645220518112183, + "logps/chosen": -1.6705214977264404, + "logps/rejected": -2.010647773742676, + "loss": 1.7979, + "nll_loss": 1.7460622787475586, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16705216467380524, + "rewards/margins": 0.03401262313127518, + "rewards/rejected": -0.201064795255661, + "step": 213 + }, + { + "epoch": 0.5909561615464274, + "grad_norm": 0.26932069659233093, + "learning_rate": 4.862343140988573e-06, + "log_odds_chosen": 0.38164323568344116, + "log_odds_ratio": -0.5293680429458618, + "logits/chosen": 0.35298576951026917, + "logits/rejected": -0.9924853444099426, + "logps/chosen": -1.7351062297821045, + "logps/rejected": -2.062347173690796, + "loss": 1.8652, + "nll_loss": 1.8122597932815552, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1735106110572815, + "rewards/margins": 0.03272408992052078, + "rewards/rejected": -0.20623470842838287, + "step": 214 + }, + { + "epoch": 0.5937176389368312, + "grad_norm": 0.26750069856643677, + "learning_rate": 4.859708325770919e-06, + "log_odds_chosen": 0.3453901410102844, + "log_odds_ratio": -0.5410705804824829, + "logits/chosen": 0.402724027633667, + "logits/rejected": -0.7651013135910034, + "logps/chosen": -1.736122727394104, + "logps/rejected": -2.027735948562622, + "loss": 1.901, + "nll_loss": 1.8469077348709106, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17361226677894592, + "rewards/margins": 0.029161330312490463, + "rewards/rejected": -0.20277361571788788, + "step": 215 + }, + { + "epoch": 0.596479116327235, + "grad_norm": 0.25545212626457214, + "learning_rate": 4.857049260899233e-06, + "log_odds_chosen": 0.42634040117263794, + "log_odds_ratio": -0.5089380145072937, + "logits/chosen": 0.2999016046524048, + "logits/rejected": -0.9428575038909912, + "logps/chosen": -1.6035995483398438, + "logps/rejected": -1.9544857740402222, + "loss": 1.7566, + "nll_loss": 1.7056996822357178, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1603599637746811, + "rewards/margins": 0.03508862853050232, + "rewards/rejected": -0.19544857740402222, + "step": 216 + }, + { + "epoch": 0.5992405937176389, + "grad_norm": 0.2341543734073639, + "learning_rate": 4.854365973699519e-06, + "log_odds_chosen": 0.29267483949661255, + "log_odds_ratio": -0.5718191862106323, + "logits/chosen": 0.32583779096603394, + "logits/rejected": -0.7815302610397339, + "logps/chosen": -1.6951828002929688, + "logps/rejected": -1.9353394508361816, + "loss": 1.8596, + "nll_loss": 1.8024464845657349, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16951829195022583, + "rewards/margins": 0.024015672504901886, + "rewards/rejected": -0.19353395700454712, + "step": 217 + }, + { + "epoch": 0.6020020711080428, + "grad_norm": 0.24366186559200287, + "learning_rate": 4.851658491746707e-06, + "log_odds_chosen": 0.3909580707550049, + "log_odds_ratio": -0.5210633277893066, + "logits/chosen": 0.25465232133865356, + "logits/rejected": -0.9531072378158569, + "logps/chosen": -1.6791120767593384, + "logps/rejected": -2.0105059146881104, + "loss": 1.8187, + "nll_loss": 1.7665891647338867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16791123151779175, + "rewards/margins": 0.03313935548067093, + "rewards/rejected": -0.20105057954788208, + "step": 218 + }, + { + "epoch": 0.6047635484984467, + "grad_norm": 0.24001233279705048, + "learning_rate": 4.848926842864361e-06, + "log_odds_chosen": 0.4899923503398895, + "log_odds_ratio": -0.48127448558807373, + "logits/chosen": 0.25574028491973877, + "logits/rejected": -1.256288766860962, + "logps/chosen": -1.6502352952957153, + "logps/rejected": -2.064323902130127, + "loss": 1.7891, + "nll_loss": 1.740965723991394, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.165023535490036, + "rewards/margins": 0.04140886664390564, + "rewards/rejected": -0.20643240213394165, + "step": 219 + }, + { + "epoch": 0.6075250258888505, + "grad_norm": 0.23009978234767914, + "learning_rate": 4.846171055124401e-06, + "log_odds_chosen": 0.3280143141746521, + "log_odds_ratio": -0.55921870470047, + "logits/chosen": 0.35827893018722534, + "logits/rejected": -0.7289958000183105, + "logps/chosen": -1.7372446060180664, + "logps/rejected": -2.02004075050354, + "loss": 1.8677, + "nll_loss": 1.811750888824463, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1737244725227356, + "rewards/margins": 0.028279609978199005, + "rewards/rejected": -0.2020040899515152, + "step": 220 + }, + { + "epoch": 0.6102865032792544, + "grad_norm": 0.24643385410308838, + "learning_rate": 4.843391156846811e-06, + "log_odds_chosen": 0.49847206473350525, + "log_odds_ratio": -0.4771438539028168, + "logits/chosen": 0.38028058409690857, + "logits/rejected": -0.8463162779808044, + "logps/chosen": -1.6676466464996338, + "logps/rejected": -2.0871989727020264, + "loss": 1.8045, + "nll_loss": 1.756779432296753, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16676466166973114, + "rewards/margins": 0.04195523262023926, + "rewards/rejected": -0.2087198942899704, + "step": 221 + }, + { + "epoch": 0.6130479806696583, + "grad_norm": 0.23624517023563385, + "learning_rate": 4.8405871765993435e-06, + "log_odds_chosen": 0.5461086630821228, + "log_odds_ratio": -0.4661559462547302, + "logits/chosen": 0.2758100926876068, + "logits/rejected": -0.9698406457901001, + "logps/chosen": -1.716646671295166, + "logps/rejected": -2.178037643432617, + "loss": 1.8539, + "nll_loss": 1.807306170463562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17166468501091003, + "rewards/margins": 0.04613909870386124, + "rewards/rejected": -0.21780376136302948, + "step": 222 + }, + { + "epoch": 0.6158094580600622, + "grad_norm": 0.24216988682746887, + "learning_rate": 4.837759143197237e-06, + "log_odds_chosen": 0.33277636766433716, + "log_odds_ratio": -0.543420672416687, + "logits/chosen": 0.25846540927886963, + "logits/rejected": -1.1584213972091675, + "logps/chosen": -1.6766406297683716, + "logps/rejected": -1.959399700164795, + "loss": 1.818, + "nll_loss": 1.7636725902557373, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1676640659570694, + "rewards/margins": 0.028275907039642334, + "rewards/rejected": -0.19593995809555054, + "step": 223 + }, + { + "epoch": 0.618570935450466, + "grad_norm": 0.22247816622257233, + "learning_rate": 4.834907085702909e-06, + "log_odds_chosen": 0.42663368582725525, + "log_odds_ratio": -0.5054609775543213, + "logits/chosen": 0.2946387827396393, + "logits/rejected": -0.8483214378356934, + "logps/chosen": -1.6203222274780273, + "logps/rejected": -1.9752922058105469, + "loss": 1.7654, + "nll_loss": 1.7148088216781616, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16203221678733826, + "rewards/margins": 0.03549701347947121, + "rewards/rejected": -0.19752921164035797, + "step": 224 + }, + { + "epoch": 0.6213324128408698, + "grad_norm": 0.23438099026679993, + "learning_rate": 4.832031033425663e-06, + "log_odds_chosen": 0.4974308907985687, + "log_odds_ratio": -0.4770738184452057, + "logits/chosen": 0.22860127687454224, + "logits/rejected": -1.1651134490966797, + "logps/chosen": -1.5770584344863892, + "logps/rejected": -1.98891282081604, + "loss": 1.7128, + "nll_loss": 1.6650840044021606, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15770584344863892, + "rewards/margins": 0.04118544980883598, + "rewards/rejected": -0.198891282081604, + "step": 225 + }, + { + "epoch": 0.6240938902312737, + "grad_norm": 0.2453928291797638, + "learning_rate": 4.829131015921386e-06, + "log_odds_chosen": 0.336994469165802, + "log_odds_ratio": -0.5416135787963867, + "logits/chosen": 0.40966925024986267, + "logits/rejected": -0.7484245300292969, + "logps/chosen": -1.7475402355194092, + "logps/rejected": -2.035691499710083, + "loss": 1.8881, + "nll_loss": 1.833910346031189, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17475402355194092, + "rewards/margins": 0.0288151316344738, + "rewards/rejected": -0.2035691738128662, + "step": 226 + }, + { + "epoch": 0.6268553676216776, + "grad_norm": 0.22821033000946045, + "learning_rate": 4.826207062992245e-06, + "log_odds_chosen": 0.47645366191864014, + "log_odds_ratio": -0.4914059638977051, + "logits/chosen": 0.2782054543495178, + "logits/rejected": -0.9900674223899841, + "logps/chosen": -1.8174843788146973, + "logps/rejected": -2.2320914268493652, + "loss": 1.95, + "nll_loss": 1.9008376598358154, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1817484349012375, + "rewards/margins": 0.041460707783699036, + "rewards/rejected": -0.22320915758609772, + "step": 227 + }, + { + "epoch": 0.6296168450120815, + "grad_norm": 0.2569068968296051, + "learning_rate": 4.82325920468638e-06, + "log_odds_chosen": 0.2413053661584854, + "log_odds_ratio": -0.5858334302902222, + "logits/chosen": 0.32741737365722656, + "logits/rejected": -0.9221272468566895, + "logps/chosen": -1.7791626453399658, + "logps/rejected": -1.9837150573730469, + "loss": 1.9203, + "nll_loss": 1.8617504835128784, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1779162585735321, + "rewards/margins": 0.020455272868275642, + "rewards/rejected": -0.1983715444803238, + "step": 228 + }, + { + "epoch": 0.6323783224024854, + "grad_norm": 0.2548074424266815, + "learning_rate": 4.820287471297598e-06, + "log_odds_chosen": 0.4002269208431244, + "log_odds_ratio": -0.5239609479904175, + "logits/chosen": 0.21428313851356506, + "logits/rejected": -0.9460724592208862, + "logps/chosen": -1.6473997831344604, + "logps/rejected": -1.9846879243850708, + "loss": 1.7941, + "nll_loss": 1.7416696548461914, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1647399663925171, + "rewards/margins": 0.03372883051633835, + "rewards/rejected": -0.19846880435943604, + "step": 229 + }, + { + "epoch": 0.6351397997928891, + "grad_norm": 0.22254504263401031, + "learning_rate": 4.817291893365055e-06, + "log_odds_chosen": 0.46622079610824585, + "log_odds_ratio": -0.4967763423919678, + "logits/chosen": 0.07556484639644623, + "logits/rejected": -1.116629719734192, + "logps/chosen": -1.5138487815856934, + "logps/rejected": -1.8917471170425415, + "loss": 1.675, + "nll_loss": 1.625287413597107, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15138490498065948, + "rewards/margins": 0.03778982535004616, + "rewards/rejected": -0.18917471170425415, + "step": 230 + }, + { + "epoch": 0.637901277183293, + "grad_norm": 0.28197240829467773, + "learning_rate": 4.81427250167295e-06, + "log_odds_chosen": 0.24183571338653564, + "log_odds_ratio": -0.583247721195221, + "logits/chosen": 0.15436850488185883, + "logits/rejected": -1.1790400743484497, + "logps/chosen": -1.6747009754180908, + "logps/rejected": -1.8760582208633423, + "loss": 1.8184, + "nll_loss": 1.7600996494293213, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16747009754180908, + "rewards/margins": 0.020135723054409027, + "rewards/rejected": -0.1876058131456375, + "step": 231 + }, + { + "epoch": 0.6406627545736969, + "grad_norm": 0.24836094677448273, + "learning_rate": 4.811229327250204e-06, + "log_odds_chosen": 0.45771628618240356, + "log_odds_ratio": -0.5010443329811096, + "logits/chosen": 0.25003117322921753, + "logits/rejected": -1.1424946784973145, + "logps/chosen": -1.714363694190979, + "logps/rejected": -2.1092028617858887, + "loss": 1.8544, + "nll_loss": 1.8042795658111572, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1714363694190979, + "rewards/margins": 0.03948391228914261, + "rewards/rejected": -0.2109202891588211, + "step": 232 + }, + { + "epoch": 0.6434242319641008, + "grad_norm": 0.23518826067447662, + "learning_rate": 4.8081624013701435e-06, + "log_odds_chosen": 0.4311872124671936, + "log_odds_ratio": -0.5104647278785706, + "logits/chosen": 0.1991138905286789, + "logits/rejected": -1.3669785261154175, + "logps/chosen": -1.6888374090194702, + "logps/rejected": -2.053189992904663, + "loss": 1.8231, + "nll_loss": 1.7720434665679932, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16888374090194702, + "rewards/margins": 0.03643525391817093, + "rewards/rejected": -0.20531900227069855, + "step": 233 + }, + { + "epoch": 0.6461857093545047, + "grad_norm": 0.23462118208408356, + "learning_rate": 4.805071755550177e-06, + "log_odds_chosen": 0.394045889377594, + "log_odds_ratio": -0.5184462070465088, + "logits/chosen": 0.2896556854248047, + "logits/rejected": -1.3101093769073486, + "logps/chosen": -1.713099479675293, + "logps/rejected": -2.048048496246338, + "loss": 1.8398, + "nll_loss": 1.7879210710525513, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1713099479675293, + "rewards/margins": 0.03349488973617554, + "rewards/rejected": -0.20480485260486603, + "step": 234 + }, + { + "epoch": 0.6489471867449085, + "grad_norm": 0.23507662117481232, + "learning_rate": 4.8019574215514705e-06, + "log_odds_chosen": 0.3388752043247223, + "log_odds_ratio": -0.5402787327766418, + "logits/chosen": 0.20577961206436157, + "logits/rejected": -0.8569263219833374, + "logps/chosen": -1.6788839101791382, + "logps/rejected": -1.9616367816925049, + "loss": 1.8351, + "nll_loss": 1.7810907363891602, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16788838803768158, + "rewards/margins": 0.028275297954678535, + "rewards/rejected": -0.19616368412971497, + "step": 235 + }, + { + "epoch": 0.6517086641353124, + "grad_norm": 0.2070939689874649, + "learning_rate": 4.7988194313786275e-06, + "log_odds_chosen": 0.37065887451171875, + "log_odds_ratio": -0.5329977869987488, + "logits/chosen": 0.22679734230041504, + "logits/rejected": -0.9689663052558899, + "logps/chosen": -1.629687786102295, + "logps/rejected": -1.9382522106170654, + "loss": 1.7528, + "nll_loss": 1.6994796991348267, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16296879947185516, + "rewards/margins": 0.03085644729435444, + "rewards/rejected": -0.19382524490356445, + "step": 236 + }, + { + "epoch": 0.6544701415257163, + "grad_norm": 0.2251017838716507, + "learning_rate": 4.795657817279349e-06, + "log_odds_chosen": 0.3826752305030823, + "log_odds_ratio": -0.5331037044525146, + "logits/chosen": 0.143580362200737, + "logits/rejected": -1.0959559679031372, + "logps/chosen": -1.568124532699585, + "logps/rejected": -1.8851323127746582, + "loss": 1.7164, + "nll_loss": 1.663122296333313, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15681245923042297, + "rewards/margins": 0.03170077130198479, + "rewards/rejected": -0.18851323425769806, + "step": 237 + }, + { + "epoch": 0.6572316189161201, + "grad_norm": 0.24112077057361603, + "learning_rate": 4.7924726117441135e-06, + "log_odds_chosen": 0.45494502782821655, + "log_odds_ratio": -0.5097072124481201, + "logits/chosen": 0.12531203031539917, + "logits/rejected": -1.1530197858810425, + "logps/chosen": -1.6881966590881348, + "logps/rejected": -2.074317216873169, + "loss": 1.8279, + "nll_loss": 1.7769427299499512, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16881968080997467, + "rewards/margins": 0.03861205279827118, + "rewards/rejected": -0.20743173360824585, + "step": 238 + }, + { + "epoch": 0.659993096306524, + "grad_norm": 0.23394882678985596, + "learning_rate": 4.789263847505835e-06, + "log_odds_chosen": 0.48239994049072266, + "log_odds_ratio": -0.484576016664505, + "logits/chosen": 0.19327585399150848, + "logits/rejected": -1.0120890140533447, + "logps/chosen": -1.673211693763733, + "logps/rejected": -2.078700542449951, + "loss": 1.8042, + "nll_loss": 1.7557491064071655, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16732117533683777, + "rewards/margins": 0.04054888337850571, + "rewards/rejected": -0.20787005126476288, + "step": 239 + }, + { + "epoch": 0.6627545736969278, + "grad_norm": 0.24529801309108734, + "learning_rate": 4.786031557539532e-06, + "log_odds_chosen": 0.6541503667831421, + "log_odds_ratio": -0.42781609296798706, + "logits/chosen": 0.1258109211921692, + "logits/rejected": -1.3018333911895752, + "logps/chosen": -1.6192426681518555, + "logps/rejected": -2.17771577835083, + "loss": 1.7489, + "nll_loss": 1.7061513662338257, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16192427277565002, + "rewards/margins": 0.055847302079200745, + "rewards/rejected": -0.21777155995368958, + "step": 240 + }, + { + "epoch": 0.6655160510873317, + "grad_norm": 0.21988217532634735, + "learning_rate": 4.782775775061983e-06, + "log_odds_chosen": 0.4216863214969635, + "log_odds_ratio": -0.5078084468841553, + "logits/chosen": 0.22119548916816711, + "logits/rejected": -1.0060391426086426, + "logps/chosen": -1.6245155334472656, + "logps/rejected": -1.9757689237594604, + "loss": 1.7524, + "nll_loss": 1.7016619443893433, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16245155036449432, + "rewards/margins": 0.03512535244226456, + "rewards/rejected": -0.19757691025733948, + "step": 241 + }, + { + "epoch": 0.6682775284777356, + "grad_norm": 0.20372723042964935, + "learning_rate": 4.779496533531393e-06, + "log_odds_chosen": 0.5198055505752563, + "log_odds_ratio": -0.4691554009914398, + "logits/chosen": 0.20225608348846436, + "logits/rejected": -1.1654834747314453, + "logps/chosen": -1.5811524391174316, + "logps/rejected": -2.01274037361145, + "loss": 1.7073, + "nll_loss": 1.6603847742080688, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15811526775360107, + "rewards/margins": 0.043158773332834244, + "rewards/rejected": -0.20127403736114502, + "step": 242 + }, + { + "epoch": 0.6710390058681395, + "grad_norm": 0.22578665614128113, + "learning_rate": 4.7761938666470405e-06, + "log_odds_chosen": 0.4481002688407898, + "log_odds_ratio": -0.5030589699745178, + "logits/chosen": 0.21000558137893677, + "logits/rejected": -1.1474783420562744, + "logps/chosen": -1.6968859434127808, + "logps/rejected": -2.079129695892334, + "loss": 1.8331, + "nll_loss": 1.7828097343444824, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16968859732151031, + "rewards/margins": 0.03822438418865204, + "rewards/rejected": -0.20791295170783997, + "step": 243 + }, + { + "epoch": 0.6738004832585434, + "grad_norm": 0.21789832413196564, + "learning_rate": 4.7728678083489375e-06, + "log_odds_chosen": 0.3310457170009613, + "log_odds_ratio": -0.5467555522918701, + "logits/chosen": 0.21008087694644928, + "logits/rejected": -1.071950912475586, + "logps/chosen": -1.6163996458053589, + "logps/rejected": -1.8934293985366821, + "loss": 1.7531, + "nll_loss": 1.6983906030654907, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1616399735212326, + "rewards/margins": 0.027702966704964638, + "rewards/rejected": -0.1893429309129715, + "step": 244 + }, + { + "epoch": 0.6765619606489471, + "grad_norm": 0.21386297047138214, + "learning_rate": 4.7695183928174804e-06, + "log_odds_chosen": 0.4576282799243927, + "log_odds_ratio": -0.4972766637802124, + "logits/chosen": 0.11432070285081863, + "logits/rejected": -1.215945839881897, + "logps/chosen": -1.6433749198913574, + "logps/rejected": -2.0280091762542725, + "loss": 1.7829, + "nll_loss": 1.7331418991088867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16433750092983246, + "rewards/margins": 0.03846340626478195, + "rewards/rejected": -0.2028008997440338, + "step": 245 + }, + { + "epoch": 0.679323438039351, + "grad_norm": 0.23491686582565308, + "learning_rate": 4.766145654473096e-06, + "log_odds_chosen": 0.37732306122779846, + "log_odds_ratio": -0.5303936004638672, + "logits/chosen": 0.18962648510932922, + "logits/rejected": -1.3029686212539673, + "logps/chosen": -1.716407299041748, + "logps/rejected": -2.0386903285980225, + "loss": 1.8439, + "nll_loss": 1.790850043296814, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17164072394371033, + "rewards/margins": 0.03222829848527908, + "rewards/rejected": -0.2038690447807312, + "step": 246 + }, + { + "epoch": 0.6820849154297549, + "grad_norm": 0.2103213518857956, + "learning_rate": 4.762749627975888e-06, + "log_odds_chosen": 0.40977245569229126, + "log_odds_ratio": -0.5140390396118164, + "logits/chosen": 0.0982648953795433, + "logits/rejected": -1.1915839910507202, + "logps/chosen": -1.557995319366455, + "logps/rejected": -1.8929803371429443, + "loss": 1.6972, + "nll_loss": 1.6458226442337036, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1557995229959488, + "rewards/margins": 0.033498503267765045, + "rewards/rejected": -0.18929803371429443, + "step": 247 + }, + { + "epoch": 0.6848463928201588, + "grad_norm": 0.2225971668958664, + "learning_rate": 4.7593303482252835e-06, + "log_odds_chosen": 0.36240053176879883, + "log_odds_ratio": -0.5350769758224487, + "logits/chosen": 0.16154634952545166, + "logits/rejected": -0.829785943031311, + "logps/chosen": -1.6611778736114502, + "logps/rejected": -1.9659464359283447, + "loss": 1.7928, + "nll_loss": 1.7392576932907104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16611778736114502, + "rewards/margins": 0.03047686442732811, + "rewards/rejected": -0.19659464061260223, + "step": 248 + }, + { + "epoch": 0.6876078702105627, + "grad_norm": 0.19730301201343536, + "learning_rate": 4.755887850359673e-06, + "log_odds_chosen": 0.598614513874054, + "log_odds_ratio": -0.4566551446914673, + "logits/chosen": 0.10718496143817902, + "logits/rejected": -1.343011498451233, + "logps/chosen": -1.5110077857971191, + "logps/rejected": -2.002279043197632, + "loss": 1.6562, + "nll_loss": 1.6105355024337769, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1511007696390152, + "rewards/margins": 0.04912712052464485, + "rewards/rejected": -0.20022790133953094, + "step": 249 + }, + { + "epoch": 0.6903693476009665, + "grad_norm": 0.2222234308719635, + "learning_rate": 4.752422169756048e-06, + "log_odds_chosen": 0.4699355959892273, + "log_odds_ratio": -0.4920656085014343, + "logits/chosen": 0.1497102826833725, + "logits/rejected": -1.4104342460632324, + "logps/chosen": -1.6844984292984009, + "logps/rejected": -2.0840110778808594, + "loss": 1.8276, + "nll_loss": 1.7784277200698853, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16844984889030457, + "rewards/margins": 0.03995127975940704, + "rewards/rejected": -0.20840111374855042, + "step": 250 + }, + { + "epoch": 0.6931308249913704, + "grad_norm": 0.2100534588098526, + "learning_rate": 4.748933342029639e-06, + "log_odds_chosen": 0.5825514793395996, + "log_odds_ratio": -0.4510309398174286, + "logits/chosen": 0.1121891662478447, + "logits/rejected": -1.3764441013336182, + "logps/chosen": -1.560391902923584, + "logps/rejected": -2.048152446746826, + "loss": 1.7094, + "nll_loss": 1.6642597913742065, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15603917837142944, + "rewards/margins": 0.0487760454416275, + "rewards/rejected": -0.20481522381305695, + "step": 251 + }, + { + "epoch": 0.6958923023817742, + "grad_norm": 0.20206160843372345, + "learning_rate": 4.745421403033548e-06, + "log_odds_chosen": 0.4050910174846649, + "log_odds_ratio": -0.5125004649162292, + "logits/chosen": 0.10816405713558197, + "logits/rejected": -1.1761468648910522, + "logps/chosen": -1.6354482173919678, + "logps/rejected": -1.9715569019317627, + "loss": 1.7615, + "nll_loss": 1.7102546691894531, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16354484856128693, + "rewards/margins": 0.033610858023166656, + "rewards/rejected": -0.1971556842327118, + "step": 252 + }, + { + "epoch": 0.6986537797721781, + "grad_norm": 0.2380354106426239, + "learning_rate": 4.741886388858384e-06, + "log_odds_chosen": 0.3426484167575836, + "log_odds_ratio": -0.543499231338501, + "logits/chosen": 0.17392773926258087, + "logits/rejected": -1.2038366794586182, + "logps/chosen": -1.6468617916107178, + "logps/rejected": -1.928429365158081, + "loss": 1.79, + "nll_loss": 1.7356586456298828, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1646861881017685, + "rewards/margins": 0.02815674990415573, + "rewards/rejected": -0.19284293055534363, + "step": 253 + }, + { + "epoch": 0.701415257162582, + "grad_norm": 0.2207070291042328, + "learning_rate": 4.738328335831883e-06, + "log_odds_chosen": 0.3862311542034149, + "log_odds_ratio": -0.5232746005058289, + "logits/chosen": 0.13721241056919098, + "logits/rejected": -1.4115301370620728, + "logps/chosen": -1.612336277961731, + "logps/rejected": -1.9344983100891113, + "loss": 1.7501, + "nll_loss": 1.6977391242980957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16123361885547638, + "rewards/margins": 0.03221620246767998, + "rewards/rejected": -0.19344982504844666, + "step": 254 + }, + { + "epoch": 0.7041767345529858, + "grad_norm": 0.2061997801065445, + "learning_rate": 4.734747280518549e-06, + "log_odds_chosen": 0.47361427545547485, + "log_odds_ratio": -0.48697221279144287, + "logits/chosen": 0.03231241926550865, + "logits/rejected": -1.5338258743286133, + "logps/chosen": -1.6500543355941772, + "logps/rejected": -2.0473241806030273, + "loss": 1.7861, + "nll_loss": 1.737368106842041, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16500544548034668, + "rewards/margins": 0.039726972579956055, + "rewards/rejected": -0.20473241806030273, + "step": 255 + }, + { + "epoch": 0.7069382119433897, + "grad_norm": 0.20688936114311218, + "learning_rate": 4.7311432597192655e-06, + "log_odds_chosen": 0.39435428380966187, + "log_odds_ratio": -0.5248243808746338, + "logits/chosen": 0.03083261288702488, + "logits/rejected": -1.4570684432983398, + "logps/chosen": -1.648856520652771, + "logps/rejected": -1.9787046909332275, + "loss": 1.7765, + "nll_loss": 1.7240355014801025, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16488566994667053, + "rewards/margins": 0.03298482671380043, + "rewards/rejected": -0.19787049293518066, + "step": 256 + }, + { + "epoch": 0.7096996893337936, + "grad_norm": 0.22945424914360046, + "learning_rate": 4.72751631047092e-06, + "log_odds_chosen": 0.5417366027832031, + "log_odds_ratio": -0.46491706371307373, + "logits/chosen": -0.006479084491729736, + "logits/rejected": -1.1154435873031616, + "logps/chosen": -1.6812236309051514, + "logps/rejected": -2.1395444869995117, + "loss": 1.8262, + "nll_loss": 1.7797247171401978, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16812236607074738, + "rewards/margins": 0.0458320677280426, + "rewards/rejected": -0.21395443379878998, + "step": 257 + }, + { + "epoch": 0.7124611667241975, + "grad_norm": 0.22095176577568054, + "learning_rate": 4.72386647004603e-06, + "log_odds_chosen": 0.4106917977333069, + "log_odds_ratio": -0.5133163332939148, + "logits/chosen": 0.1232781782746315, + "logits/rejected": -1.1046854257583618, + "logps/chosen": -1.6649987697601318, + "logps/rejected": -2.0084848403930664, + "loss": 1.811, + "nll_loss": 1.7596914768218994, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16649989783763885, + "rewards/margins": 0.03434859216213226, + "rewards/rejected": -0.20084848999977112, + "step": 258 + }, + { + "epoch": 0.7152226441146013, + "grad_norm": 0.22131314873695374, + "learning_rate": 4.720193775952352e-06, + "log_odds_chosen": 0.2770199477672577, + "log_odds_ratio": -0.5675607919692993, + "logits/chosen": 0.09717811644077301, + "logits/rejected": -1.1877737045288086, + "logps/chosen": -1.6597505807876587, + "logps/rejected": -1.889330267906189, + "loss": 1.7861, + "nll_loss": 1.729378581047058, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16597506403923035, + "rewards/margins": 0.022957956418395042, + "rewards/rejected": -0.18893301486968994, + "step": 259 + }, + { + "epoch": 0.7179841215050051, + "grad_norm": 0.2108003944158554, + "learning_rate": 4.716498265932501e-06, + "log_odds_chosen": 0.5200653076171875, + "log_odds_ratio": -0.47012218832969666, + "logits/chosen": -0.01136242039501667, + "logits/rejected": -1.1188238859176636, + "logps/chosen": -1.4902641773223877, + "logps/rejected": -1.915147066116333, + "loss": 1.6221, + "nll_loss": 1.575091004371643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14902642369270325, + "rewards/margins": 0.04248826950788498, + "rewards/rejected": -0.19151470065116882, + "step": 260 + }, + { + "epoch": 0.720745598895409, + "grad_norm": 0.22381585836410522, + "learning_rate": 4.712779977963559e-06, + "log_odds_chosen": 0.39622795581817627, + "log_odds_ratio": -0.5201148390769958, + "logits/chosen": 0.10159610211849213, + "logits/rejected": -0.859241247177124, + "logps/chosen": -1.5900012254714966, + "logps/rejected": -1.917615294456482, + "loss": 1.7383, + "nll_loss": 1.6862510442733765, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15900012850761414, + "rewards/margins": 0.03276140242815018, + "rewards/rejected": -0.1917615383863449, + "step": 261 + }, + { + "epoch": 0.7235070762858129, + "grad_norm": 0.2046324461698532, + "learning_rate": 4.7090389502566884e-06, + "log_odds_chosen": 0.5120800733566284, + "log_odds_ratio": -0.47404342889785767, + "logits/chosen": 0.017112823203206062, + "logits/rejected": -1.1166574954986572, + "logps/chosen": -1.6049623489379883, + "logps/rejected": -2.0321714878082275, + "loss": 1.7168, + "nll_loss": 1.6693758964538574, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16049623489379883, + "rewards/margins": 0.042720913887023926, + "rewards/rejected": -0.20321716368198395, + "step": 262 + }, + { + "epoch": 0.7262685536762168, + "grad_norm": 0.20920297503471375, + "learning_rate": 4.705275221256738e-06, + "log_odds_chosen": 0.42567548155784607, + "log_odds_ratio": -0.5072777271270752, + "logits/chosen": 0.11223579943180084, + "logits/rejected": -1.1794720888137817, + "logps/chosen": -1.6436134576797485, + "logps/rejected": -2.0019237995147705, + "loss": 1.7837, + "nll_loss": 1.7329978942871094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16436134278774261, + "rewards/margins": 0.035831037908792496, + "rewards/rejected": -0.2001923769712448, + "step": 263 + }, + { + "epoch": 0.7290300310666207, + "grad_norm": 0.215216726064682, + "learning_rate": 4.701488829641845e-06, + "log_odds_chosen": 0.3972318172454834, + "log_odds_ratio": -0.5163053870201111, + "logits/chosen": 0.07385722547769547, + "logits/rejected": -1.1585350036621094, + "logps/chosen": -1.5981651544570923, + "logps/rejected": -1.9258975982666016, + "loss": 1.7361, + "nll_loss": 1.6844836473464966, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15981650352478027, + "rewards/margins": 0.032773248851299286, + "rewards/rejected": -0.19258975982666016, + "step": 264 + }, + { + "epoch": 0.7317915084570245, + "grad_norm": 0.20215153694152832, + "learning_rate": 4.697679814323044e-06, + "log_odds_chosen": 0.373761922121048, + "log_odds_ratio": -0.5314816832542419, + "logits/chosen": 0.0431194081902504, + "logits/rejected": -1.3062759637832642, + "logps/chosen": -1.6237438917160034, + "logps/rejected": -1.9311178922653198, + "loss": 1.746, + "nll_loss": 1.6928824186325073, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1623743772506714, + "rewards/margins": 0.030737407505512238, + "rewards/rejected": -0.19311177730560303, + "step": 265 + }, + { + "epoch": 0.7345529858474283, + "grad_norm": 0.2043098360300064, + "learning_rate": 4.693848214443858e-06, + "log_odds_chosen": 0.3958456516265869, + "log_odds_ratio": -0.5203114151954651, + "logits/chosen": 0.016560683026909828, + "logits/rejected": -1.4892038106918335, + "logps/chosen": -1.690280795097351, + "logps/rejected": -2.024770736694336, + "loss": 1.8103, + "nll_loss": 1.7582213878631592, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16902808845043182, + "rewards/margins": 0.03344898298382759, + "rewards/rejected": -0.2024770826101303, + "step": 266 + }, + { + "epoch": 0.7373144632378322, + "grad_norm": 0.2200057953596115, + "learning_rate": 4.689994069379905e-06, + "log_odds_chosen": 0.6603919863700867, + "log_odds_ratio": -0.4194304049015045, + "logits/chosen": 0.0796060711145401, + "logits/rejected": -1.6784858703613281, + "logps/chosen": -1.5854812860488892, + "logps/rejected": -2.141174554824829, + "loss": 1.7061, + "nll_loss": 1.6641736030578613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15854813158512115, + "rewards/margins": 0.05556933209300041, + "rewards/rejected": -0.21411746740341187, + "step": 267 + }, + { + "epoch": 0.7400759406282361, + "grad_norm": 0.207722008228302, + "learning_rate": 4.686117418738489e-06, + "log_odds_chosen": 0.4980109930038452, + "log_odds_ratio": -0.477446049451828, + "logits/chosen": 0.0639527440071106, + "logits/rejected": -1.2904075384140015, + "logps/chosen": -1.632141351699829, + "logps/rejected": -2.049818515777588, + "loss": 1.7618, + "nll_loss": 1.7140535116195679, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16321413218975067, + "rewards/margins": 0.041767701506614685, + "rewards/rejected": -0.20498183369636536, + "step": 268 + }, + { + "epoch": 0.74283741801864, + "grad_norm": 0.20242716372013092, + "learning_rate": 4.6822183023581945e-06, + "log_odds_chosen": 0.42001479864120483, + "log_odds_ratio": -0.517326831817627, + "logits/chosen": 0.07094614952802658, + "logits/rejected": -1.5565531253814697, + "logps/chosen": -1.6155306100845337, + "logps/rejected": -1.9663753509521484, + "loss": 1.7551, + "nll_loss": 1.7033692598342896, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1615530550479889, + "rewards/margins": 0.035084471106529236, + "rewards/rejected": -0.19663754105567932, + "step": 269 + }, + { + "epoch": 0.7455988954090439, + "grad_norm": 0.19576282799243927, + "learning_rate": 4.678296760308474e-06, + "log_odds_chosen": 0.34514501690864563, + "log_odds_ratio": -0.5403321385383606, + "logits/chosen": -0.05599237233400345, + "logits/rejected": -1.4408526420593262, + "logps/chosen": -1.5795042514801025, + "logps/rejected": -1.8630210161209106, + "loss": 1.7087, + "nll_loss": 1.6546752452850342, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15795043110847473, + "rewards/margins": 0.028351658955216408, + "rewards/rejected": -0.1863020956516266, + "step": 270 + }, + { + "epoch": 0.7483603727994477, + "grad_norm": 0.22211046516895294, + "learning_rate": 4.674352832889239e-06, + "log_odds_chosen": 0.5620113611221313, + "log_odds_ratio": -0.4555080533027649, + "logits/chosen": -0.005663935095071793, + "logits/rejected": -1.2683230638504028, + "logps/chosen": -1.6520458459854126, + "logps/rejected": -2.1268177032470703, + "loss": 1.7996, + "nll_loss": 1.7540650367736816, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16520458459854126, + "rewards/margins": 0.04747716709971428, + "rewards/rejected": -0.21268175542354584, + "step": 271 + }, + { + "epoch": 0.7511218501898516, + "grad_norm": 0.1922963708639145, + "learning_rate": 4.670386560630446e-06, + "log_odds_chosen": 0.444943368434906, + "log_odds_ratio": -0.5015792846679688, + "logits/chosen": -0.06714704632759094, + "logits/rejected": -1.379449725151062, + "logps/chosen": -1.5659842491149902, + "logps/rejected": -1.9318914413452148, + "loss": 1.6989, + "nll_loss": 1.6487019062042236, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15659843385219574, + "rewards/margins": 0.036590706557035446, + "rewards/rejected": -0.19318914413452148, + "step": 272 + }, + { + "epoch": 0.7538833275802554, + "grad_norm": 0.19772395491600037, + "learning_rate": 4.66639798429168e-06, + "log_odds_chosen": 0.5909014940261841, + "log_odds_ratio": -0.4440915584564209, + "logits/chosen": -0.010252359323203564, + "logits/rejected": -1.6117687225341797, + "logps/chosen": -1.6227033138275146, + "logps/rejected": -2.1234261989593506, + "loss": 1.7368, + "nll_loss": 1.69236421585083, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16227032244205475, + "rewards/margins": 0.05007229745388031, + "rewards/rejected": -0.21234261989593506, + "step": 273 + }, + { + "epoch": 0.7566448049706593, + "grad_norm": 0.21296410262584686, + "learning_rate": 4.6623871448617345e-06, + "log_odds_chosen": 0.37625253200531006, + "log_odds_ratio": -0.5279226303100586, + "logits/chosen": -0.11291900277137756, + "logits/rejected": -1.474963665008545, + "logps/chosen": -1.5688221454620361, + "logps/rejected": -1.8770910501480103, + "loss": 1.6973, + "nll_loss": 1.6444581747055054, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15688221156597137, + "rewards/margins": 0.03082689456641674, + "rewards/rejected": -0.18770912289619446, + "step": 274 + }, + { + "epoch": 0.7594062823610632, + "grad_norm": 0.21504846215248108, + "learning_rate": 4.6583540835581885e-06, + "log_odds_chosen": 0.446832537651062, + "log_odds_ratio": -0.5026010870933533, + "logits/chosen": -0.0652085542678833, + "logits/rejected": -1.2765593528747559, + "logps/chosen": -1.5856618881225586, + "logps/rejected": -1.9579052925109863, + "loss": 1.7239, + "nll_loss": 1.6736685037612915, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1585661768913269, + "rewards/margins": 0.037224359810352325, + "rewards/rejected": -0.19579055905342102, + "step": 275 + }, + { + "epoch": 0.762167759751467, + "grad_norm": 0.1951994001865387, + "learning_rate": 4.654298841826988e-06, + "log_odds_chosen": 0.38752269744873047, + "log_odds_ratio": -0.5222585797309875, + "logits/chosen": -0.09070023894309998, + "logits/rejected": -1.2031896114349365, + "logps/chosen": -1.5128428936004639, + "logps/rejected": -1.8278968334197998, + "loss": 1.6382, + "nll_loss": 1.586016058921814, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15128430724143982, + "rewards/margins": 0.031505391001701355, + "rewards/rejected": -0.18278968334197998, + "step": 276 + }, + { + "epoch": 0.7649292371418709, + "grad_norm": 0.20356132090091705, + "learning_rate": 4.6502214613420164e-06, + "log_odds_chosen": 0.661637008190155, + "log_odds_ratio": -0.4188510775566101, + "logits/chosen": -0.07356397807598114, + "logits/rejected": -1.270960807800293, + "logps/chosen": -1.4853070974349976, + "logps/rejected": -2.030834674835205, + "loss": 1.6179, + "nll_loss": 1.5760544538497925, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14853070676326752, + "rewards/margins": 0.054552774876356125, + "rewards/rejected": -0.20308347046375275, + "step": 277 + }, + { + "epoch": 0.7676907145322748, + "grad_norm": 0.21808800101280212, + "learning_rate": 4.646121984004666e-06, + "log_odds_chosen": 0.535017192363739, + "log_odds_ratio": -0.4694536030292511, + "logits/chosen": -0.058985427021980286, + "logits/rejected": -1.204667329788208, + "logps/chosen": -1.6129412651062012, + "logps/rejected": -2.0625391006469727, + "loss": 1.7558, + "nll_loss": 1.7088611125946045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1612941473722458, + "rewards/margins": 0.04495978727936745, + "rewards/rejected": -0.20625391602516174, + "step": 278 + }, + { + "epoch": 0.7704521919226787, + "grad_norm": 0.21462294459342957, + "learning_rate": 4.642000451943409e-06, + "log_odds_chosen": 0.4302963316440582, + "log_odds_ratio": -0.5060604214668274, + "logits/chosen": 0.040374599397182465, + "logits/rejected": -1.187546968460083, + "logps/chosen": -1.6709057092666626, + "logps/rejected": -2.035818576812744, + "loss": 1.7989, + "nll_loss": 1.7482545375823975, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16709057986736298, + "rewards/margins": 0.03649128973484039, + "rewards/rejected": -0.20358186960220337, + "step": 279 + }, + { + "epoch": 0.7732136693130826, + "grad_norm": 0.19471004605293274, + "learning_rate": 4.637856907513366e-06, + "log_odds_chosen": 0.5729100704193115, + "log_odds_ratio": -0.45996299386024475, + "logits/chosen": -0.0875316932797432, + "logits/rejected": -1.2949634790420532, + "logps/chosen": -1.6255837678909302, + "logps/rejected": -2.107527256011963, + "loss": 1.7409, + "nll_loss": 1.6949416399002075, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1625583916902542, + "rewards/margins": 0.04819435626268387, + "rewards/rejected": -0.21075274050235748, + "step": 280 + }, + { + "epoch": 0.7759751467034863, + "grad_norm": 0.20401322841644287, + "learning_rate": 4.633691393295865e-06, + "log_odds_chosen": 0.3522525429725647, + "log_odds_ratio": -0.5362752676010132, + "logits/chosen": -0.07532086223363876, + "logits/rejected": -1.3928956985473633, + "logps/chosen": -1.6393187046051025, + "logps/rejected": -1.9348175525665283, + "loss": 1.7628, + "nll_loss": 1.7092012166976929, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16393187642097473, + "rewards/margins": 0.02954990416765213, + "rewards/rejected": -0.19348177313804626, + "step": 281 + }, + { + "epoch": 0.7787366240938902, + "grad_norm": 0.19418881833553314, + "learning_rate": 4.629503952098011e-06, + "log_odds_chosen": 0.6238572597503662, + "log_odds_ratio": -0.43670332431793213, + "logits/chosen": -0.08066678047180176, + "logits/rejected": -1.6776320934295654, + "logps/chosen": -1.5727202892303467, + "logps/rejected": -2.097163200378418, + "loss": 1.6952, + "nll_loss": 1.6514896154403687, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15727202594280243, + "rewards/margins": 0.05244428664445877, + "rewards/rejected": -0.2097163200378418, + "step": 282 + }, + { + "epoch": 0.7814981014842941, + "grad_norm": 0.19461016356945038, + "learning_rate": 4.6252946269522406e-06, + "log_odds_chosen": 0.41456982493400574, + "log_odds_ratio": -0.5249388813972473, + "logits/chosen": -0.09783484041690826, + "logits/rejected": -1.5897575616836548, + "logps/chosen": -1.5939539670944214, + "logps/rejected": -1.9444385766983032, + "loss": 1.7211, + "nll_loss": 1.6686402559280396, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15939539670944214, + "rewards/margins": 0.0350484773516655, + "rewards/rejected": -0.19444386661052704, + "step": 283 + }, + { + "epoch": 0.784259578874698, + "grad_norm": 0.2026386708021164, + "learning_rate": 4.621063461115882e-06, + "log_odds_chosen": 0.42722252011299133, + "log_odds_ratio": -0.5157784223556519, + "logits/chosen": -0.04370000213384628, + "logits/rejected": -1.5634028911590576, + "logps/chosen": -1.6642380952835083, + "logps/rejected": -2.0263044834136963, + "loss": 1.7849, + "nll_loss": 1.733304738998413, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16642381250858307, + "rewards/margins": 0.03620663285255432, + "rewards/rejected": -0.20263046026229858, + "step": 284 + }, + { + "epoch": 0.7870210562651019, + "grad_norm": 0.20439012348651886, + "learning_rate": 4.6168104980707105e-06, + "log_odds_chosen": 0.4680078625679016, + "log_odds_ratio": -0.4906473755836487, + "logits/chosen": -0.09853056073188782, + "logits/rejected": -1.5130950212478638, + "logps/chosen": -1.5937169790267944, + "logps/rejected": -1.9836535453796387, + "loss": 1.7178, + "nll_loss": 1.668696641921997, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15937167406082153, + "rewards/margins": 0.038993678987026215, + "rewards/rejected": -0.19836536049842834, + "step": 285 + }, + { + "epoch": 0.7897825336555057, + "grad_norm": 0.1877906173467636, + "learning_rate": 4.612535781522504e-06, + "log_odds_chosen": 0.37254124879837036, + "log_odds_ratio": -0.525785505771637, + "logits/chosen": -0.08526084572076797, + "logits/rejected": -1.4526267051696777, + "logps/chosen": -1.5574032068252563, + "logps/rejected": -1.863482117652893, + "loss": 1.6777, + "nll_loss": 1.6251548528671265, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15574032068252563, + "rewards/margins": 0.03060789778828621, + "rewards/rejected": -0.18634822964668274, + "step": 286 + }, + { + "epoch": 0.7925440110459095, + "grad_norm": 0.19580195844173431, + "learning_rate": 4.6082393554005855e-06, + "log_odds_chosen": 0.5562997460365295, + "log_odds_ratio": -0.4619132876396179, + "logits/chosen": 0.01294594258069992, + "logits/rejected": -1.4304238557815552, + "logps/chosen": -1.5266389846801758, + "logps/rejected": -1.9919108152389526, + "loss": 1.6616, + "nll_loss": 1.6153795719146729, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.152663916349411, + "rewards/margins": 0.046527184545993805, + "rewards/rejected": -0.19919107854366302, + "step": 287 + }, + { + "epoch": 0.7953054884363134, + "grad_norm": 0.1980113834142685, + "learning_rate": 4.6039212638573835e-06, + "log_odds_chosen": 0.3346819281578064, + "log_odds_ratio": -0.5443198680877686, + "logits/chosen": -0.15275517106056213, + "logits/rejected": -1.6041910648345947, + "logps/chosen": -1.6962547302246094, + "logps/rejected": -1.9770543575286865, + "loss": 1.8067, + "nll_loss": 1.752286672592163, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16962547600269318, + "rewards/margins": 0.028079960495233536, + "rewards/rejected": -0.1977054327726364, + "step": 288 + }, + { + "epoch": 0.7980669658267173, + "grad_norm": 0.2273865044116974, + "learning_rate": 4.599581551267969e-06, + "log_odds_chosen": 0.5466289520263672, + "log_odds_ratio": -0.46321243047714233, + "logits/chosen": 0.03993244469165802, + "logits/rejected": -1.286005973815918, + "logps/chosen": -1.6738882064819336, + "logps/rejected": -2.1377532482147217, + "loss": 1.8, + "nll_loss": 1.7537211179733276, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16738884150981903, + "rewards/margins": 0.046386465430259705, + "rewards/rejected": -0.21377530694007874, + "step": 289 + }, + { + "epoch": 0.8008284432171212, + "grad_norm": 0.20244112610816956, + "learning_rate": 4.5952202622296015e-06, + "log_odds_chosen": 0.3539222180843353, + "log_odds_ratio": -0.53514164686203, + "logits/chosen": -0.11812002211809158, + "logits/rejected": -1.4588350057601929, + "logps/chosen": -1.5985196828842163, + "logps/rejected": -1.8924399614334106, + "loss": 1.7271, + "nll_loss": 1.6736091375350952, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15985198318958282, + "rewards/margins": 0.029392031952738762, + "rewards/rejected": -0.18924400210380554, + "step": 290 + }, + { + "epoch": 0.803589920607525, + "grad_norm": 0.20306488871574402, + "learning_rate": 4.590837441561277e-06, + "log_odds_chosen": 0.4768354296684265, + "log_odds_ratio": -0.49164149165153503, + "logits/chosen": -0.09056994318962097, + "logits/rejected": -1.4742063283920288, + "logps/chosen": -1.6228840351104736, + "logps/rejected": -2.022940158843994, + "loss": 1.7341, + "nll_loss": 1.6849125623703003, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1622883826494217, + "rewards/margins": 0.040005628019571304, + "rewards/rejected": -0.2022940218448639, + "step": 291 + }, + { + "epoch": 0.8063513979979289, + "grad_norm": 0.19205592572689056, + "learning_rate": 4.586433134303257e-06, + "log_odds_chosen": 0.5734292268753052, + "log_odds_ratio": -0.44972163438796997, + "logits/chosen": -0.07964983582496643, + "logits/rejected": -1.3166429996490479, + "logps/chosen": -1.4941394329071045, + "logps/rejected": -1.9636625051498413, + "loss": 1.6364, + "nll_loss": 1.5913902521133423, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14941394329071045, + "rewards/margins": 0.04695230349898338, + "rewards/rejected": -0.19636625051498413, + "step": 292 + }, + { + "epoch": 0.8091128753883328, + "grad_norm": 0.20325274765491486, + "learning_rate": 4.582007385716614e-06, + "log_odds_chosen": 0.4024369418621063, + "log_odds_ratio": -0.5175961852073669, + "logits/chosen": -0.07684268802404404, + "logits/rejected": -1.502617597579956, + "logps/chosen": -1.6142741441726685, + "logps/rejected": -1.9492229223251343, + "loss": 1.7574, + "nll_loss": 1.7056207656860352, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16142742335796356, + "rewards/margins": 0.033494893461465836, + "rewards/rejected": -0.1949223130941391, + "step": 293 + }, + { + "epoch": 0.8118743527787367, + "grad_norm": 0.20083336532115936, + "learning_rate": 4.57756024128276e-06, + "log_odds_chosen": 0.5593162178993225, + "log_odds_ratio": -0.45524081587791443, + "logits/chosen": -0.17213015258312225, + "logits/rejected": -1.3977055549621582, + "logps/chosen": -1.5258231163024902, + "logps/rejected": -1.9875348806381226, + "loss": 1.6559, + "nll_loss": 1.610384225845337, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1525823026895523, + "rewards/margins": 0.04617120325565338, + "rewards/rejected": -0.1987534910440445, + "step": 294 + }, + { + "epoch": 0.8146358301691405, + "grad_norm": 0.19001390039920807, + "learning_rate": 4.573091746702988e-06, + "log_odds_chosen": 0.5878125429153442, + "log_odds_ratio": -0.44817692041397095, + "logits/chosen": -0.16976626217365265, + "logits/rejected": -1.3345102071762085, + "logps/chosen": -1.5175349712371826, + "logps/rejected": -2.003750801086426, + "loss": 1.6573, + "nll_loss": 1.612461805343628, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1517535001039505, + "rewards/margins": 0.048621561378240585, + "rewards/rejected": -0.20037508010864258, + "step": 295 + }, + { + "epoch": 0.8173973075595443, + "grad_norm": 0.18019473552703857, + "learning_rate": 4.5686019478979915e-06, + "log_odds_chosen": 0.6261818408966064, + "log_odds_ratio": -0.43093276023864746, + "logits/chosen": -0.045413050800561905, + "logits/rejected": -1.8382006883621216, + "logps/chosen": -1.5509614944458008, + "logps/rejected": -2.0732412338256836, + "loss": 1.6609, + "nll_loss": 1.617802619934082, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1550961434841156, + "rewards/margins": 0.05222797393798828, + "rewards/rejected": -0.20732411742210388, + "step": 296 + }, + { + "epoch": 0.8201587849499482, + "grad_norm": 0.20908498764038086, + "learning_rate": 4.564090891007401e-06, + "log_odds_chosen": 0.5261347889900208, + "log_odds_ratio": -0.4701959192752838, + "logits/chosen": -0.1871597170829773, + "logits/rejected": -1.5989326238632202, + "logps/chosen": -1.648465633392334, + "logps/rejected": -2.0937981605529785, + "loss": 1.7682, + "nll_loss": 1.7211995124816895, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16484656929969788, + "rewards/margins": 0.04453325271606445, + "rewards/rejected": -0.20937982201576233, + "step": 297 + }, + { + "epoch": 0.8229202623403521, + "grad_norm": 0.188289076089859, + "learning_rate": 4.559558622389304e-06, + "log_odds_chosen": 0.5777133703231812, + "log_odds_ratio": -0.4563387632369995, + "logits/chosen": -0.13260145485401154, + "logits/rejected": -1.7002828121185303, + "logps/chosen": -1.5394691228866577, + "logps/rejected": -2.0138742923736572, + "loss": 1.6561, + "nll_loss": 1.6105040311813354, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1539469212293625, + "rewards/margins": 0.047440510243177414, + "rewards/rejected": -0.201387420296669, + "step": 298 + }, + { + "epoch": 0.825681739730756, + "grad_norm": 0.19627498090267181, + "learning_rate": 4.555005188619776e-06, + "log_odds_chosen": 0.5525295734405518, + "log_odds_ratio": -0.4643310308456421, + "logits/chosen": -0.20222769677639008, + "logits/rejected": -1.6397647857666016, + "logps/chosen": -1.5733509063720703, + "logps/rejected": -2.0376038551330566, + "loss": 1.6892, + "nll_loss": 1.642791748046875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1573350876569748, + "rewards/margins": 0.04642530530691147, + "rewards/rejected": -0.20376040041446686, + "step": 299 + }, + { + "epoch": 0.8284432171211599, + "grad_norm": 0.2084610015153885, + "learning_rate": 4.55043063649239e-06, + "log_odds_chosen": 0.6468226313591003, + "log_odds_ratio": -0.4257048964500427, + "logits/chosen": -0.23111680150032043, + "logits/rejected": -1.959672212600708, + "logps/chosen": -1.5862557888031006, + "logps/rejected": -2.1299490928649902, + "loss": 1.7007, + "nll_loss": 1.6581330299377441, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15862558782100677, + "rewards/margins": 0.054369326680898666, + "rewards/rejected": -0.21299490332603455, + "step": 300 + }, + { + "epoch": 0.8312046945115636, + "grad_norm": 0.18870840966701508, + "learning_rate": 4.54583501301775e-06, + "log_odds_chosen": 0.5869650840759277, + "log_odds_ratio": -0.45076417922973633, + "logits/chosen": -0.1821564882993698, + "logits/rejected": -1.9625955820083618, + "logps/chosen": -1.6092771291732788, + "logps/rejected": -2.104759693145752, + "loss": 1.7134, + "nll_loss": 1.668313980102539, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16092771291732788, + "rewards/margins": 0.04954826086759567, + "rewards/rejected": -0.21047596633434296, + "step": 301 + }, + { + "epoch": 0.8339661719019675, + "grad_norm": 0.1871059238910675, + "learning_rate": 4.541218365422997e-06, + "log_odds_chosen": 0.6819782257080078, + "log_odds_ratio": -0.4187195301055908, + "logits/chosen": -0.2113940417766571, + "logits/rejected": -1.5733145475387573, + "logps/chosen": -1.4704827070236206, + "logps/rejected": -2.02266001701355, + "loss": 1.5975, + "nll_loss": 1.5556209087371826, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14704826474189758, + "rewards/margins": 0.05521773174405098, + "rewards/rejected": -0.20226599276065826, + "step": 302 + }, + { + "epoch": 0.8367276492923714, + "grad_norm": 0.1986740678548813, + "learning_rate": 4.536580741151328e-06, + "log_odds_chosen": 0.427354633808136, + "log_odds_ratio": -0.5044746398925781, + "logits/chosen": -0.17407816648483276, + "logits/rejected": -1.2958568334579468, + "logps/chosen": -1.6126006841659546, + "logps/rejected": -1.9662362337112427, + "loss": 1.7285, + "nll_loss": 1.678093433380127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16126006841659546, + "rewards/margins": 0.03536355867981911, + "rewards/rejected": -0.19662362337112427, + "step": 303 + }, + { + "epoch": 0.8394891266827753, + "grad_norm": 0.20910044014453888, + "learning_rate": 4.531922187861507e-06, + "log_odds_chosen": 0.6855639219284058, + "log_odds_ratio": -0.41133368015289307, + "logits/chosen": -0.17175668478012085, + "logits/rejected": -1.4931915998458862, + "logps/chosen": -1.491321086883545, + "logps/rejected": -2.059537649154663, + "loss": 1.631, + "nll_loss": 1.589834451675415, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1491321176290512, + "rewards/margins": 0.056821659207344055, + "rewards/rejected": -0.20595377683639526, + "step": 304 + }, + { + "epoch": 0.8422506040731792, + "grad_norm": 0.2135782539844513, + "learning_rate": 4.527242753427378e-06, + "log_odds_chosen": 0.4499048590660095, + "log_odds_ratio": -0.4998108744621277, + "logits/chosen": -0.23703832924365997, + "logits/rejected": -1.2717434167861938, + "logps/chosen": -1.6932170391082764, + "logps/rejected": -2.074625015258789, + "loss": 1.8238, + "nll_loss": 1.7738076448440552, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1693217009305954, + "rewards/margins": 0.038140811026096344, + "rewards/rejected": -0.20746250450611115, + "step": 305 + }, + { + "epoch": 0.845012081463583, + "grad_norm": 0.1993100643157959, + "learning_rate": 4.522542485937369e-06, + "log_odds_chosen": 0.6878387331962585, + "log_odds_ratio": -0.4140303134918213, + "logits/chosen": -0.17333604395389557, + "logits/rejected": -1.6610162258148193, + "logps/chosen": -1.5485466718673706, + "logps/rejected": -2.122166395187378, + "loss": 1.6836, + "nll_loss": 1.6422399282455444, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1548546552658081, + "rewards/margins": 0.057361967861652374, + "rewards/rejected": -0.21221664547920227, + "step": 306 + }, + { + "epoch": 0.8477735588539869, + "grad_norm": 0.20089736580848694, + "learning_rate": 4.5178214336940015e-06, + "log_odds_chosen": 0.5427862405776978, + "log_odds_ratio": -0.4643491506576538, + "logits/chosen": -0.1863488256931305, + "logits/rejected": -1.4939963817596436, + "logps/chosen": -1.556074857711792, + "logps/rejected": -2.007810592651367, + "loss": 1.6867, + "nll_loss": 1.640239953994751, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15560749173164368, + "rewards/margins": 0.04517355561256409, + "rewards/rejected": -0.20078104734420776, + "step": 307 + }, + { + "epoch": 0.8505350362443908, + "grad_norm": 0.19465354084968567, + "learning_rate": 4.513079645213391e-06, + "log_odds_chosen": 0.6006003618240356, + "log_odds_ratio": -0.4448007047176361, + "logits/chosen": -0.21229855716228485, + "logits/rejected": -1.460700273513794, + "logps/chosen": -1.4619970321655273, + "logps/rejected": -1.951730728149414, + "loss": 1.5808, + "nll_loss": 1.5363428592681885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14619971811771393, + "rewards/margins": 0.04897337406873703, + "rewards/rejected": -0.19517306983470917, + "step": 308 + }, + { + "epoch": 0.8532965136347946, + "grad_norm": 0.22921526432037354, + "learning_rate": 4.508317169224752e-06, + "log_odds_chosen": 0.28590530157089233, + "log_odds_ratio": -0.5672011375427246, + "logits/chosen": -0.3028235137462616, + "logits/rejected": -1.3352372646331787, + "logps/chosen": -1.5896085500717163, + "logps/rejected": -1.8217551708221436, + "loss": 1.7142, + "nll_loss": 1.6574809551239014, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15896086394786835, + "rewards/margins": 0.0232146717607975, + "rewards/rejected": -0.18217553198337555, + "step": 309 + }, + { + "epoch": 0.8560579910251985, + "grad_norm": 0.20834492146968842, + "learning_rate": 4.5035340546698915e-06, + "log_odds_chosen": 0.5302640795707703, + "log_odds_ratio": -0.47237080335617065, + "logits/chosen": -0.13226553797721863, + "logits/rejected": -1.6689988374710083, + "logps/chosen": -1.5815542936325073, + "logps/rejected": -2.017597198486328, + "loss": 1.7019, + "nll_loss": 1.6546366214752197, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1581554412841797, + "rewards/margins": 0.0436042957007885, + "rewards/rejected": -0.2017597258090973, + "step": 310 + }, + { + "epoch": 0.8588194684156023, + "grad_norm": 0.2186000496149063, + "learning_rate": 4.4987303507027155e-06, + "log_odds_chosen": 0.3287478983402252, + "log_odds_ratio": -0.549705982208252, + "logits/chosen": -0.11287423223257065, + "logits/rejected": -1.349088430404663, + "logps/chosen": -1.6460450887680054, + "logps/rejected": -1.9183804988861084, + "loss": 1.7738, + "nll_loss": 1.718807339668274, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16460449993610382, + "rewards/margins": 0.027233552187681198, + "rewards/rejected": -0.19183805584907532, + "step": 311 + }, + { + "epoch": 0.8615809458060062, + "grad_norm": 0.19687776267528534, + "learning_rate": 4.493906106688712e-06, + "log_odds_chosen": 0.6679433584213257, + "log_odds_ratio": -0.41612738370895386, + "logits/chosen": -0.1544291228055954, + "logits/rejected": -1.6344720125198364, + "logps/chosen": -1.5363831520080566, + "logps/rejected": -2.0934810638427734, + "loss": 1.6699, + "nll_loss": 1.6282765865325928, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1536383181810379, + "rewards/margins": 0.05570977181196213, + "rewards/rejected": -0.20934809744358063, + "step": 312 + }, + { + "epoch": 0.8643424231964101, + "grad_norm": 0.18424390256404877, + "learning_rate": 4.4890613722044526e-06, + "log_odds_chosen": 0.5501468181610107, + "log_odds_ratio": -0.4591679573059082, + "logits/chosen": -0.21150004863739014, + "logits/rejected": -1.5727708339691162, + "logps/chosen": -1.529168963432312, + "logps/rejected": -1.9859933853149414, + "loss": 1.6331, + "nll_loss": 1.587223768234253, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15291690826416016, + "rewards/margins": 0.04568243771791458, + "rewards/rejected": -0.19859933853149414, + "step": 313 + }, + { + "epoch": 0.867103900586814, + "grad_norm": 0.20299668610095978, + "learning_rate": 4.484196197037082e-06, + "log_odds_chosen": 0.5865733027458191, + "log_odds_ratio": -0.4465975761413574, + "logits/chosen": -0.13622766733169556, + "logits/rejected": -1.5913082361221313, + "logps/chosen": -1.5737249851226807, + "logps/rejected": -2.0672624111175537, + "loss": 1.691, + "nll_loss": 1.6463013887405396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15737250447273254, + "rewards/margins": 0.04935373738408089, + "rewards/rejected": -0.20672622323036194, + "step": 314 + }, + { + "epoch": 0.8698653779772179, + "grad_norm": 0.19080907106399536, + "learning_rate": 4.4793106311838e-06, + "log_odds_chosen": 0.556647002696991, + "log_odds_ratio": -0.46323782205581665, + "logits/chosen": -0.2553403377532959, + "logits/rejected": -1.4489879608154297, + "logps/chosen": -1.562751054763794, + "logps/rejected": -2.0281355381011963, + "loss": 1.6685, + "nll_loss": 1.6221669912338257, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15627512335777283, + "rewards/margins": 0.04653845354914665, + "rewards/rejected": -0.20281356573104858, + "step": 315 + }, + { + "epoch": 0.8726268553676216, + "grad_norm": 0.21013295650482178, + "learning_rate": 4.474404724851356e-06, + "log_odds_chosen": 0.5066735148429871, + "log_odds_ratio": -0.48284393548965454, + "logits/chosen": -0.18822398781776428, + "logits/rejected": -1.6378931999206543, + "logps/chosen": -1.6176685094833374, + "logps/rejected": -2.041299819946289, + "loss": 1.7356, + "nll_loss": 1.687273621559143, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16176684200763702, + "rewards/margins": 0.04236314073204994, + "rewards/rejected": -0.20412999391555786, + "step": 316 + }, + { + "epoch": 0.8753883327580255, + "grad_norm": 0.2111610472202301, + "learning_rate": 4.469478528455529e-06, + "log_odds_chosen": 0.4905741810798645, + "log_odds_ratio": -0.4830451011657715, + "logits/chosen": -0.3353807330131531, + "logits/rejected": -1.3652547597885132, + "logps/chosen": -1.6042860746383667, + "logps/rejected": -2.011214017868042, + "loss": 1.7255, + "nll_loss": 1.677234411239624, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16042861342430115, + "rewards/margins": 0.04069279134273529, + "rewards/rejected": -0.20112140476703644, + "step": 317 + }, + { + "epoch": 0.8781498101484294, + "grad_norm": 0.20293498039245605, + "learning_rate": 4.464532092620607e-06, + "log_odds_chosen": 0.523048996925354, + "log_odds_ratio": -0.47193020582199097, + "logits/chosen": -0.18486103415489197, + "logits/rejected": -1.4264463186264038, + "logps/chosen": -1.638154149055481, + "logps/rejected": -2.0816562175750732, + "loss": 1.7478, + "nll_loss": 1.7006094455718994, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1638154238462448, + "rewards/margins": 0.04435021057724953, + "rewards/rejected": -0.20816563069820404, + "step": 318 + }, + { + "epoch": 0.8809112875388333, + "grad_norm": 0.17995508015155792, + "learning_rate": 4.4595654681788715e-06, + "log_odds_chosen": 0.647110641002655, + "log_odds_ratio": -0.428227037191391, + "logits/chosen": -0.23671314120292664, + "logits/rejected": -1.8144798278808594, + "logps/chosen": -1.5134212970733643, + "logps/rejected": -2.050638437271118, + "loss": 1.6212, + "nll_loss": 1.5783425569534302, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15134215354919434, + "rewards/margins": 0.053721703588962555, + "rewards/rejected": -0.2050638496875763, + "step": 319 + }, + { + "epoch": 0.8836727649292372, + "grad_norm": 0.20032061636447906, + "learning_rate": 4.454578706170075e-06, + "log_odds_chosen": 0.35773491859436035, + "log_odds_ratio": -0.5383328795433044, + "logits/chosen": -0.1582796573638916, + "logits/rejected": -1.6286273002624512, + "logps/chosen": -1.6196863651275635, + "logps/rejected": -1.9159085750579834, + "loss": 1.7371, + "nll_loss": 1.6832914352416992, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1619686335325241, + "rewards/margins": 0.02962222881615162, + "rewards/rejected": -0.19159086048603058, + "step": 320 + }, + { + "epoch": 0.886434242319641, + "grad_norm": 0.2055967003107071, + "learning_rate": 4.449571857840911e-06, + "log_odds_chosen": 0.586254358291626, + "log_odds_ratio": -0.45622390508651733, + "logits/chosen": -0.09812657535076141, + "logits/rejected": -1.682092547416687, + "logps/chosen": -1.5706590414047241, + "logps/rejected": -2.0691263675689697, + "loss": 1.6908, + "nll_loss": 1.6451488733291626, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15706591308116913, + "rewards/margins": 0.04984673112630844, + "rewards/rejected": -0.20691262185573578, + "step": 321 + }, + { + "epoch": 0.8891957197100449, + "grad_norm": 0.2170560508966446, + "learning_rate": 4.444544974644493e-06, + "log_odds_chosen": 0.42108941078186035, + "log_odds_ratio": -0.5122984647750854, + "logits/chosen": -0.20832902193069458, + "logits/rejected": -1.468292236328125, + "logps/chosen": -1.5347344875335693, + "logps/rejected": -1.8808050155639648, + "loss": 1.6619, + "nll_loss": 1.61066472530365, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15347345173358917, + "rewards/margins": 0.03460706025362015, + "rewards/rejected": -0.18808050453662872, + "step": 322 + }, + { + "epoch": 0.8919571971004487, + "grad_norm": 0.20523445308208466, + "learning_rate": 4.4394981082398254e-06, + "log_odds_chosen": 0.4536086320877075, + "log_odds_ratio": -0.5041500926017761, + "logits/chosen": -0.26630857586860657, + "logits/rejected": -1.5968291759490967, + "logps/chosen": -1.5988963842391968, + "logps/rejected": -1.9788540601730347, + "loss": 1.7272, + "nll_loss": 1.676782250404358, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15988965332508087, + "rewards/margins": 0.037995755672454834, + "rewards/rejected": -0.19788537919521332, + "step": 323 + }, + { + "epoch": 0.8947186744908526, + "grad_norm": 0.21546316146850586, + "learning_rate": 4.434431310491267e-06, + "log_odds_chosen": 0.6247555017471313, + "log_odds_ratio": -0.43717044591903687, + "logits/chosen": -0.27398669719696045, + "logits/rejected": -1.5846881866455078, + "logps/chosen": -1.660336971282959, + "logps/rejected": -2.190629243850708, + "loss": 1.7647, + "nll_loss": 1.7209898233413696, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16603372991085052, + "rewards/margins": 0.05302921682596207, + "rewards/rejected": -0.2190629243850708, + "step": 324 + }, + { + "epoch": 0.8974801518812565, + "grad_norm": 0.21981805562973022, + "learning_rate": 4.429344633468005e-06, + "log_odds_chosen": 0.5096076130867004, + "log_odds_ratio": -0.4749549627304077, + "logits/chosen": -0.22183284163475037, + "logits/rejected": -1.6317569017410278, + "logps/chosen": -1.528499722480774, + "logps/rejected": -1.9499012231826782, + "loss": 1.6449, + "nll_loss": 1.597440242767334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1528499722480774, + "rewards/margins": 0.0421401672065258, + "rewards/rejected": -0.1949901133775711, + "step": 325 + }, + { + "epoch": 0.9002416292716603, + "grad_norm": 0.23145584762096405, + "learning_rate": 4.424238129443515e-06, + "log_odds_chosen": 0.4823288321495056, + "log_odds_ratio": -0.48343366384506226, + "logits/chosen": -0.1781751811504364, + "logits/rejected": -1.2557498216629028, + "logps/chosen": -1.6358767747879028, + "logps/rejected": -2.0398380756378174, + "loss": 1.7597, + "nll_loss": 1.7113655805587769, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16358768939971924, + "rewards/margins": 0.04039612039923668, + "rewards/rejected": -0.20398379862308502, + "step": 326 + }, + { + "epoch": 0.9030031066620642, + "grad_norm": 0.21986520290374756, + "learning_rate": 4.4191118508950286e-06, + "log_odds_chosen": 0.5763283371925354, + "log_odds_ratio": -0.4508843421936035, + "logits/chosen": -0.2733724117279053, + "logits/rejected": -1.4346749782562256, + "logps/chosen": -1.5722405910491943, + "logps/rejected": -2.0537078380584717, + "loss": 1.7093, + "nll_loss": 1.6641618013381958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15722407400608063, + "rewards/margins": 0.048146720975637436, + "rewards/rejected": -0.20537079870700836, + "step": 327 + }, + { + "epoch": 0.9057645840524681, + "grad_norm": 0.19916215538978577, + "learning_rate": 4.413965850502987e-06, + "log_odds_chosen": 0.6543524265289307, + "log_odds_ratio": -0.42543232440948486, + "logits/chosen": -0.28132179379463196, + "logits/rejected": -1.6163063049316406, + "logps/chosen": -1.4477849006652832, + "logps/rejected": -1.9837119579315186, + "loss": 1.5427, + "nll_loss": 1.5002020597457886, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14477849006652832, + "rewards/margins": 0.05359271913766861, + "rewards/rejected": -0.19837118685245514, + "step": 328 + }, + { + "epoch": 0.908526061442872, + "grad_norm": 0.20786328613758087, + "learning_rate": 4.408800181150509e-06, + "log_odds_chosen": 0.7103330492973328, + "log_odds_ratio": -0.405758798122406, + "logits/chosen": -0.27468955516815186, + "logits/rejected": -1.7239093780517578, + "logps/chosen": -1.5232479572296143, + "logps/rejected": -2.113443374633789, + "loss": 1.6308, + "nll_loss": 1.5901868343353271, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15232481062412262, + "rewards/margins": 0.0590195432305336, + "rewards/rejected": -0.21134433150291443, + "step": 329 + }, + { + "epoch": 0.9112875388332758, + "grad_norm": 0.21173468232154846, + "learning_rate": 4.4036148959228365e-06, + "log_odds_chosen": 0.5421361327171326, + "log_odds_ratio": -0.45979243516921997, + "logits/chosen": -0.19346265494823456, + "logits/rejected": -1.3866770267486572, + "logps/chosen": -1.5939050912857056, + "logps/rejected": -2.0476269721984863, + "loss": 1.7068, + "nll_loss": 1.660808801651001, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15939049422740936, + "rewards/margins": 0.045372217893600464, + "rewards/rejected": -0.20476271212100983, + "step": 330 + }, + { + "epoch": 0.9140490162236796, + "grad_norm": 0.20240680873394012, + "learning_rate": 4.3984100481068e-06, + "log_odds_chosen": 0.5311475992202759, + "log_odds_ratio": -0.47231799364089966, + "logits/chosen": -0.10619790852069855, + "logits/rejected": -1.8089945316314697, + "logps/chosen": -1.528999924659729, + "logps/rejected": -1.9681410789489746, + "loss": 1.6383, + "nll_loss": 1.5910669565200806, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15289999544620514, + "rewards/margins": 0.04391412436962128, + "rewards/rejected": -0.19681411981582642, + "step": 331 + }, + { + "epoch": 0.9168104936140835, + "grad_norm": 0.18672843277454376, + "learning_rate": 4.3931856911902635e-06, + "log_odds_chosen": 0.6374708414077759, + "log_odds_ratio": -0.433654248714447, + "logits/chosen": -0.31257641315460205, + "logits/rejected": -1.766423225402832, + "logps/chosen": -1.5535167455673218, + "logps/rejected": -2.087543487548828, + "loss": 1.6746, + "nll_loss": 1.631203293800354, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1553516834974289, + "rewards/margins": 0.05340268462896347, + "rewards/rejected": -0.20875434577465057, + "step": 332 + }, + { + "epoch": 0.9195719710044874, + "grad_norm": 0.20278650522232056, + "learning_rate": 4.387941878861578e-06, + "log_odds_chosen": 0.442794531583786, + "log_odds_ratio": -0.5059284567832947, + "logits/chosen": -0.2818780243396759, + "logits/rejected": -1.41943359375, + "logps/chosen": -1.475003957748413, + "logps/rejected": -1.838566541671753, + "loss": 1.5936, + "nll_loss": 1.5429768562316895, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1475003957748413, + "rewards/margins": 0.03635626286268234, + "rewards/rejected": -0.18385665118694305, + "step": 333 + }, + { + "epoch": 0.9223334483948913, + "grad_norm": 0.21072961390018463, + "learning_rate": 4.382678665009028e-06, + "log_odds_chosen": 0.5503413677215576, + "log_odds_ratio": -0.4611153304576874, + "logits/chosen": -0.24195455014705658, + "logits/rejected": -1.77567720413208, + "logps/chosen": -1.6017379760742188, + "logps/rejected": -2.0620176792144775, + "loss": 1.7086, + "nll_loss": 1.6624857187271118, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16017380356788635, + "rewards/margins": 0.04602799564599991, + "rewards/rejected": -0.20620179176330566, + "step": 334 + }, + { + "epoch": 0.9250949257852952, + "grad_norm": 0.21240665018558502, + "learning_rate": 4.3773961037202784e-06, + "log_odds_chosen": 0.5787122249603271, + "log_odds_ratio": -0.4496590793132782, + "logits/chosen": -0.24867500364780426, + "logits/rejected": -1.7745471000671387, + "logps/chosen": -1.6565507650375366, + "logps/rejected": -2.1457858085632324, + "loss": 1.7518, + "nll_loss": 1.706835150718689, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16565507650375366, + "rewards/margins": 0.04892349988222122, + "rewards/rejected": -0.21457859873771667, + "step": 335 + }, + { + "epoch": 0.927856403175699, + "grad_norm": 0.20056602358818054, + "learning_rate": 4.37209424928182e-06, + "log_odds_chosen": 0.5505763292312622, + "log_odds_ratio": -0.4596712589263916, + "logits/chosen": -0.18031375110149384, + "logits/rejected": -1.4376002550125122, + "logps/chosen": -1.557979941368103, + "logps/rejected": -2.017378091812134, + "loss": 1.6716, + "nll_loss": 1.6256715059280396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1557980179786682, + "rewards/margins": 0.04593981057405472, + "rewards/rejected": -0.20173780620098114, + "step": 336 + }, + { + "epoch": 0.9306178805661028, + "grad_norm": 0.18808571994304657, + "learning_rate": 4.366773156178413e-06, + "log_odds_chosen": 0.42912667989730835, + "log_odds_ratio": -0.5055819153785706, + "logits/chosen": -0.3135528564453125, + "logits/rejected": -1.5518022775650024, + "logps/chosen": -1.481281042098999, + "logps/rejected": -1.826270341873169, + "loss": 1.5991, + "nll_loss": 1.5485769510269165, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14812810719013214, + "rewards/margins": 0.0344989076256752, + "rewards/rejected": -0.18262703716754913, + "step": 337 + }, + { + "epoch": 0.9333793579565067, + "grad_norm": 0.21728971600532532, + "learning_rate": 4.361432879092518e-06, + "log_odds_chosen": 0.5635970234870911, + "log_odds_ratio": -0.4642696976661682, + "logits/chosen": -0.32608091831207275, + "logits/rejected": -1.478355050086975, + "logps/chosen": -1.5051325559616089, + "logps/rejected": -1.974266529083252, + "loss": 1.6273, + "nll_loss": 1.5808420181274414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15051327645778656, + "rewards/margins": 0.04691339656710625, + "rewards/rejected": -0.1974266618490219, + "step": 338 + }, + { + "epoch": 0.9361408353469106, + "grad_norm": 0.20923133194446564, + "learning_rate": 4.356073472903747e-06, + "log_odds_chosen": 0.5899335145950317, + "log_odds_ratio": -0.44590240716934204, + "logits/chosen": -0.17045272886753082, + "logits/rejected": -1.5179771184921265, + "logps/chosen": -1.4507163763046265, + "logps/rejected": -1.9340299367904663, + "loss": 1.5672, + "nll_loss": 1.5226441621780396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14507164061069489, + "rewards/margins": 0.048331368714571, + "rewards/rejected": -0.1934029906988144, + "step": 339 + }, + { + "epoch": 0.9389023127373145, + "grad_norm": 0.20438268780708313, + "learning_rate": 4.350694992688289e-06, + "log_odds_chosen": 0.6362269520759583, + "log_odds_ratio": -0.43733319640159607, + "logits/chosen": -0.14821594953536987, + "logits/rejected": -1.5649621486663818, + "logps/chosen": -1.4848688840866089, + "logps/rejected": -2.009694814682007, + "loss": 1.6031, + "nll_loss": 1.5594022274017334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1484868973493576, + "rewards/margins": 0.05248260498046875, + "rewards/rejected": -0.20096951723098755, + "step": 340 + }, + { + "epoch": 0.9416637901277183, + "grad_norm": 0.20621763169765472, + "learning_rate": 4.345297493718352e-06, + "log_odds_chosen": 0.547203540802002, + "log_odds_ratio": -0.462637722492218, + "logits/chosen": -0.31976550817489624, + "logits/rejected": -1.318708062171936, + "logps/chosen": -1.4836325645446777, + "logps/rejected": -1.930004358291626, + "loss": 1.6093, + "nll_loss": 1.563034176826477, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14836326241493225, + "rewards/margins": 0.044637180864810944, + "rewards/rejected": -0.1930004358291626, + "step": 341 + }, + { + "epoch": 0.9444252675181222, + "grad_norm": 0.18609599769115448, + "learning_rate": 4.339881031461588e-06, + "log_odds_chosen": 0.47142109274864197, + "log_odds_ratio": -0.48913687467575073, + "logits/chosen": -0.26566094160079956, + "logits/rejected": -1.550631046295166, + "logps/chosen": -1.4443541765213013, + "logps/rejected": -1.8148910999298096, + "loss": 1.578, + "nll_loss": 1.5290637016296387, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14443542063236237, + "rewards/margins": 0.03705369308590889, + "rewards/rejected": -0.18148910999298096, + "step": 342 + }, + { + "epoch": 0.9471867449085261, + "grad_norm": 0.1996508240699768, + "learning_rate": 4.334445661580527e-06, + "log_odds_chosen": 0.5693928599357605, + "log_odds_ratio": -0.4520787000656128, + "logits/chosen": -0.33670923113822937, + "logits/rejected": -1.7492713928222656, + "logps/chosen": -1.5564992427825928, + "logps/rejected": -2.030904531478882, + "loss": 1.6725, + "nll_loss": 1.6272705793380737, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15564994513988495, + "rewards/margins": 0.04744052141904831, + "rewards/rejected": -0.20309044420719147, + "step": 343 + }, + { + "epoch": 0.94994822229893, + "grad_norm": 0.19215835630893707, + "learning_rate": 4.328991439932003e-06, + "log_odds_chosen": 0.6314505934715271, + "log_odds_ratio": -0.4288046658039093, + "logits/chosen": -0.29215654730796814, + "logits/rejected": -1.5209310054779053, + "logps/chosen": -1.486976981163025, + "logps/rejected": -2.0075106620788574, + "loss": 1.6116, + "nll_loss": 1.568747878074646, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14869770407676697, + "rewards/margins": 0.05205334722995758, + "rewards/rejected": -0.20075105130672455, + "step": 344 + }, + { + "epoch": 0.9527096996893338, + "grad_norm": 0.20573773980140686, + "learning_rate": 4.323518422566586e-06, + "log_odds_chosen": 0.7072029113769531, + "log_odds_ratio": -0.40426695346832275, + "logits/chosen": -0.23872965574264526, + "logits/rejected": -1.5439509153366089, + "logps/chosen": -1.5577975511550903, + "logps/rejected": -2.149085760116577, + "loss": 1.6903, + "nll_loss": 1.649876594543457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15577976405620575, + "rewards/margins": 0.059128809720277786, + "rewards/rejected": -0.21490855515003204, + "step": 345 + }, + { + "epoch": 0.9554711770797376, + "grad_norm": 0.1889713704586029, + "learning_rate": 4.318026665727993e-06, + "log_odds_chosen": 0.6957321166992188, + "log_odds_ratio": -0.411748468875885, + "logits/chosen": -0.3132311701774597, + "logits/rejected": -1.7383476495742798, + "logps/chosen": -1.4288955926895142, + "logps/rejected": -1.9945143461227417, + "loss": 1.5422, + "nll_loss": 1.501028299331665, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14288955926895142, + "rewards/margins": 0.056561872363090515, + "rewards/rejected": -0.19945143163204193, + "step": 346 + }, + { + "epoch": 0.9582326544701415, + "grad_norm": 0.21370814740657806, + "learning_rate": 4.3125162258525265e-06, + "log_odds_chosen": 0.37931889295578003, + "log_odds_ratio": -0.5239338874816895, + "logits/chosen": -0.34714582562446594, + "logits/rejected": -1.3666698932647705, + "logps/chosen": -1.6313103437423706, + "logps/rejected": -1.9475239515304565, + "loss": 1.7528, + "nll_loss": 1.7004497051239014, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16313102841377258, + "rewards/margins": 0.031621355563402176, + "rewards/rejected": -0.19475241005420685, + "step": 347 + }, + { + "epoch": 0.9609941318605454, + "grad_norm": 0.20202378928661346, + "learning_rate": 4.3069871595684795e-06, + "log_odds_chosen": 0.5231636762619019, + "log_odds_ratio": -0.4718437194824219, + "logits/chosen": -0.3397434949874878, + "logits/rejected": -1.5151662826538086, + "logps/chosen": -1.4959080219268799, + "logps/rejected": -1.9243054389953613, + "loss": 1.6233, + "nll_loss": 1.5760971307754517, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14959080517292023, + "rewards/margins": 0.04283975064754486, + "rewards/rejected": -0.1924305558204651, + "step": 348 + }, + { + "epoch": 0.9637556092509493, + "grad_norm": 0.1996474266052246, + "learning_rate": 4.3014395236955635e-06, + "log_odds_chosen": 0.6398702263832092, + "log_odds_ratio": -0.4294753670692444, + "logits/chosen": -0.2723667621612549, + "logits/rejected": -1.6896021366119385, + "logps/chosen": -1.4915456771850586, + "logps/rejected": -2.0213863849639893, + "loss": 1.6075, + "nll_loss": 1.5645424127578735, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14915457367897034, + "rewards/margins": 0.0529840886592865, + "rewards/rejected": -0.20213866233825684, + "step": 349 + }, + { + "epoch": 0.9665170866413532, + "grad_norm": 0.18923406302928925, + "learning_rate": 4.295873375244319e-06, + "log_odds_chosen": 0.5657058358192444, + "log_odds_ratio": -0.4569021463394165, + "logits/chosen": -0.2960559129714966, + "logits/rejected": -1.5297155380249023, + "logps/chosen": -1.446962833404541, + "logps/rejected": -1.9055999517440796, + "loss": 1.5727, + "nll_loss": 1.527005672454834, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14469628036022186, + "rewards/margins": 0.045863717794418335, + "rewards/rejected": -0.1905599981546402, + "step": 350 + }, + { + "epoch": 0.9692785640317569, + "grad_norm": 0.2020733803510666, + "learning_rate": 4.290288771415536e-06, + "log_odds_chosen": 0.47521698474884033, + "log_odds_ratio": -0.48997536301612854, + "logits/chosen": -0.2961908280849457, + "logits/rejected": -1.738245964050293, + "logps/chosen": -1.6070820093154907, + "logps/rejected": -2.003466844558716, + "loss": 1.7209, + "nll_loss": 1.6719499826431274, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16070818901062012, + "rewards/margins": 0.039638496935367584, + "rewards/rejected": -0.2003466784954071, + "step": 351 + }, + { + "epoch": 0.9720400414221608, + "grad_norm": 0.19740743935108185, + "learning_rate": 4.284685769599658e-06, + "log_odds_chosen": 0.5527662038803101, + "log_odds_ratio": -0.4617147445678711, + "logits/chosen": -0.3190363645553589, + "logits/rejected": -1.6859633922576904, + "logps/chosen": -1.5147120952606201, + "logps/rejected": -1.9768071174621582, + "loss": 1.6243, + "nll_loss": 1.5781550407409668, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15147122740745544, + "rewards/margins": 0.04620949178934097, + "rewards/rejected": -0.19768072664737701, + "step": 352 + }, + { + "epoch": 0.9748015188125647, + "grad_norm": 0.19644393026828766, + "learning_rate": 4.279064427376199e-06, + "log_odds_chosen": 0.5512232780456543, + "log_odds_ratio": -0.45938748121261597, + "logits/chosen": -0.2991534173488617, + "logits/rejected": -1.5584678649902344, + "logps/chosen": -1.5441019535064697, + "logps/rejected": -2.002403736114502, + "loss": 1.6453, + "nll_loss": 1.5993587970733643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1544102132320404, + "rewards/margins": 0.045830175280570984, + "rewards/rejected": -0.2002403736114502, + "step": 353 + }, + { + "epoch": 0.9775629962029686, + "grad_norm": 0.19680863618850708, + "learning_rate": 4.273424802513145e-06, + "log_odds_chosen": 0.5857518911361694, + "log_odds_ratio": -0.4452309012413025, + "logits/chosen": -0.2941315174102783, + "logits/rejected": -1.6820390224456787, + "logps/chosen": -1.5470666885375977, + "logps/rejected": -2.031536102294922, + "loss": 1.6621, + "nll_loss": 1.6175867319107056, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1547066569328308, + "rewards/margins": 0.04844695329666138, + "rewards/rejected": -0.2031536102294922, + "step": 354 + }, + { + "epoch": 0.9803244735933725, + "grad_norm": 0.18272706866264343, + "learning_rate": 4.267766952966369e-06, + "log_odds_chosen": 0.7239515781402588, + "log_odds_ratio": -0.4105169177055359, + "logits/chosen": -0.23595450818538666, + "logits/rejected": -1.5807809829711914, + "logps/chosen": -1.3758444786071777, + "logps/rejected": -1.9615696668624878, + "loss": 1.4949, + "nll_loss": 1.4538719654083252, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13758444786071777, + "rewards/margins": 0.058572523295879364, + "rewards/rejected": -0.19615697860717773, + "step": 355 + }, + { + "epoch": 0.9830859509837763, + "grad_norm": 0.19880905747413635, + "learning_rate": 4.26209093687903e-06, + "log_odds_chosen": 0.4604690670967102, + "log_odds_ratio": -0.4963679313659668, + "logits/chosen": -0.30030739307403564, + "logits/rejected": -1.8233280181884766, + "logps/chosen": -1.5396702289581299, + "logps/rejected": -1.922170639038086, + "loss": 1.6588, + "nll_loss": 1.6091995239257812, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.153967022895813, + "rewards/margins": 0.038250040262937546, + "rewards/rejected": -0.19221706688404083, + "step": 356 + }, + { + "epoch": 0.9858474283741802, + "grad_norm": 0.19747234880924225, + "learning_rate": 4.2563968125809734e-06, + "log_odds_chosen": 0.5938950777053833, + "log_odds_ratio": -0.44322288036346436, + "logits/chosen": -0.17701445519924164, + "logits/rejected": -1.6618235111236572, + "logps/chosen": -1.60313880443573, + "logps/rejected": -2.101062774658203, + "loss": 1.7054, + "nll_loss": 1.6611095666885376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16031388938426971, + "rewards/margins": 0.04979238659143448, + "rewards/rejected": -0.2101062536239624, + "step": 357 + }, + { + "epoch": 0.988608905764584, + "grad_norm": 0.19918540120124817, + "learning_rate": 4.2506846385881375e-06, + "log_odds_chosen": 0.6773942708969116, + "log_odds_ratio": -0.4168775975704193, + "logits/chosen": -0.4877479672431946, + "logits/rejected": -1.6255730390548706, + "logps/chosen": -1.409053087234497, + "logps/rejected": -1.9528459310531616, + "loss": 1.5337, + "nll_loss": 1.491982340812683, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14090532064437866, + "rewards/margins": 0.05437929555773735, + "rewards/rejected": -0.19528460502624512, + "step": 358 + }, + { + "epoch": 0.9913703831549879, + "grad_norm": 0.1901620328426361, + "learning_rate": 4.2449544736019486e-06, + "log_odds_chosen": 0.5646210312843323, + "log_odds_ratio": -0.45523136854171753, + "logits/chosen": -0.22285765409469604, + "logits/rejected": -1.4501441717147827, + "logps/chosen": -1.479009985923767, + "logps/rejected": -1.9385974407196045, + "loss": 1.5958, + "nll_loss": 1.5503125190734863, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1479010134935379, + "rewards/margins": 0.045958735048770905, + "rewards/rejected": -0.1938597410917282, + "step": 359 + }, + { + "epoch": 0.9941318605453918, + "grad_norm": 0.18198014795780182, + "learning_rate": 4.239206376508716e-06, + "log_odds_chosen": 0.7485941648483276, + "log_odds_ratio": -0.39402005076408386, + "logits/chosen": -0.25095412135124207, + "logits/rejected": -1.9625945091247559, + "logps/chosen": -1.4998773336410522, + "logps/rejected": -2.127000570297241, + "loss": 1.6257, + "nll_loss": 1.5863466262817383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14998774230480194, + "rewards/margins": 0.06271231919527054, + "rewards/rejected": -0.21270006895065308, + "step": 360 + }, + { + "epoch": 0.9968933379357956, + "grad_norm": 0.18113547563552856, + "learning_rate": 4.233440406379032e-06, + "log_odds_chosen": 0.560468852519989, + "log_odds_ratio": -0.45691755414009094, + "logits/chosen": -0.35995978116989136, + "logits/rejected": -1.630347490310669, + "logps/chosen": -1.4223926067352295, + "logps/rejected": -1.871058702468872, + "loss": 1.5472, + "nll_loss": 1.501501202583313, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1422392576932907, + "rewards/margins": 0.04486660659313202, + "rewards/rejected": -0.18710586428642273, + "step": 361 + }, + { + "epoch": 0.9996548153261995, + "grad_norm": 0.1977638155221939, + "learning_rate": 4.227656622467162e-06, + "log_odds_chosen": 0.6151151657104492, + "log_odds_ratio": -0.4389611482620239, + "logits/chosen": -0.34485432505607605, + "logits/rejected": -1.5607213973999023, + "logps/chosen": -1.528998851776123, + "logps/rejected": -2.0421009063720703, + "loss": 1.6442, + "nll_loss": 1.6003334522247314, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15289989113807678, + "rewards/margins": 0.05131019651889801, + "rewards/rejected": -0.2042100876569748, + "step": 362 + }, + { + "epoch": 1.0, + "grad_norm": 0.4796704351902008, + "learning_rate": 4.221855084210433e-06, + "log_odds_chosen": 0.42105579376220703, + "log_odds_ratio": -0.5046184659004211, + "logits/chosen": -0.522539496421814, + "logits/rejected": -2.0535831451416016, + "logps/chosen": -1.5670783519744873, + "logps/rejected": -1.9137303829193115, + "loss": 1.6859, + "nll_loss": 1.635398030281067, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15670783817768097, + "rewards/margins": 0.034665197134017944, + "rewards/rejected": -0.1913730353116989, + "step": 363 + }, + { + "epoch": 1.0027614773904039, + "grad_norm": 0.2030143141746521, + "learning_rate": 4.2160358512286266e-06, + "log_odds_chosen": 0.5737169981002808, + "log_odds_ratio": -0.45416390895843506, + "logits/chosen": -0.2930692732334137, + "logits/rejected": -1.6667262315750122, + "logps/chosen": -1.5078061819076538, + "logps/rejected": -1.9778721332550049, + "loss": 1.6304, + "nll_loss": 1.5849525928497314, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1507806032896042, + "rewards/margins": 0.04700660705566406, + "rewards/rejected": -0.19778722524642944, + "step": 364 + }, + { + "epoch": 1.0055229547808078, + "grad_norm": 0.18416902422904968, + "learning_rate": 4.210198983323366e-06, + "log_odds_chosen": 0.6264990568161011, + "log_odds_ratio": -0.4338659346103668, + "logits/chosen": -0.2520604133605957, + "logits/rejected": -1.8486303091049194, + "logps/chosen": -1.5509365797042847, + "logps/rejected": -2.071831703186035, + "loss": 1.6494, + "nll_loss": 1.606053113937378, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15509365499019623, + "rewards/margins": 0.052089497447013855, + "rewards/rejected": -0.20718316733837128, + "step": 365 + }, + { + "epoch": 1.0082844321712117, + "grad_norm": 0.19729849696159363, + "learning_rate": 4.204344540477499e-06, + "log_odds_chosen": 0.6402697563171387, + "log_odds_ratio": -0.429515540599823, + "logits/chosen": -0.288376122713089, + "logits/rejected": -1.6332948207855225, + "logps/chosen": -1.5338780879974365, + "logps/rejected": -2.068819046020508, + "loss": 1.6531, + "nll_loss": 1.6101782321929932, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15338779985904694, + "rewards/margins": 0.053494103252887726, + "rewards/rejected": -0.20688191056251526, + "step": 366 + }, + { + "epoch": 1.0110459095616156, + "grad_norm": 0.18270985782146454, + "learning_rate": 4.1984725828544855e-06, + "log_odds_chosen": 0.6716368198394775, + "log_odds_ratio": -0.41607195138931274, + "logits/chosen": -0.27237242460250854, + "logits/rejected": -1.8720418214797974, + "logps/chosen": -1.4672292470932007, + "logps/rejected": -2.0190887451171875, + "loss": 1.5786, + "nll_loss": 1.5369707345962524, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1467229127883911, + "rewards/margins": 0.0551859587430954, + "rewards/rejected": -0.2019088715314865, + "step": 367 + }, + { + "epoch": 1.0138073869520194, + "grad_norm": 0.20039811730384827, + "learning_rate": 4.192583170797775e-06, + "log_odds_chosen": 0.5032901167869568, + "log_odds_ratio": -0.4751865863800049, + "logits/chosen": -0.3330274224281311, + "logits/rejected": -1.4479092359542847, + "logps/chosen": -1.5037407875061035, + "logps/rejected": -1.9160493612289429, + "loss": 1.6307, + "nll_loss": 1.5831364393234253, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15037408471107483, + "rewards/margins": 0.041230857372283936, + "rewards/rejected": -0.19160494208335876, + "step": 368 + }, + { + "epoch": 1.016568864342423, + "grad_norm": 0.1855594515800476, + "learning_rate": 4.186676364830187e-06, + "log_odds_chosen": 0.5586062669754028, + "log_odds_ratio": -0.45506125688552856, + "logits/chosen": -0.23738795518875122, + "logits/rejected": -1.6253269910812378, + "logps/chosen": -1.566367745399475, + "logps/rejected": -2.030472755432129, + "loss": 1.6675, + "nll_loss": 1.6219737529754639, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1566367745399475, + "rewards/margins": 0.04641049727797508, + "rewards/rejected": -0.20304730534553528, + "step": 369 + }, + { + "epoch": 1.019330341732827, + "grad_norm": 0.2150687575340271, + "learning_rate": 4.1807522256532925e-06, + "log_odds_chosen": 0.6354942321777344, + "log_odds_ratio": -0.43560659885406494, + "logits/chosen": -0.29759910702705383, + "logits/rejected": -1.7936424016952515, + "logps/chosen": -1.621231198310852, + "logps/rejected": -2.1618242263793945, + "loss": 1.7339, + "nll_loss": 1.6903626918792725, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16212311387062073, + "rewards/margins": 0.054059334099292755, + "rewards/rejected": -0.21618244051933289, + "step": 370 + }, + { + "epoch": 1.0220918191232309, + "grad_norm": 0.18705667555332184, + "learning_rate": 4.174810814146789e-06, + "log_odds_chosen": 0.6377235651016235, + "log_odds_ratio": -0.4334976375102997, + "logits/chosen": -0.17882226407527924, + "logits/rejected": -1.4699535369873047, + "logps/chosen": -1.5574746131896973, + "logps/rejected": -2.092029094696045, + "loss": 1.6697, + "nll_loss": 1.626399040222168, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1557474583387375, + "rewards/margins": 0.05345546454191208, + "rewards/rejected": -0.20920291543006897, + "step": 371 + }, + { + "epoch": 1.0248532965136348, + "grad_norm": 0.18511676788330078, + "learning_rate": 4.1688521913678706e-06, + "log_odds_chosen": 0.7460950613021851, + "log_odds_ratio": -0.39828044176101685, + "logits/chosen": -0.24553045630455017, + "logits/rejected": -1.5528843402862549, + "logps/chosen": -1.506624460220337, + "logps/rejected": -2.1312265396118164, + "loss": 1.6125, + "nll_loss": 1.5727205276489258, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15066243708133698, + "rewards/margins": 0.06246021389961243, + "rewards/rejected": -0.2131226509809494, + "step": 372 + }, + { + "epoch": 1.0276147739040387, + "grad_norm": 0.18531759083271027, + "learning_rate": 4.162876418550606e-06, + "log_odds_chosen": 0.5151762962341309, + "log_odds_ratio": -0.47239360213279724, + "logits/chosen": -0.3635602593421936, + "logits/rejected": -1.655716061592102, + "logps/chosen": -1.509564995765686, + "logps/rejected": -1.9321736097335815, + "loss": 1.6228, + "nll_loss": 1.5756025314331055, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15095649659633636, + "rewards/margins": 0.042260847985744476, + "rewards/rejected": -0.19321735203266144, + "step": 373 + }, + { + "epoch": 1.0303762512944425, + "grad_norm": 0.19160981476306915, + "learning_rate": 4.156883557105308e-06, + "log_odds_chosen": 0.537889301776886, + "log_odds_ratio": -0.46930158138275146, + "logits/chosen": -0.3415728509426117, + "logits/rejected": -1.7609002590179443, + "logps/chosen": -1.4833260774612427, + "logps/rejected": -1.925763726234436, + "loss": 1.5902, + "nll_loss": 1.5432225465774536, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1483326107263565, + "rewards/margins": 0.044243764132261276, + "rewards/rejected": -0.1925763636827469, + "step": 374 + }, + { + "epoch": 1.0331377286848464, + "grad_norm": 0.2014252096414566, + "learning_rate": 4.150873668617899e-06, + "log_odds_chosen": 0.5346254110336304, + "log_odds_ratio": -0.4631119966506958, + "logits/chosen": -0.4020942449569702, + "logits/rejected": -1.6286017894744873, + "logps/chosen": -1.554813265800476, + "logps/rejected": -1.9967365264892578, + "loss": 1.6587, + "nll_loss": 1.6124264001846313, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15548132359981537, + "rewards/margins": 0.0441923588514328, + "rewards/rejected": -0.19967366755008698, + "step": 375 + }, + { + "epoch": 1.0358992060752503, + "grad_norm": 0.1909962296485901, + "learning_rate": 4.144846814849282e-06, + "log_odds_chosen": 0.4685186743736267, + "log_odds_ratio": -0.4881165623664856, + "logits/chosen": -0.19894683361053467, + "logits/rejected": -1.4885890483856201, + "logps/chosen": -1.5590081214904785, + "logps/rejected": -1.9445595741271973, + "loss": 1.6834, + "nll_loss": 1.634606957435608, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15590080618858337, + "rewards/margins": 0.03855516016483307, + "rewards/rejected": -0.19445598125457764, + "step": 376 + }, + { + "epoch": 1.0386606834656542, + "grad_norm": 0.1877821981906891, + "learning_rate": 4.138803057734705e-06, + "log_odds_chosen": 0.7200895547866821, + "log_odds_ratio": -0.4008093476295471, + "logits/chosen": -0.25993677973747253, + "logits/rejected": -1.574484944343567, + "logps/chosen": -1.456033706665039, + "logps/rejected": -2.0489182472229004, + "loss": 1.5752, + "nll_loss": 1.5350782871246338, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14560337364673615, + "rewards/margins": 0.05928843468427658, + "rewards/rejected": -0.20489181578159332, + "step": 377 + }, + { + "epoch": 1.041422160856058, + "grad_norm": 0.17346148192882538, + "learning_rate": 4.132742459383122e-06, + "log_odds_chosen": 0.7830832004547119, + "log_odds_ratio": -0.3833864629268646, + "logits/chosen": -0.27740198373794556, + "logits/rejected": -1.7669141292572021, + "logps/chosen": -1.380853533744812, + "logps/rejected": -2.0105137825012207, + "loss": 1.4985, + "nll_loss": 1.4602102041244507, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13808535039424896, + "rewards/margins": 0.06296603381633759, + "rewards/rejected": -0.20105136930942535, + "step": 378 + }, + { + "epoch": 1.0441836382464618, + "grad_norm": 0.183350071310997, + "learning_rate": 4.126665082076559e-06, + "log_odds_chosen": 0.5126218795776367, + "log_odds_ratio": -0.47373971343040466, + "logits/chosen": -0.3141680955886841, + "logits/rejected": -1.3418911695480347, + "logps/chosen": -1.516774296760559, + "logps/rejected": -1.933377742767334, + "loss": 1.6442, + "nll_loss": 1.5968701839447021, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1516774296760559, + "rewards/margins": 0.04166034609079361, + "rewards/rejected": -0.19333778321743011, + "step": 379 + }, + { + "epoch": 1.0469451156368657, + "grad_norm": 0.18558627367019653, + "learning_rate": 4.120570988269472e-06, + "log_odds_chosen": 0.651531994342804, + "log_odds_ratio": -0.423582524061203, + "logits/chosen": -0.38820552825927734, + "logits/rejected": -1.826667070388794, + "logps/chosen": -1.5054341554641724, + "logps/rejected": -2.0457534790039062, + "loss": 1.5926, + "nll_loss": 1.5502351522445679, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1505434215068817, + "rewards/margins": 0.05403192341327667, + "rewards/rejected": -0.20457535982131958, + "step": 380 + }, + { + "epoch": 1.0497065930272695, + "grad_norm": 0.19876988232135773, + "learning_rate": 4.114460240588101e-06, + "log_odds_chosen": 0.7609922885894775, + "log_odds_ratio": -0.3903619050979614, + "logits/chosen": -0.28415167331695557, + "logits/rejected": -1.509800910949707, + "logps/chosen": -1.5427088737487793, + "logps/rejected": -2.184783458709717, + "loss": 1.6545, + "nll_loss": 1.6154515743255615, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15427090227603912, + "rewards/margins": 0.06420743465423584, + "rewards/rejected": -0.21847833693027496, + "step": 381 + }, + { + "epoch": 1.0524680704176734, + "grad_norm": 0.17985652387142181, + "learning_rate": 4.1083329018298356e-06, + "log_odds_chosen": 0.7214679718017578, + "log_odds_ratio": -0.401907354593277, + "logits/chosen": -0.2644188404083252, + "logits/rejected": -1.569467544555664, + "logps/chosen": -1.4487037658691406, + "logps/rejected": -2.043267250061035, + "loss": 1.5525, + "nll_loss": 1.5123515129089355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14487037062644958, + "rewards/margins": 0.05945635586977005, + "rewards/rejected": -0.20432673394680023, + "step": 382 + }, + { + "epoch": 1.0552295478080773, + "grad_norm": 0.19042105972766876, + "learning_rate": 4.102189034962561e-06, + "log_odds_chosen": 0.6154542565345764, + "log_odds_ratio": -0.4352980852127075, + "logits/chosen": -0.21143808960914612, + "logits/rejected": -1.4588818550109863, + "logps/chosen": -1.583728551864624, + "logps/rejected": -2.1008310317993164, + "loss": 1.6812, + "nll_loss": 1.6376571655273438, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15837284922599792, + "rewards/margins": 0.051710255444049835, + "rewards/rejected": -0.21008309721946716, + "step": 383 + }, + { + "epoch": 1.0579910251984812, + "grad_norm": 0.202115997672081, + "learning_rate": 4.096028703124014e-06, + "log_odds_chosen": 0.6282423138618469, + "log_odds_ratio": -0.43798086047172546, + "logits/chosen": -0.3399620056152344, + "logits/rejected": -1.302710771560669, + "logps/chosen": -1.5835597515106201, + "logps/rejected": -2.1131136417388916, + "loss": 1.67, + "nll_loss": 1.6261749267578125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1583559811115265, + "rewards/margins": 0.05295538902282715, + "rewards/rejected": -0.21131137013435364, + "step": 384 + }, + { + "epoch": 1.060752502588885, + "grad_norm": 0.18484216928482056, + "learning_rate": 4.089851969621138e-06, + "log_odds_chosen": 0.7135899662971497, + "log_odds_ratio": -0.41620948910713196, + "logits/chosen": -0.28658849000930786, + "logits/rejected": -1.5913586616516113, + "logps/chosen": -1.425489902496338, + "logps/rejected": -2.015927791595459, + "loss": 1.5551, + "nll_loss": 1.5135202407836914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14254900813102722, + "rewards/margins": 0.059043798595666885, + "rewards/rejected": -0.20159278810024261, + "step": 385 + }, + { + "epoch": 1.063513979979289, + "grad_norm": 0.20954498648643494, + "learning_rate": 4.083658897929425e-06, + "log_odds_chosen": 0.6113450527191162, + "log_odds_ratio": -0.4348878562450409, + "logits/chosen": -0.3510580360889435, + "logits/rejected": -1.5097743272781372, + "logps/chosen": -1.5490520000457764, + "logps/rejected": -2.056283950805664, + "loss": 1.6599, + "nll_loss": 1.616385817527771, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15490520000457764, + "rewards/margins": 0.050723206251859665, + "rewards/rejected": -0.2056283950805664, + "step": 386 + }, + { + "epoch": 1.0662754573696929, + "grad_norm": 0.19627171754837036, + "learning_rate": 4.077449551692268e-06, + "log_odds_chosen": 0.5390780568122864, + "log_odds_ratio": -0.46470367908477783, + "logits/chosen": -0.23739197850227356, + "logits/rejected": -1.621540904045105, + "logps/chosen": -1.555280089378357, + "logps/rejected": -2.001728057861328, + "loss": 1.6682, + "nll_loss": 1.6217542886734009, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1555280089378357, + "rewards/margins": 0.044644795358181, + "rewards/rejected": -0.20017282664775848, + "step": 387 + }, + { + "epoch": 1.0690369347600965, + "grad_norm": 0.1885172724723816, + "learning_rate": 4.071223994720309e-06, + "log_odds_chosen": 0.7198303937911987, + "log_odds_ratio": -0.40047964453697205, + "logits/chosen": -0.38902541995048523, + "logits/rejected": -1.71836519241333, + "logps/chosen": -1.4929300546646118, + "logps/rejected": -2.0929007530212402, + "loss": 1.6048, + "nll_loss": 1.5647118091583252, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14929300546646118, + "rewards/margins": 0.0599970780313015, + "rewards/rejected": -0.20929010212421417, + "step": 388 + }, + { + "epoch": 1.0717984121505004, + "grad_norm": 0.20288851857185364, + "learning_rate": 4.064982290990777e-06, + "log_odds_chosen": 0.5762845277786255, + "log_odds_ratio": -0.45192721486091614, + "logits/chosen": -0.3292975425720215, + "logits/rejected": -1.4449775218963623, + "logps/chosen": -1.4430099725723267, + "logps/rejected": -1.9106184244155884, + "loss": 1.5566, + "nll_loss": 1.511430263519287, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14430099725723267, + "rewards/margins": 0.04676084965467453, + "rewards/rejected": -0.1910618394613266, + "step": 389 + }, + { + "epoch": 1.0745598895409043, + "grad_norm": 0.18857373297214508, + "learning_rate": 4.058724504646834e-06, + "log_odds_chosen": 0.698142945766449, + "log_odds_ratio": -0.41126978397369385, + "logits/chosen": -0.3490840494632721, + "logits/rejected": -1.5612983703613281, + "logps/chosen": -1.4951398372650146, + "logps/rejected": -2.07562255859375, + "loss": 1.6046, + "nll_loss": 1.563432216644287, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14951398968696594, + "rewards/margins": 0.058048274368047714, + "rewards/rejected": -0.20756226778030396, + "step": 390 + }, + { + "epoch": 1.0773213669313082, + "grad_norm": 0.18817338347434998, + "learning_rate": 4.0524506999969185e-06, + "log_odds_chosen": 0.5495921969413757, + "log_odds_ratio": -0.46190890669822693, + "logits/chosen": -0.32165098190307617, + "logits/rejected": -1.7347975969314575, + "logps/chosen": -1.5509448051452637, + "logps/rejected": -2.0019354820251465, + "loss": 1.6579, + "nll_loss": 1.6116652488708496, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15509448945522308, + "rewards/margins": 0.045099057257175446, + "rewards/rejected": -0.20019352436065674, + "step": 391 + }, + { + "epoch": 1.080082844321712, + "grad_norm": 0.18304969370365143, + "learning_rate": 4.046160941514079e-06, + "log_odds_chosen": 0.7243356704711914, + "log_odds_ratio": -0.4019787311553955, + "logits/chosen": -0.20833127200603485, + "logits/rejected": -1.3766155242919922, + "logps/chosen": -1.4449529647827148, + "logps/rejected": -2.0372161865234375, + "loss": 1.5421, + "nll_loss": 1.5019174814224243, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14449530839920044, + "rewards/margins": 0.05922630429267883, + "rewards/rejected": -0.20372159779071808, + "step": 392 + }, + { + "epoch": 1.082844321712116, + "grad_norm": 0.23034730553627014, + "learning_rate": 4.039855293835316e-06, + "log_odds_chosen": 0.6121989488601685, + "log_odds_ratio": -0.4352457523345947, + "logits/chosen": -0.37496262788772583, + "logits/rejected": -1.6959614753723145, + "logps/chosen": -1.4758626222610474, + "logps/rejected": -1.9751558303833008, + "loss": 1.5737, + "nll_loss": 1.530150294303894, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14758625626564026, + "rewards/margins": 0.04992932081222534, + "rewards/rejected": -0.1975155770778656, + "step": 393 + }, + { + "epoch": 1.0856057991025199, + "grad_norm": 0.17257992923259735, + "learning_rate": 4.033533821760917e-06, + "log_odds_chosen": 0.7732821702957153, + "log_odds_ratio": -0.3887802064418793, + "logits/chosen": -0.3277055025100708, + "logits/rejected": -1.868857741355896, + "logps/chosen": -1.4420204162597656, + "logps/rejected": -2.0812883377075195, + "loss": 1.5466, + "nll_loss": 1.5076910257339478, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14420203864574432, + "rewards/margins": 0.06392678618431091, + "rewards/rejected": -0.20812883973121643, + "step": 394 + }, + { + "epoch": 1.0883672764929238, + "grad_norm": 0.1858333796262741, + "learning_rate": 4.027196590253786e-06, + "log_odds_chosen": 0.4944222569465637, + "log_odds_ratio": -0.4831688404083252, + "logits/chosen": -0.28738462924957275, + "logits/rejected": -1.3927885293960571, + "logps/chosen": -1.4958670139312744, + "logps/rejected": -1.8963196277618408, + "loss": 1.6172, + "nll_loss": 1.5688387155532837, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14958669245243073, + "rewards/margins": 0.04004526883363724, + "rewards/rejected": -0.18963195383548737, + "step": 395 + }, + { + "epoch": 1.0911287538833276, + "grad_norm": 0.1886051446199417, + "learning_rate": 4.020843664438783e-06, + "log_odds_chosen": 0.6189723610877991, + "log_odds_ratio": -0.4430191218852997, + "logits/chosen": -0.30028706789016724, + "logits/rejected": -1.7108986377716064, + "logps/chosen": -1.446290135383606, + "logps/rejected": -1.9520014524459839, + "loss": 1.5704, + "nll_loss": 1.5261311531066895, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14462901651859283, + "rewards/margins": 0.05057113245129585, + "rewards/rejected": -0.1952001452445984, + "step": 396 + }, + { + "epoch": 1.0938902312737315, + "grad_norm": 0.189390167593956, + "learning_rate": 4.01447510960205e-06, + "log_odds_chosen": 0.7106426358222961, + "log_odds_ratio": -0.40562015771865845, + "logits/chosen": -0.4158485531806946, + "logits/rejected": -1.7321726083755493, + "logps/chosen": -1.5334806442260742, + "logps/rejected": -2.1250576972961426, + "loss": 1.6332, + "nll_loss": 1.592668890953064, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15334807336330414, + "rewards/margins": 0.059157684445381165, + "rewards/rejected": -0.2125057429075241, + "step": 397 + }, + { + "epoch": 1.0966517086641354, + "grad_norm": 0.19324593245983124, + "learning_rate": 4.008090991190341e-06, + "log_odds_chosen": 0.49737095832824707, + "log_odds_ratio": -0.47871023416519165, + "logits/chosen": -0.20496344566345215, + "logits/rejected": -1.5982781648635864, + "logps/chosen": -1.570819616317749, + "logps/rejected": -1.9830747842788696, + "loss": 1.6702, + "nll_loss": 1.622377634048462, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1570819765329361, + "rewards/margins": 0.04122550040483475, + "rewards/rejected": -0.19830746948719025, + "step": 398 + }, + { + "epoch": 1.099413186054539, + "grad_norm": 0.17873111367225647, + "learning_rate": 4.001691374810352e-06, + "log_odds_chosen": 0.7614402770996094, + "log_odds_ratio": -0.39445751905441284, + "logits/chosen": -0.27544835209846497, + "logits/rejected": -1.992354154586792, + "logps/chosen": -1.5202884674072266, + "logps/rejected": -2.1587905883789062, + "loss": 1.6232, + "nll_loss": 1.5837651491165161, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15202882885932922, + "rewards/margins": 0.06385020911693573, + "rewards/rejected": -0.21587903797626495, + "step": 399 + }, + { + "epoch": 1.102174663444943, + "grad_norm": 0.17042513191699982, + "learning_rate": 3.99527632622804e-06, + "log_odds_chosen": 0.6560631990432739, + "log_odds_ratio": -0.4252350628376007, + "logits/chosen": -0.3314496576786041, + "logits/rejected": -1.559665322303772, + "logps/chosen": -1.4409160614013672, + "logps/rejected": -1.9774514436721802, + "loss": 1.5645, + "nll_loss": 1.5220184326171875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14409160614013672, + "rewards/margins": 0.05365355312824249, + "rewards/rejected": -0.19774514436721802, + "step": 400 + }, + { + "epoch": 1.1049361408353469, + "grad_norm": 0.1929718255996704, + "learning_rate": 3.988845911367957e-06, + "log_odds_chosen": 0.6236564517021179, + "log_odds_ratio": -0.4364378750324249, + "logits/chosen": -0.3177638649940491, + "logits/rejected": -1.5883958339691162, + "logps/chosen": -1.598113775253296, + "logps/rejected": -2.1222128868103027, + "loss": 1.6877, + "nll_loss": 1.644063949584961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1598113626241684, + "rewards/margins": 0.052409932017326355, + "rewards/rejected": -0.21222129464149475, + "step": 401 + }, + { + "epoch": 1.1076976182257507, + "grad_norm": 0.17265060544013977, + "learning_rate": 3.982400196312565e-06, + "log_odds_chosen": 0.6908576488494873, + "log_odds_ratio": -0.4076445698738098, + "logits/chosen": -0.28831660747528076, + "logits/rejected": -1.4703201055526733, + "logps/chosen": -1.4102579355239868, + "logps/rejected": -1.9662861824035645, + "loss": 1.5263, + "nll_loss": 1.4855272769927979, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14102579653263092, + "rewards/margins": 0.05560281127691269, + "rewards/rejected": -0.1966286301612854, + "step": 402 + }, + { + "epoch": 1.1104590956161546, + "grad_norm": 0.18258056044578552, + "learning_rate": 3.975939247301558e-06, + "log_odds_chosen": 0.6171894669532776, + "log_odds_ratio": -0.43477940559387207, + "logits/chosen": -0.34336021542549133, + "logits/rejected": -1.718552589416504, + "logps/chosen": -1.5480486154556274, + "logps/rejected": -2.061230421066284, + "loss": 1.6639, + "nll_loss": 1.620387077331543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15480485558509827, + "rewards/margins": 0.051318198442459106, + "rewards/rejected": -0.20612303912639618, + "step": 403 + }, + { + "epoch": 1.1132205730065585, + "grad_norm": 0.17894716560840607, + "learning_rate": 3.969463130731183e-06, + "log_odds_chosen": 0.601898729801178, + "log_odds_ratio": -0.44408684968948364, + "logits/chosen": -0.2595762610435486, + "logits/rejected": -1.671970248222351, + "logps/chosen": -1.480761170387268, + "logps/rejected": -1.973817229270935, + "loss": 1.5901, + "nll_loss": 1.5457121133804321, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.148076131939888, + "rewards/margins": 0.049305595457553864, + "rewards/rejected": -0.19738171994686127, + "step": 404 + }, + { + "epoch": 1.1159820503969624, + "grad_norm": 0.182702898979187, + "learning_rate": 3.9629719131535595e-06, + "log_odds_chosen": 0.5538443922996521, + "log_odds_ratio": -0.463712602853775, + "logits/chosen": -0.3739258646965027, + "logits/rejected": -1.5419493913650513, + "logps/chosen": -1.4951484203338623, + "logps/rejected": -1.9511198997497559, + "loss": 1.6103, + "nll_loss": 1.563896656036377, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14951485395431519, + "rewards/margins": 0.04559716209769249, + "rewards/rejected": -0.19511200487613678, + "step": 405 + }, + { + "epoch": 1.1187435277873663, + "grad_norm": 0.1713375300168991, + "learning_rate": 3.9564656612759904e-06, + "log_odds_chosen": 0.6431328058242798, + "log_odds_ratio": -0.42764732241630554, + "logits/chosen": -0.29896292090415955, + "logits/rejected": -1.8772965669631958, + "logps/chosen": -1.4999200105667114, + "logps/rejected": -2.031973361968994, + "loss": 1.6098, + "nll_loss": 1.5670844316482544, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14999200403690338, + "rewards/margins": 0.05320533737540245, + "rewards/rejected": -0.20319733023643494, + "step": 406 + }, + { + "epoch": 1.1215050051777702, + "grad_norm": 0.1880570352077484, + "learning_rate": 3.94994444196028e-06, + "log_odds_chosen": 0.6853294968605042, + "log_odds_ratio": -0.41755619645118713, + "logits/chosen": -0.2672858238220215, + "logits/rejected": -1.5635608434677124, + "logps/chosen": -1.5350513458251953, + "logps/rejected": -2.1063671112060547, + "loss": 1.6214, + "nll_loss": 1.579624056816101, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1535051316022873, + "rewards/margins": 0.0571315735578537, + "rewards/rejected": -0.21063672006130219, + "step": 407 + }, + { + "epoch": 1.124266482568174, + "grad_norm": 0.17640192806720734, + "learning_rate": 3.943408322222049e-06, + "log_odds_chosen": 0.6473301649093628, + "log_odds_ratio": -0.430767297744751, + "logits/chosen": -0.25322699546813965, + "logits/rejected": -1.3362030982971191, + "logps/chosen": -1.4611663818359375, + "logps/rejected": -1.9962221384048462, + "loss": 1.5796, + "nll_loss": 1.5365506410598755, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14611662924289703, + "rewards/margins": 0.053505584597587585, + "rewards/rejected": -0.19962221384048462, + "step": 408 + }, + { + "epoch": 1.127027959958578, + "grad_norm": 0.1975967288017273, + "learning_rate": 3.936857369230037e-06, + "log_odds_chosen": 0.5013847351074219, + "log_odds_ratio": -0.48179811239242554, + "logits/chosen": -0.24628515541553497, + "logits/rejected": -1.5175528526306152, + "logps/chosen": -1.5307085514068604, + "logps/rejected": -1.9439282417297363, + "loss": 1.6331, + "nll_loss": 1.584876537322998, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.153070867061615, + "rewards/margins": 0.04132195562124252, + "rewards/rejected": -0.1943928301334381, + "step": 409 + }, + { + "epoch": 1.1297894373489816, + "grad_norm": 0.19114039838314056, + "learning_rate": 3.930291650305424e-06, + "log_odds_chosen": 0.6988283395767212, + "log_odds_ratio": -0.40597397089004517, + "logits/chosen": -0.25875231623649597, + "logits/rejected": -1.3210163116455078, + "logps/chosen": -1.4922490119934082, + "logps/rejected": -2.0712697505950928, + "loss": 1.5831, + "nll_loss": 1.5425041913986206, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1492249071598053, + "rewards/margins": 0.05790204927325249, + "rewards/rejected": -0.20712696015834808, + "step": 410 + }, + { + "epoch": 1.1325509147393855, + "grad_norm": 0.1878093183040619, + "learning_rate": 3.92371123292113e-06, + "log_odds_chosen": 0.6247770190238953, + "log_odds_ratio": -0.43160274624824524, + "logits/chosen": -0.2506449520587921, + "logits/rejected": -1.691251516342163, + "logps/chosen": -1.5277273654937744, + "logps/rejected": -2.0446317195892334, + "loss": 1.6273, + "nll_loss": 1.5841726064682007, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15277275443077087, + "rewards/margins": 0.051690444350242615, + "rewards/rejected": -0.2044631838798523, + "step": 411 + }, + { + "epoch": 1.1353123921297894, + "grad_norm": 0.1842024326324463, + "learning_rate": 3.917116184701125e-06, + "log_odds_chosen": 0.7279251217842102, + "log_odds_ratio": -0.4019698202610016, + "logits/chosen": -0.2969970107078552, + "logits/rejected": -1.9143730401992798, + "logps/chosen": -1.4923046827316284, + "logps/rejected": -2.099824905395508, + "loss": 1.5904, + "nll_loss": 1.5501734018325806, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1492304652929306, + "rewards/margins": 0.0607520155608654, + "rewards/rejected": -0.2099824696779251, + "step": 412 + }, + { + "epoch": 1.1380738695201933, + "grad_norm": 0.2017175853252411, + "learning_rate": 3.910506573419734e-06, + "log_odds_chosen": 0.8388389348983765, + "log_odds_ratio": -0.3656638562679291, + "logits/chosen": -0.27687397599220276, + "logits/rejected": -1.5076192617416382, + "logps/chosen": -1.4898481369018555, + "logps/rejected": -2.190901279449463, + "loss": 1.5889, + "nll_loss": 1.5523051023483276, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14898481965065002, + "rewards/margins": 0.07010531425476074, + "rewards/rejected": -0.21909013390541077, + "step": 413 + }, + { + "epoch": 1.1408353469105972, + "grad_norm": 0.18677456676959991, + "learning_rate": 3.903882467000938e-06, + "log_odds_chosen": 0.789838433265686, + "log_odds_ratio": -0.37885501980781555, + "logits/chosen": -0.2613790035247803, + "logits/rejected": -1.6607636213302612, + "logps/chosen": -1.5320932865142822, + "logps/rejected": -2.194924831390381, + "loss": 1.6278, + "nll_loss": 1.5898661613464355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15320934355258942, + "rewards/margins": 0.06628316640853882, + "rewards/rejected": -0.21949250996112823, + "step": 414 + }, + { + "epoch": 1.143596824301001, + "grad_norm": 0.18295292556285858, + "learning_rate": 3.897243933517679e-06, + "log_odds_chosen": 0.697784423828125, + "log_odds_ratio": -0.40981873869895935, + "logits/chosen": -0.36293089389801025, + "logits/rejected": -1.7186784744262695, + "logps/chosen": -1.4446918964385986, + "logps/rejected": -2.0167624950408936, + "loss": 1.5485, + "nll_loss": 1.50748872756958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14446918666362762, + "rewards/margins": 0.05720707029104233, + "rewards/rejected": -0.20167624950408936, + "step": 415 + }, + { + "epoch": 1.146358301691405, + "grad_norm": 0.18878361582756042, + "learning_rate": 3.890591041191162e-06, + "log_odds_chosen": 0.7591959834098816, + "log_odds_ratio": -0.39290934801101685, + "logits/chosen": -0.2790437340736389, + "logits/rejected": -1.4805063009262085, + "logps/chosen": -1.4139214754104614, + "logps/rejected": -2.03341007232666, + "loss": 1.5176, + "nll_loss": 1.478288173675537, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14139214158058167, + "rewards/margins": 0.06194887310266495, + "rewards/rejected": -0.20334100723266602, + "step": 416 + }, + { + "epoch": 1.1491197790818088, + "grad_norm": 0.2020062804222107, + "learning_rate": 3.883923858390149e-06, + "log_odds_chosen": 0.75853431224823, + "log_odds_ratio": -0.38675639033317566, + "logits/chosen": -0.3220999538898468, + "logits/rejected": -1.7704015970230103, + "logps/chosen": -1.5146489143371582, + "logps/rejected": -2.149966239929199, + "loss": 1.6306, + "nll_loss": 1.5919427871704102, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15146489441394806, + "rewards/margins": 0.06353174149990082, + "rewards/rejected": -0.21499663591384888, + "step": 417 + }, + { + "epoch": 1.1518812564722127, + "grad_norm": 0.17892806231975555, + "learning_rate": 3.8772424536302565e-06, + "log_odds_chosen": 0.5580451488494873, + "log_odds_ratio": -0.4568861126899719, + "logits/chosen": -0.3027711510658264, + "logits/rejected": -1.707384467124939, + "logps/chosen": -1.5017491579055786, + "logps/rejected": -1.9595609903335571, + "loss": 1.5943, + "nll_loss": 1.5486559867858887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15017491579055786, + "rewards/margins": 0.04578119143843651, + "rewards/rejected": -0.19595609605312347, + "step": 418 + }, + { + "epoch": 1.1546427338626164, + "grad_norm": 0.19901612401008606, + "learning_rate": 3.870546895573258e-06, + "log_odds_chosen": 0.6981452703475952, + "log_odds_ratio": -0.4115220010280609, + "logits/chosen": -0.3455246686935425, + "logits/rejected": -1.3123235702514648, + "logps/chosen": -1.479371428489685, + "logps/rejected": -2.0550196170806885, + "loss": 1.5787, + "nll_loss": 1.5375878810882568, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14793714880943298, + "rewards/margins": 0.05756482481956482, + "rewards/rejected": -0.2055019587278366, + "step": 419 + }, + { + "epoch": 1.1574042112530203, + "grad_norm": 0.1906924694776535, + "learning_rate": 3.863837253026372e-06, + "log_odds_chosen": 0.7406031489372253, + "log_odds_ratio": -0.40153640508651733, + "logits/chosen": -0.37284234166145325, + "logits/rejected": -1.6314030885696411, + "logps/chosen": -1.4350590705871582, + "logps/rejected": -2.0481374263763428, + "loss": 1.5474, + "nll_loss": 1.507211446762085, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14350590109825134, + "rewards/margins": 0.061307840049266815, + "rewards/rejected": -0.20481374859809875, + "step": 420 + }, + { + "epoch": 1.1601656886434242, + "grad_norm": 0.19268250465393066, + "learning_rate": 3.857113594941556e-06, + "log_odds_chosen": 0.5946828126907349, + "log_odds_ratio": -0.44611191749572754, + "logits/chosen": -0.3333417475223541, + "logits/rejected": -1.820539116859436, + "logps/chosen": -1.5357310771942139, + "logps/rejected": -2.030311346054077, + "loss": 1.6374, + "nll_loss": 1.5927873849868774, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15357311069965363, + "rewards/margins": 0.04945802688598633, + "rewards/rejected": -0.20303113758563995, + "step": 421 + }, + { + "epoch": 1.162927166033828, + "grad_norm": 0.1852002888917923, + "learning_rate": 3.8503759904148005e-06, + "log_odds_chosen": 0.753965437412262, + "log_odds_ratio": -0.39210668206214905, + "logits/chosen": -0.2807456851005554, + "logits/rejected": -1.7219178676605225, + "logps/chosen": -1.4659725427627563, + "logps/rejected": -2.0921542644500732, + "loss": 1.5699, + "nll_loss": 1.5306528806686401, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1465972512960434, + "rewards/margins": 0.0626181811094284, + "rewards/rejected": -0.209215447306633, + "step": 422 + }, + { + "epoch": 1.165688643424232, + "grad_norm": 0.18899178504943848, + "learning_rate": 3.843624508685416e-06, + "log_odds_chosen": 0.6049898266792297, + "log_odds_ratio": -0.43925538659095764, + "logits/chosen": -0.3452371060848236, + "logits/rejected": -1.7807867527008057, + "logps/chosen": -1.5395519733428955, + "logps/rejected": -2.040437698364258, + "loss": 1.6476, + "nll_loss": 1.603638768196106, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15395519137382507, + "rewards/margins": 0.05008859187364578, + "rewards/rejected": -0.20404377579689026, + "step": 423 + }, + { + "epoch": 1.1684501208146358, + "grad_norm": 0.1812516748905182, + "learning_rate": 3.8368592191353246e-06, + "log_odds_chosen": 0.6994820833206177, + "log_odds_ratio": -0.412604957818985, + "logits/chosen": -0.32768282294273376, + "logits/rejected": -1.7750557661056519, + "logps/chosen": -1.4714299440383911, + "logps/rejected": -2.0467658042907715, + "loss": 1.5632, + "nll_loss": 1.5219401121139526, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14714302122592926, + "rewards/margins": 0.05753358453512192, + "rewards/rejected": -0.20467659831047058, + "step": 424 + }, + { + "epoch": 1.1712115982050397, + "grad_norm": 0.18136382102966309, + "learning_rate": 3.830080191288342e-06, + "log_odds_chosen": 0.6253668665885925, + "log_odds_ratio": -0.42983123660087585, + "logits/chosen": -0.2867172360420227, + "logits/rejected": -1.5802618265151978, + "logps/chosen": -1.5029292106628418, + "logps/rejected": -2.018980026245117, + "loss": 1.6134, + "nll_loss": 1.5704439878463745, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15029294788837433, + "rewards/margins": 0.05160506069660187, + "rewards/rejected": -0.2018979787826538, + "step": 425 + }, + { + "epoch": 1.1739730755954436, + "grad_norm": 0.19747108221054077, + "learning_rate": 3.823287494809469e-06, + "log_odds_chosen": 0.7677461504936218, + "log_odds_ratio": -0.38692528009414673, + "logits/chosen": -0.38328537344932556, + "logits/rejected": -1.6817901134490967, + "logps/chosen": -1.4731775522232056, + "logps/rejected": -2.1067628860473633, + "loss": 1.5885, + "nll_loss": 1.5498508214950562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14731775224208832, + "rewards/margins": 0.06335853040218353, + "rewards/rejected": -0.21067628264427185, + "step": 426 + }, + { + "epoch": 1.1767345529858475, + "grad_norm": 0.185506671667099, + "learning_rate": 3.816481199504171e-06, + "log_odds_chosen": 0.6367388963699341, + "log_odds_ratio": -0.4264675974845886, + "logits/chosen": -0.3647797703742981, + "logits/rejected": -1.5863122940063477, + "logps/chosen": -1.5056264400482178, + "logps/rejected": -2.031801462173462, + "loss": 1.6257, + "nll_loss": 1.5830763578414917, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15056264400482178, + "rewards/margins": 0.052617497742176056, + "rewards/rejected": -0.20318013429641724, + "step": 427 + }, + { + "epoch": 1.1794960303762512, + "grad_norm": 0.1755794882774353, + "learning_rate": 3.8096613753176635e-06, + "log_odds_chosen": 0.584825873374939, + "log_odds_ratio": -0.4493084251880646, + "logits/chosen": -0.27612411975860596, + "logits/rejected": -1.4378138780593872, + "logps/chosen": -1.3924418687820435, + "logps/rejected": -1.8575388193130493, + "loss": 1.5104, + "nll_loss": 1.4654783010482788, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1392441838979721, + "rewards/margins": 0.046509698033332825, + "rewards/rejected": -0.18575388193130493, + "step": 428 + }, + { + "epoch": 1.1822575077666553, + "grad_norm": 0.19249552488327026, + "learning_rate": 3.8028280923341927e-06, + "log_odds_chosen": 0.6473675966262817, + "log_odds_ratio": -0.4286215901374817, + "logits/chosen": -0.384939968585968, + "logits/rejected": -1.5598773956298828, + "logps/chosen": -1.4334121942520142, + "logps/rejected": -1.9639126062393188, + "loss": 1.5432, + "nll_loss": 1.500383973121643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14334121346473694, + "rewards/margins": 0.05305003747344017, + "rewards/rejected": -0.1963912546634674, + "step": 429 + }, + { + "epoch": 1.185018985157059, + "grad_norm": 0.19077420234680176, + "learning_rate": 3.7959814207763134e-06, + "log_odds_chosen": 0.7534008026123047, + "log_odds_ratio": -0.3935543894767761, + "logits/chosen": -0.33575281500816345, + "logits/rejected": -1.7560871839523315, + "logps/chosen": -1.5460617542266846, + "logps/rejected": -2.17948579788208, + "loss": 1.6361, + "nll_loss": 1.5967589616775513, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1546061784029007, + "rewards/margins": 0.0633423924446106, + "rewards/rejected": -0.2179485559463501, + "step": 430 + }, + { + "epoch": 1.1877804625474628, + "grad_norm": 0.18546007573604584, + "learning_rate": 3.789121431004168e-06, + "log_odds_chosen": 0.6419503092765808, + "log_odds_ratio": -0.42676353454589844, + "logits/chosen": -0.36785295605659485, + "logits/rejected": -1.6887683868408203, + "logps/chosen": -1.5084519386291504, + "logps/rejected": -2.036573886871338, + "loss": 1.5997, + "nll_loss": 1.5570096969604492, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15084518492221832, + "rewards/margins": 0.05281219631433487, + "rewards/rejected": -0.2036573737859726, + "step": 431 + }, + { + "epoch": 1.1905419399378667, + "grad_norm": 0.194391131401062, + "learning_rate": 3.782248193514766e-06, + "log_odds_chosen": 0.6094563007354736, + "log_odds_ratio": -0.43929269909858704, + "logits/chosen": -0.29947635531425476, + "logits/rejected": -1.4955629110336304, + "logps/chosen": -1.4138904809951782, + "logps/rejected": -1.906821846961975, + "loss": 1.5403, + "nll_loss": 1.4963343143463135, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14138904213905334, + "rewards/margins": 0.049293152987957, + "rewards/rejected": -0.19068220257759094, + "step": 432 + }, + { + "epoch": 1.1933034173282706, + "grad_norm": 0.19166447222232819, + "learning_rate": 3.775361778941257e-06, + "log_odds_chosen": 0.7113713026046753, + "log_odds_ratio": -0.403501957654953, + "logits/chosen": -0.308038592338562, + "logits/rejected": -1.6283587217330933, + "logps/chosen": -1.501065731048584, + "logps/rejected": -2.0941624641418457, + "loss": 1.6079, + "nll_loss": 1.567560076713562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15010657906532288, + "rewards/margins": 0.05930966138839722, + "rewards/rejected": -0.2094162404537201, + "step": 433 + }, + { + "epoch": 1.1960648947186745, + "grad_norm": 0.18622690439224243, + "learning_rate": 3.7684622580522057e-06, + "log_odds_chosen": 0.42840951681137085, + "log_odds_ratio": -0.5051460862159729, + "logits/chosen": -0.2970636785030365, + "logits/rejected": -1.6902334690093994, + "logps/chosen": -1.4522030353546143, + "logps/rejected": -1.7967474460601807, + "loss": 1.5557, + "nll_loss": 1.5051684379577637, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1452203243970871, + "rewards/margins": 0.03445442020893097, + "rewards/rejected": -0.17967472970485687, + "step": 434 + }, + { + "epoch": 1.1988263721090784, + "grad_norm": 0.17388908565044403, + "learning_rate": 3.761549701750865e-06, + "log_odds_chosen": 0.7077789902687073, + "log_odds_ratio": -0.41333314776420593, + "logits/chosen": -0.3090924024581909, + "logits/rejected": -1.843423843383789, + "logps/chosen": -1.4652737379074097, + "logps/rejected": -2.052661418914795, + "loss": 1.5629, + "nll_loss": 1.5216063261032104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14652739465236664, + "rewards/margins": 0.058738768100738525, + "rewards/rejected": -0.20526614785194397, + "step": 435 + }, + { + "epoch": 1.2015878494994823, + "grad_norm": 0.17293590307235718, + "learning_rate": 3.7546241810744444e-06, + "log_odds_chosen": 0.5752092003822327, + "log_odds_ratio": -0.4525831639766693, + "logits/chosen": -0.32182741165161133, + "logits/rejected": -1.3567001819610596, + "logps/chosen": -1.4578838348388672, + "logps/rejected": -1.9286342859268188, + "loss": 1.5493, + "nll_loss": 1.50407075881958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14578840136528015, + "rewards/margins": 0.047075025737285614, + "rewards/rejected": -0.19286341965198517, + "step": 436 + }, + { + "epoch": 1.2043493268898862, + "grad_norm": 0.19718696177005768, + "learning_rate": 3.747685767193385e-06, + "log_odds_chosen": 0.5642775297164917, + "log_odds_ratio": -0.4569299817085266, + "logits/chosen": -0.3862457871437073, + "logits/rejected": -1.4825284481048584, + "logps/chosen": -1.5323418378829956, + "logps/rejected": -1.9992895126342773, + "loss": 1.6529, + "nll_loss": 1.6072373390197754, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15323419868946075, + "rewards/margins": 0.046694785356521606, + "rewards/rejected": -0.19992896914482117, + "step": 437 + }, + { + "epoch": 1.20711080428029, + "grad_norm": 0.17915678024291992, + "learning_rate": 3.740734531410626e-06, + "log_odds_chosen": 0.7052585482597351, + "log_odds_ratio": -0.4100540578365326, + "logits/chosen": -0.3775138258934021, + "logits/rejected": -1.859032154083252, + "logps/chosen": -1.448523998260498, + "logps/rejected": -2.021770477294922, + "loss": 1.5652, + "nll_loss": 1.5241587162017822, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1448523849248886, + "rewards/margins": 0.057324644178152084, + "rewards/rejected": -0.202177032828331, + "step": 438 + }, + { + "epoch": 1.2098722816706937, + "grad_norm": 0.19827017188072205, + "learning_rate": 3.7337705451608676e-06, + "log_odds_chosen": 0.859776496887207, + "log_odds_ratio": -0.3666639029979706, + "logits/chosen": -0.4492262601852417, + "logits/rejected": -1.819311261177063, + "logps/chosen": -1.3853495121002197, + "logps/rejected": -2.082892894744873, + "loss": 1.4954, + "nll_loss": 1.4586937427520752, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13853494822978973, + "rewards/margins": 0.06975432485342026, + "rewards/rejected": -0.2082892656326294, + "step": 439 + }, + { + "epoch": 1.2126337590610976, + "grad_norm": 0.19302192330360413, + "learning_rate": 3.7267938800098454e-06, + "log_odds_chosen": 0.8500153422355652, + "log_odds_ratio": -0.36537328362464905, + "logits/chosen": -0.5428920984268188, + "logits/rejected": -1.821568489074707, + "logps/chosen": -1.3568768501281738, + "logps/rejected": -2.052211046218872, + "loss": 1.4693, + "nll_loss": 1.4327150583267212, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1356876939535141, + "rewards/margins": 0.06953340768814087, + "rewards/rejected": -0.20522108674049377, + "step": 440 + }, + { + "epoch": 1.2153952364515015, + "grad_norm": 0.19974324107170105, + "learning_rate": 3.7198046076535865e-06, + "log_odds_chosen": 0.7102779746055603, + "log_odds_ratio": -0.40437954664230347, + "logits/chosen": -0.3492871820926666, + "logits/rejected": -1.5553520917892456, + "logps/chosen": -1.4951152801513672, + "logps/rejected": -2.0831298828125, + "loss": 1.6058, + "nll_loss": 1.565347671508789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14951153099536896, + "rewards/margins": 0.05880144238471985, + "rewards/rejected": -0.20831298828125, + "step": 441 + }, + { + "epoch": 1.2181567138419054, + "grad_norm": 0.18230760097503662, + "learning_rate": 3.71280279991768e-06, + "log_odds_chosen": 0.7506513595581055, + "log_odds_ratio": -0.3923317492008209, + "logits/chosen": -0.4152137339115143, + "logits/rejected": -1.5510238409042358, + "logps/chosen": -1.4440295696258545, + "logps/rejected": -2.0645270347595215, + "loss": 1.5654, + "nll_loss": 1.5261952877044678, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14440295100212097, + "rewards/margins": 0.06204976886510849, + "rewards/rejected": -0.20645272731781006, + "step": 442 + }, + { + "epoch": 1.2209181912323093, + "grad_norm": 0.17220567166805267, + "learning_rate": 3.705788528756533e-06, + "log_odds_chosen": 0.7942907810211182, + "log_odds_ratio": -0.37784573435783386, + "logits/chosen": -0.43315792083740234, + "logits/rejected": -1.8588192462921143, + "logps/chosen": -1.4370851516723633, + "logps/rejected": -2.0935044288635254, + "loss": 1.5386, + "nll_loss": 1.500786304473877, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14370852708816528, + "rewards/margins": 0.06564193964004517, + "rewards/rejected": -0.20935045182704926, + "step": 443 + }, + { + "epoch": 1.2236796686227132, + "grad_norm": 0.34615159034729004, + "learning_rate": 3.698761866252635e-06, + "log_odds_chosen": 0.6159027218818665, + "log_odds_ratio": -0.44577932357788086, + "logits/chosen": -0.4024880826473236, + "logits/rejected": -1.7161859273910522, + "logps/chosen": -1.533597707748413, + "logps/rejected": -2.050471067428589, + "loss": 1.6393, + "nll_loss": 1.5947318077087402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1533597856760025, + "rewards/margins": 0.0516873374581337, + "rewards/rejected": -0.2050471305847168, + "step": 444 + }, + { + "epoch": 1.226441146013117, + "grad_norm": 0.19309046864509583, + "learning_rate": 3.691722884615814e-06, + "log_odds_chosen": 0.6141983270645142, + "log_odds_ratio": -0.43802952766418457, + "logits/chosen": -0.2951904237270355, + "logits/rejected": -1.1809048652648926, + "logps/chosen": -1.4468181133270264, + "logps/rejected": -1.9419705867767334, + "loss": 1.5744, + "nll_loss": 1.5305941104888916, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14468181133270264, + "rewards/margins": 0.04951523244380951, + "rewards/rejected": -0.19419705867767334, + "step": 445 + }, + { + "epoch": 1.229202623403521, + "grad_norm": 0.19239898025989532, + "learning_rate": 3.684671656182497e-06, + "log_odds_chosen": 0.7434544563293457, + "log_odds_ratio": -0.39474281668663025, + "logits/chosen": -0.4174625277519226, + "logits/rejected": -1.7456690073013306, + "logps/chosen": -1.4840799570083618, + "logps/rejected": -2.0977742671966553, + "loss": 1.5985, + "nll_loss": 1.5590325593948364, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14840799570083618, + "rewards/margins": 0.06136942654848099, + "rewards/rejected": -0.20977741479873657, + "step": 446 + }, + { + "epoch": 1.2319641007939248, + "grad_norm": 0.19208678603172302, + "learning_rate": 3.6776082534149664e-06, + "log_odds_chosen": 0.7092225551605225, + "log_odds_ratio": -0.40430110692977905, + "logits/chosen": -0.4059605598449707, + "logits/rejected": -1.8414146900177002, + "logps/chosen": -1.5050946474075317, + "logps/rejected": -2.0934860706329346, + "loss": 1.5881, + "nll_loss": 1.5477027893066406, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15050947666168213, + "rewards/margins": 0.05883914604783058, + "rewards/rejected": -0.2093486189842224, + "step": 447 + }, + { + "epoch": 1.2347255781843287, + "grad_norm": 0.18131853640079498, + "learning_rate": 3.670532748900615e-06, + "log_odds_chosen": 0.7535479664802551, + "log_odds_ratio": -0.3983326554298401, + "logits/chosen": -0.3855303227901459, + "logits/rejected": -1.6423949003219604, + "logps/chosen": -1.4195005893707275, + "logps/rejected": -2.0328564643859863, + "loss": 1.5252, + "nll_loss": 1.4853616952896118, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14195004105567932, + "rewards/margins": 0.06133558601140976, + "rewards/rejected": -0.20328564941883087, + "step": 448 + }, + { + "epoch": 1.2374870555747326, + "grad_norm": 0.1773749440908432, + "learning_rate": 3.663445215351198e-06, + "log_odds_chosen": 0.794275164604187, + "log_odds_ratio": -0.38321515917778015, + "logits/chosen": -0.4386705160140991, + "logits/rejected": -1.5580754280090332, + "logps/chosen": -1.4094889163970947, + "logps/rejected": -2.067706823348999, + "loss": 1.5152, + "nll_loss": 1.476863145828247, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14094889163970947, + "rewards/margins": 0.06582178175449371, + "rewards/rejected": -0.20677067339420319, + "step": 449 + }, + { + "epoch": 1.2402485329651363, + "grad_norm": 0.18508228659629822, + "learning_rate": 3.656345725602089e-06, + "log_odds_chosen": 0.8697155714035034, + "log_odds_ratio": -0.35649657249450684, + "logits/chosen": -0.36322081089019775, + "logits/rejected": -1.8790079355239868, + "logps/chosen": -1.5029995441436768, + "logps/rejected": -2.235288143157959, + "loss": 1.5982, + "nll_loss": 1.5625407695770264, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15029993653297424, + "rewards/margins": 0.0732288807630539, + "rewards/rejected": -0.22352883219718933, + "step": 450 + }, + { + "epoch": 1.2430100103555402, + "grad_norm": 0.19414876401424408, + "learning_rate": 3.6492343526115292e-06, + "log_odds_chosen": 0.6894505023956299, + "log_odds_ratio": -0.4081804156303406, + "logits/chosen": -0.26924291253089905, + "logits/rejected": -1.8292944431304932, + "logps/chosen": -1.5266273021697998, + "logps/rejected": -2.100613594055176, + "loss": 1.612, + "nll_loss": 1.5711458921432495, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1526627391576767, + "rewards/margins": 0.05739862471818924, + "rewards/rejected": -0.21006137132644653, + "step": 451 + }, + { + "epoch": 1.245771487745944, + "grad_norm": 0.18495413661003113, + "learning_rate": 3.642111169459879e-06, + "log_odds_chosen": 0.5452620983123779, + "log_odds_ratio": -0.4608571529388428, + "logits/chosen": -0.3355942666530609, + "logits/rejected": -1.535355806350708, + "logps/chosen": -1.489123821258545, + "logps/rejected": -1.9351115226745605, + "loss": 1.5796, + "nll_loss": 1.533546805381775, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14891238510608673, + "rewards/margins": 0.044598765671253204, + "rewards/rejected": -0.19351115822792053, + "step": 452 + }, + { + "epoch": 1.248532965136348, + "grad_norm": 0.18209105730056763, + "learning_rate": 3.634976249348867e-06, + "log_odds_chosen": 0.6647239923477173, + "log_odds_ratio": -0.42399975657463074, + "logits/chosen": -0.3849475681781769, + "logits/rejected": -1.8075506687164307, + "logps/chosen": -1.4811009168624878, + "logps/rejected": -2.028481960296631, + "loss": 1.5873, + "nll_loss": 1.5449340343475342, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14811009168624878, + "rewards/margins": 0.05473810061812401, + "rewards/rejected": -0.2028481811285019, + "step": 453 + }, + { + "epoch": 1.2512944425267518, + "grad_norm": 0.20314469933509827, + "learning_rate": 3.6278296656008366e-06, + "log_odds_chosen": 0.8694459199905396, + "log_odds_ratio": -0.371795654296875, + "logits/chosen": -0.4006415903568268, + "logits/rejected": -1.7445772886276245, + "logps/chosen": -1.3579434156417847, + "logps/rejected": -2.065506935119629, + "loss": 1.4782, + "nll_loss": 1.4409888982772827, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13579432666301727, + "rewards/margins": 0.07075636833906174, + "rewards/rejected": -0.2065506875514984, + "step": 454 + }, + { + "epoch": 1.2540559199171557, + "grad_norm": 0.1716334968805313, + "learning_rate": 3.6206714916579925e-06, + "log_odds_chosen": 0.7703570127487183, + "log_odds_ratio": -0.3888989984989166, + "logits/chosen": -0.38477808237075806, + "logits/rejected": -1.7767083644866943, + "logps/chosen": -1.398410439491272, + "logps/rejected": -2.027221441268921, + "loss": 1.4987, + "nll_loss": 1.4598515033721924, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13984103500843048, + "rewards/margins": 0.06288108229637146, + "rewards/rejected": -0.20272211730480194, + "step": 455 + }, + { + "epoch": 1.2568173973075596, + "grad_norm": 0.18591812252998352, + "learning_rate": 3.613501801081648e-06, + "log_odds_chosen": 0.617323637008667, + "log_odds_ratio": -0.4372296929359436, + "logits/chosen": -0.36923906207084656, + "logits/rejected": -1.4284781217575073, + "logps/chosen": -1.4494431018829346, + "logps/rejected": -1.9530479907989502, + "loss": 1.5618, + "nll_loss": 1.5181188583374023, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14494431018829346, + "rewards/margins": 0.05036048963665962, + "rewards/rejected": -0.19530481100082397, + "step": 456 + }, + { + "epoch": 1.2595788746979635, + "grad_norm": 0.1984872668981552, + "learning_rate": 3.606320667551466e-06, + "log_odds_chosen": 0.7281745672225952, + "log_odds_ratio": -0.4030551612377167, + "logits/chosen": -0.36235833168029785, + "logits/rejected": -1.7550745010375977, + "logps/chosen": -1.49772310256958, + "logps/rejected": -2.1043448448181152, + "loss": 1.6083, + "nll_loss": 1.5679997205734253, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1497723013162613, + "rewards/margins": 0.06066218018531799, + "rewards/rejected": -0.21043448150157928, + "step": 457 + }, + { + "epoch": 1.2623403520883674, + "grad_norm": 0.1827681064605713, + "learning_rate": 3.599128164864706e-06, + "log_odds_chosen": 0.8096928596496582, + "log_odds_ratio": -0.37837105989456177, + "logits/chosen": -0.45445504784584045, + "logits/rejected": -1.770817756652832, + "logps/chosen": -1.4516648054122925, + "logps/rejected": -2.1250061988830566, + "loss": 1.5451, + "nll_loss": 1.5072699785232544, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14516648650169373, + "rewards/margins": 0.0673341453075409, + "rewards/rejected": -0.21250061690807343, + "step": 458 + }, + { + "epoch": 1.265101829478771, + "grad_norm": 0.19308075308799744, + "learning_rate": 3.5919243669354585e-06, + "log_odds_chosen": 0.6114473342895508, + "log_odds_ratio": -0.4423988461494446, + "logits/chosen": -0.405925452709198, + "logits/rejected": -1.2670363187789917, + "logps/chosen": -1.5176535844802856, + "logps/rejected": -2.0253612995147705, + "loss": 1.623, + "nll_loss": 1.5787138938903809, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1517653614282608, + "rewards/margins": 0.05077076703310013, + "rewards/rejected": -0.20253612101078033, + "step": 459 + }, + { + "epoch": 1.2678633068691751, + "grad_norm": 0.2073579728603363, + "learning_rate": 3.5847093477938955e-06, + "log_odds_chosen": 0.6397342681884766, + "log_odds_ratio": -0.4318113625049591, + "logits/chosen": -0.4677776098251343, + "logits/rejected": -1.5898178815841675, + "logps/chosen": -1.5188252925872803, + "logps/rejected": -2.048469066619873, + "loss": 1.6166, + "nll_loss": 1.5734236240386963, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15188252925872803, + "rewards/margins": 0.05296438932418823, + "rewards/rejected": -0.20484691858291626, + "step": 460 + }, + { + "epoch": 1.2706247842595788, + "grad_norm": 0.18931885063648224, + "learning_rate": 3.5774831815855017e-06, + "log_odds_chosen": 0.8001710772514343, + "log_odds_ratio": -0.3749019205570221, + "logits/chosen": -0.4591723084449768, + "logits/rejected": -1.6894080638885498, + "logps/chosen": -1.4038267135620117, + "logps/rejected": -2.0569498538970947, + "loss": 1.5196, + "nll_loss": 1.4821062088012695, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14038267731666565, + "rewards/margins": 0.06531231105327606, + "rewards/rejected": -0.2056949883699417, + "step": 461 + }, + { + "epoch": 1.2733862616499827, + "grad_norm": 0.17583203315734863, + "learning_rate": 3.5702459425703146e-06, + "log_odds_chosen": 0.8868230581283569, + "log_odds_ratio": -0.3547811210155487, + "logits/chosen": -0.43302029371261597, + "logits/rejected": -1.9412630796432495, + "logps/chosen": -1.508098840713501, + "logps/rejected": -2.258622884750366, + "loss": 1.6074, + "nll_loss": 1.5718796253204346, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1508098840713501, + "rewards/margins": 0.0750524029135704, + "rewards/rejected": -0.2258622944355011, + "step": 462 + }, + { + "epoch": 1.2761477390403866, + "grad_norm": 0.18731163442134857, + "learning_rate": 3.562997705122162e-06, + "log_odds_chosen": 0.7847945094108582, + "log_odds_ratio": -0.37928110361099243, + "logits/chosen": -0.38437145948410034, + "logits/rejected": -1.6156420707702637, + "logps/chosen": -1.4260213375091553, + "logps/rejected": -2.070751428604126, + "loss": 1.5279, + "nll_loss": 1.489925503730774, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14260214567184448, + "rewards/margins": 0.0644729733467102, + "rewards/rejected": -0.20707513391971588, + "step": 463 + }, + { + "epoch": 1.2789092164307905, + "grad_norm": 0.17475299537181854, + "learning_rate": 3.5557385437279e-06, + "log_odds_chosen": 0.838699221611023, + "log_odds_ratio": -0.36731863021850586, + "logits/chosen": -0.3592332601547241, + "logits/rejected": -1.4209747314453125, + "logps/chosen": -1.4949580430984497, + "logps/rejected": -2.196079730987549, + "loss": 1.5793, + "nll_loss": 1.5425434112548828, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14949579536914825, + "rewards/margins": 0.07011217623949051, + "rewards/rejected": -0.21960797905921936, + "step": 464 + }, + { + "epoch": 1.2816706938211944, + "grad_norm": 0.19598983228206635, + "learning_rate": 3.5484685329866424e-06, + "log_odds_chosen": 0.5413783192634583, + "log_odds_ratio": -0.4634351134300232, + "logits/chosen": -0.3708513081073761, + "logits/rejected": -1.4379222393035889, + "logps/chosen": -1.5153756141662598, + "logps/rejected": -1.9605016708374023, + "loss": 1.6065, + "nll_loss": 1.5601321458816528, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15153755247592926, + "rewards/margins": 0.044512614607810974, + "rewards/rejected": -0.19605018198490143, + "step": 465 + }, + { + "epoch": 1.2844321712115983, + "grad_norm": 0.18149885535240173, + "learning_rate": 3.541187747608998e-06, + "log_odds_chosen": 0.9340339303016663, + "log_odds_ratio": -0.34465107321739197, + "logits/chosen": -0.48461854457855225, + "logits/rejected": -1.7797619104385376, + "logps/chosen": -1.5739936828613281, + "logps/rejected": -2.3727049827575684, + "loss": 1.6472, + "nll_loss": 1.6127065420150757, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15739935636520386, + "rewards/margins": 0.07987111061811447, + "rewards/rejected": -0.23727048933506012, + "step": 466 + }, + { + "epoch": 1.2871936486020021, + "grad_norm": 0.19978304207324982, + "learning_rate": 3.533896262416302e-06, + "log_odds_chosen": 0.7202765941619873, + "log_odds_ratio": -0.4014014005661011, + "logits/chosen": -0.4147559404373169, + "logits/rejected": -1.5541059970855713, + "logps/chosen": -1.4875589609146118, + "logps/rejected": -2.0849876403808594, + "loss": 1.5897, + "nll_loss": 1.5495574474334717, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14875589311122894, + "rewards/margins": 0.05974285677075386, + "rewards/rejected": -0.2084987461566925, + "step": 467 + }, + { + "epoch": 1.2899551259924058, + "grad_norm": 0.21298463642597198, + "learning_rate": 3.5265941523398455e-06, + "log_odds_chosen": 0.8375830054283142, + "log_odds_ratio": -0.36393651366233826, + "logits/chosen": -0.42123696208000183, + "logits/rejected": -1.5165983438491821, + "logps/chosen": -1.5110690593719482, + "logps/rejected": -2.2138826847076416, + "loss": 1.6014, + "nll_loss": 1.5650498867034912, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15110690891742706, + "rewards/margins": 0.07028134167194366, + "rewards/rejected": -0.22138825058937073, + "step": 468 + }, + { + "epoch": 1.29271660338281, + "grad_norm": 0.1818659007549286, + "learning_rate": 3.519281492420108e-06, + "log_odds_chosen": 0.5199939012527466, + "log_odds_ratio": -0.47489961981773376, + "logits/chosen": -0.4219110906124115, + "logits/rejected": -1.4790453910827637, + "logps/chosen": -1.5349076986312866, + "logps/rejected": -1.9625968933105469, + "loss": 1.6236, + "nll_loss": 1.5760908126831055, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15349076688289642, + "rewards/margins": 0.042768917977809906, + "rewards/rejected": -0.19625969231128693, + "step": 469 + }, + { + "epoch": 1.2954780807732136, + "grad_norm": 0.19590230286121368, + "learning_rate": 3.5119583578059845e-06, + "log_odds_chosen": 0.6841270327568054, + "log_odds_ratio": -0.43147027492523193, + "logits/chosen": -0.4008702337741852, + "logits/rejected": -1.6819429397583008, + "logps/chosen": -1.6086608171463013, + "logps/rejected": -2.1926984786987305, + "loss": 1.6814, + "nll_loss": 1.6382827758789062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16086608171463013, + "rewards/margins": 0.05840376764535904, + "rewards/rejected": -0.21926987171173096, + "step": 470 + }, + { + "epoch": 1.2982395581636175, + "grad_norm": 0.17711324989795685, + "learning_rate": 3.504624823754014e-06, + "log_odds_chosen": 0.8761448860168457, + "log_odds_ratio": -0.3517749607563019, + "logits/chosen": -0.41212159395217896, + "logits/rejected": -1.957320213317871, + "logps/chosen": -1.4138109683990479, + "logps/rejected": -2.131588935852051, + "loss": 1.508, + "nll_loss": 1.4727998971939087, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14138111472129822, + "rewards/margins": 0.07177779823541641, + "rewards/rejected": -0.21315890550613403, + "step": 471 + }, + { + "epoch": 1.3010010355540214, + "grad_norm": 0.19494958221912384, + "learning_rate": 3.4972809656276047e-06, + "log_odds_chosen": 0.8046627640724182, + "log_odds_ratio": -0.3767206370830536, + "logits/chosen": -0.36885958909988403, + "logits/rejected": -1.8168095350265503, + "logps/chosen": -1.5026159286499023, + "logps/rejected": -2.176860809326172, + "loss": 1.6015, + "nll_loss": 1.5638599395751953, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15026158094406128, + "rewards/margins": 0.06742450594902039, + "rewards/rejected": -0.21768608689308167, + "step": 472 + }, + { + "epoch": 1.3037625129444252, + "grad_norm": 0.1766563355922699, + "learning_rate": 3.4899268588962613e-06, + "log_odds_chosen": 0.8576854467391968, + "log_odds_ratio": -0.3584885597229004, + "logits/chosen": -0.436603844165802, + "logits/rejected": -1.6729352474212646, + "logps/chosen": -1.3645590543746948, + "logps/rejected": -2.0580813884735107, + "loss": 1.4773, + "nll_loss": 1.4414961338043213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13645590841770172, + "rewards/margins": 0.06935223937034607, + "rewards/rejected": -0.20580816268920898, + "step": 473 + }, + { + "epoch": 1.3065239903348291, + "grad_norm": 0.18817074596881866, + "learning_rate": 3.4825625791348093e-06, + "log_odds_chosen": 0.8350617289543152, + "log_odds_ratio": -0.3708168864250183, + "logits/chosen": -0.3747791051864624, + "logits/rejected": -1.553459882736206, + "logps/chosen": -1.4124037027359009, + "logps/rejected": -2.1011507511138916, + "loss": 1.5145, + "nll_loss": 1.477430820465088, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14124037325382233, + "rewards/margins": 0.06887470930814743, + "rewards/rejected": -0.21011507511138916, + "step": 474 + }, + { + "epoch": 1.309285467725233, + "grad_norm": 0.1850864142179489, + "learning_rate": 3.4751882020226174e-06, + "log_odds_chosen": 0.7951568365097046, + "log_odds_ratio": -0.38151904940605164, + "logits/chosen": -0.41285592317581177, + "logits/rejected": -1.676164984703064, + "logps/chosen": -1.519218921661377, + "logps/rejected": -2.190464973449707, + "loss": 1.6106, + "nll_loss": 1.5724141597747803, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15192189812660217, + "rewards/margins": 0.06712460517883301, + "rewards/rejected": -0.21904650330543518, + "step": 475 + }, + { + "epoch": 1.312046945115637, + "grad_norm": 0.19581526517868042, + "learning_rate": 3.467803803342821e-06, + "log_odds_chosen": 0.7770808339118958, + "log_odds_ratio": -0.38217130303382874, + "logits/chosen": -0.3372090756893158, + "logits/rejected": -1.6060659885406494, + "logps/chosen": -1.4900453090667725, + "logps/rejected": -2.1375834941864014, + "loss": 1.5763, + "nll_loss": 1.5380480289459229, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14900454878807068, + "rewards/margins": 0.06475377827882767, + "rewards/rejected": -0.21375833451747894, + "step": 476 + }, + { + "epoch": 1.3148084225060408, + "grad_norm": 0.19793154299259186, + "learning_rate": 3.4604094589815402e-06, + "log_odds_chosen": 0.6940678358078003, + "log_odds_ratio": -0.40892940759658813, + "logits/chosen": -0.42878496646881104, + "logits/rejected": -1.5839377641677856, + "logps/chosen": -1.4118523597717285, + "logps/rejected": -1.9752473831176758, + "loss": 1.5164, + "nll_loss": 1.4755195379257202, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1411852389574051, + "rewards/margins": 0.056339483708143234, + "rewards/rejected": -0.19752474129199982, + "step": 477 + }, + { + "epoch": 1.3175698998964447, + "grad_norm": 0.18727529048919678, + "learning_rate": 3.4530052449271044e-06, + "log_odds_chosen": 0.8616414666175842, + "log_odds_ratio": -0.36362677812576294, + "logits/chosen": -0.4634791910648346, + "logits/rejected": -1.5600810050964355, + "logps/chosen": -1.4240953922271729, + "logps/rejected": -2.1343352794647217, + "loss": 1.521, + "nll_loss": 1.4846872091293335, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.142409548163414, + "rewards/margins": 0.07102398574352264, + "rewards/rejected": -0.21343351900577545, + "step": 478 + }, + { + "epoch": 1.3203313772868484, + "grad_norm": 0.19879932701587677, + "learning_rate": 3.4455912372692696e-06, + "log_odds_chosen": 0.5470461845397949, + "log_odds_ratio": -0.46493563055992126, + "logits/chosen": -0.3689558207988739, + "logits/rejected": -1.5988726615905762, + "logps/chosen": -1.4872140884399414, + "logps/rejected": -1.9357192516326904, + "loss": 1.5782, + "nll_loss": 1.5316959619522095, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14872139692306519, + "rewards/margins": 0.04485052451491356, + "rewards/rejected": -0.19357194006443024, + "step": 479 + }, + { + "epoch": 1.3230928546772525, + "grad_norm": 0.19459912180900574, + "learning_rate": 3.438167512198436e-06, + "log_odds_chosen": 0.5693470239639282, + "log_odds_ratio": -0.4518805146217346, + "logits/chosen": -0.41612759232521057, + "logits/rejected": -1.5004498958587646, + "logps/chosen": -1.5251268148422241, + "logps/rejected": -1.994539499282837, + "loss": 1.6246, + "nll_loss": 1.5794237852096558, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15251268446445465, + "rewards/margins": 0.046941258013248444, + "rewards/rejected": -0.1994539499282837, + "step": 480 + }, + { + "epoch": 1.3258543320676561, + "grad_norm": 0.1880672574043274, + "learning_rate": 3.4307341460048633e-06, + "log_odds_chosen": 0.7143114805221558, + "log_odds_ratio": -0.40364253520965576, + "logits/chosen": -0.4024220108985901, + "logits/rejected": -1.5502820014953613, + "logps/chosen": -1.5507985353469849, + "logps/rejected": -2.1468398571014404, + "loss": 1.6399, + "nll_loss": 1.5995802879333496, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15507985651493073, + "rewards/margins": 0.059604134410619736, + "rewards/rejected": -0.21468399465084076, + "step": 481 + }, + { + "epoch": 1.32861580945806, + "grad_norm": 0.19784514605998993, + "learning_rate": 3.4232912150778914e-06, + "log_odds_chosen": 0.7461116909980774, + "log_odds_ratio": -0.3941587507724762, + "logits/chosen": -0.43998983502388, + "logits/rejected": -1.7154864072799683, + "logps/chosen": -1.4538161754608154, + "logps/rejected": -2.0666849613189697, + "loss": 1.5731, + "nll_loss": 1.5336874723434448, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1453816145658493, + "rewards/margins": 0.06128688156604767, + "rewards/rejected": -0.20666849613189697, + "step": 482 + }, + { + "epoch": 1.331377286848464, + "grad_norm": 0.18405503034591675, + "learning_rate": 3.415838795905151e-06, + "log_odds_chosen": 0.9300730228424072, + "log_odds_ratio": -0.34047600626945496, + "logits/chosen": -0.5350120067596436, + "logits/rejected": -1.9728281497955322, + "logps/chosen": -1.4228546619415283, + "logps/rejected": -2.1867175102233887, + "loss": 1.5168, + "nll_loss": 1.4828009605407715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14228546619415283, + "rewards/margins": 0.07638627290725708, + "rewards/rejected": -0.2186717540025711, + "step": 483 + }, + { + "epoch": 1.3341387642388678, + "grad_norm": 0.19047723710536957, + "learning_rate": 3.408376965071779e-06, + "log_odds_chosen": 0.6742444634437561, + "log_odds_ratio": -0.41640961170196533, + "logits/chosen": -0.37510067224502563, + "logits/rejected": -1.5923813581466675, + "logps/chosen": -1.5178049802780151, + "logps/rejected": -2.0749661922454834, + "loss": 1.6055, + "nll_loss": 1.5638338327407837, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15178050100803375, + "rewards/margins": 0.05571611970663071, + "rewards/rejected": -0.20749662816524506, + "step": 484 + }, + { + "epoch": 1.3369002416292717, + "grad_norm": 0.1780618578195572, + "learning_rate": 3.400905799259634e-06, + "log_odds_chosen": 0.653221607208252, + "log_odds_ratio": -0.4260196387767792, + "logits/chosen": -0.42139092087745667, + "logits/rejected": -1.648254632949829, + "logps/chosen": -1.4374747276306152, + "logps/rejected": -1.9686298370361328, + "loss": 1.5329, + "nll_loss": 1.490256667137146, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.143747478723526, + "rewards/margins": 0.05311552435159683, + "rewards/rejected": -0.19686299562454224, + "step": 485 + }, + { + "epoch": 1.3396617190196756, + "grad_norm": 0.1804792881011963, + "learning_rate": 3.393425375246503e-06, + "log_odds_chosen": 0.6369755268096924, + "log_odds_ratio": -0.4262813925743103, + "logits/chosen": -0.4290909469127655, + "logits/rejected": -1.7290550470352173, + "logps/chosen": -1.4642930030822754, + "logps/rejected": -1.98381507396698, + "loss": 1.5519, + "nll_loss": 1.509236454963684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14642930030822754, + "rewards/margins": 0.05195220559835434, + "rewards/rejected": -0.1983814835548401, + "step": 486 + }, + { + "epoch": 1.3424231964100795, + "grad_norm": 0.19493341445922852, + "learning_rate": 3.3859357699053165e-06, + "log_odds_chosen": 0.7284951210021973, + "log_odds_ratio": -0.40332838892936707, + "logits/chosen": -0.40109413862228394, + "logits/rejected": -1.4352595806121826, + "logps/chosen": -1.402358055114746, + "logps/rejected": -1.9987090826034546, + "loss": 1.4967, + "nll_loss": 1.456404447555542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1402358114719391, + "rewards/margins": 0.059635113924741745, + "rewards/rejected": -0.19987091422080994, + "step": 487 + }, + { + "epoch": 1.3451846738004831, + "grad_norm": 0.19050319492816925, + "learning_rate": 3.3784370602033572e-06, + "log_odds_chosen": 0.8729457259178162, + "log_odds_ratio": -0.35851162672042847, + "logits/chosen": -0.4902225732803345, + "logits/rejected": -1.7542881965637207, + "logps/chosen": -1.4271174669265747, + "logps/rejected": -2.151813507080078, + "loss": 1.5277, + "nll_loss": 1.4917997121810913, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14271175861358643, + "rewards/margins": 0.07246959954500198, + "rewards/rejected": -0.2151813805103302, + "step": 488 + }, + { + "epoch": 1.3479461511908872, + "grad_norm": 0.18820421397686005, + "learning_rate": 3.3709293232014705e-06, + "log_odds_chosen": 0.73007732629776, + "log_odds_ratio": -0.40136751532554626, + "logits/chosen": -0.44850000739097595, + "logits/rejected": -2.0052490234375, + "logps/chosen": -1.4974483251571655, + "logps/rejected": -2.1092348098754883, + "loss": 1.5884, + "nll_loss": 1.5482368469238281, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14974482357501984, + "rewards/margins": 0.06117865815758705, + "rewards/rejected": -0.21092349290847778, + "step": 489 + }, + { + "epoch": 1.350707628581291, + "grad_norm": 0.18799753487110138, + "learning_rate": 3.3634126360532694e-06, + "log_odds_chosen": 0.8482156991958618, + "log_odds_ratio": -0.36712414026260376, + "logits/chosen": -0.32226884365081787, + "logits/rejected": -1.778417944908142, + "logps/chosen": -1.577460765838623, + "logps/rejected": -2.302645206451416, + "loss": 1.6644, + "nll_loss": 1.6277295351028442, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1577460616827011, + "rewards/margins": 0.07251846790313721, + "rewards/rejected": -0.23026452958583832, + "step": 490 + }, + { + "epoch": 1.3534691059716948, + "grad_norm": 0.20365531742572784, + "learning_rate": 3.355887076004345e-06, + "log_odds_chosen": 0.7709956765174866, + "log_odds_ratio": -0.383152574300766, + "logits/chosen": -0.5297287702560425, + "logits/rejected": -1.6290040016174316, + "logps/chosen": -1.4873484373092651, + "logps/rejected": -2.12845778465271, + "loss": 1.5899, + "nll_loss": 1.551561713218689, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14873485267162323, + "rewards/margins": 0.06411094218492508, + "rewards/rejected": -0.21284577250480652, + "step": 491 + }, + { + "epoch": 1.3562305833620987, + "grad_norm": 0.18781210482120514, + "learning_rate": 3.3483527203914694e-06, + "log_odds_chosen": 0.8746954798698425, + "log_odds_ratio": -0.3559949994087219, + "logits/chosen": -0.3871724307537079, + "logits/rejected": -1.6837961673736572, + "logps/chosen": -1.5181317329406738, + "logps/rejected": -2.254382610321045, + "loss": 1.613, + "nll_loss": 1.5773670673370361, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15181316435337067, + "rewards/margins": 0.0736251100897789, + "rewards/rejected": -0.22543828189373016, + "step": 492 + }, + { + "epoch": 1.3589920607525026, + "grad_norm": 0.18413974344730377, + "learning_rate": 3.340809646641805e-06, + "log_odds_chosen": 0.6966791749000549, + "log_odds_ratio": -0.4073706269264221, + "logits/chosen": -0.32745176553726196, + "logits/rejected": -1.734323501586914, + "logps/chosen": -1.5864430665969849, + "logps/rejected": -2.1757755279541016, + "loss": 1.6634, + "nll_loss": 1.6226191520690918, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15864431858062744, + "rewards/margins": 0.05893322825431824, + "rewards/rejected": -0.21757756173610687, + "step": 493 + }, + { + "epoch": 1.3617535381429065, + "grad_norm": 0.1795225441455841, + "learning_rate": 3.333257932272105e-06, + "log_odds_chosen": 0.7916382551193237, + "log_odds_ratio": -0.3854186236858368, + "logits/chosen": -0.3654117286205292, + "logits/rejected": -1.731317400932312, + "logps/chosen": -1.4387598037719727, + "logps/rejected": -2.0977869033813477, + "loss": 1.5401, + "nll_loss": 1.5015188455581665, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14387598633766174, + "rewards/margins": 0.0659027174115181, + "rewards/rejected": -0.20977871119976044, + "step": 494 + }, + { + "epoch": 1.3645150155333103, + "grad_norm": 0.19810736179351807, + "learning_rate": 3.3256976548879183e-06, + "log_odds_chosen": 0.7499178647994995, + "log_odds_ratio": -0.391695499420166, + "logits/chosen": -0.4586635231971741, + "logits/rejected": -1.5185637474060059, + "logps/chosen": -1.494145393371582, + "logps/rejected": -2.1175270080566406, + "loss": 1.591, + "nll_loss": 1.551873803138733, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1494145393371582, + "rewards/margins": 0.06233816593885422, + "rewards/rejected": -0.21175269782543182, + "step": 495 + }, + { + "epoch": 1.3672764929237142, + "grad_norm": 0.17777174711227417, + "learning_rate": 3.3181288921827925e-06, + "log_odds_chosen": 0.8323229551315308, + "log_odds_ratio": -0.37165793776512146, + "logits/chosen": -0.3486481308937073, + "logits/rejected": -1.4729896783828735, + "logps/chosen": -1.3969337940216064, + "logps/rejected": -2.0784289836883545, + "loss": 1.5068, + "nll_loss": 1.4696803092956543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13969337940216064, + "rewards/margins": 0.06814949959516525, + "rewards/rejected": -0.2078428864479065, + "step": 496 + }, + { + "epoch": 1.3700379703141181, + "grad_norm": 0.18033576011657715, + "learning_rate": 3.310551721937475e-06, + "log_odds_chosen": 0.7983404994010925, + "log_odds_ratio": -0.3745984435081482, + "logits/chosen": -0.3823201656341553, + "logits/rejected": -1.5644809007644653, + "logps/chosen": -1.4277198314666748, + "logps/rejected": -2.0840868949890137, + "loss": 1.5007, + "nll_loss": 1.4631915092468262, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14277197420597076, + "rewards/margins": 0.06563669443130493, + "rewards/rejected": -0.2084086835384369, + "step": 497 + }, + { + "epoch": 1.372799447704522, + "grad_norm": 0.19242247939109802, + "learning_rate": 3.3029662220191146e-06, + "log_odds_chosen": 0.8101353645324707, + "log_odds_ratio": -0.37228965759277344, + "logits/chosen": -0.4509270191192627, + "logits/rejected": -1.8056501150131226, + "logps/chosen": -1.3381811380386353, + "logps/rejected": -1.9927046298980713, + "loss": 1.4395, + "nll_loss": 1.4023054838180542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.133818119764328, + "rewards/margins": 0.06545236706733704, + "rewards/rejected": -0.19927047193050385, + "step": 498 + }, + { + "epoch": 1.3755609250949257, + "grad_norm": 0.1753472536802292, + "learning_rate": 3.2953724703804572e-06, + "log_odds_chosen": 0.892001211643219, + "log_odds_ratio": -0.35052934288978577, + "logits/chosen": -0.4044141173362732, + "logits/rejected": -1.8937467336654663, + "logps/chosen": -1.407699704170227, + "logps/rejected": -2.1476714611053467, + "loss": 1.4972, + "nll_loss": 1.4621860980987549, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14076997339725494, + "rewards/margins": 0.07399718463420868, + "rewards/rejected": -0.21476714313030243, + "step": 499 + }, + { + "epoch": 1.3783224024853298, + "grad_norm": 0.1958237588405609, + "learning_rate": 3.2877705450590525e-06, + "log_odds_chosen": 0.7319251298904419, + "log_odds_ratio": -0.396656334400177, + "logits/chosen": -0.3846544325351715, + "logits/rejected": -1.6542454957962036, + "logps/chosen": -1.4686310291290283, + "logps/rejected": -2.0715951919555664, + "loss": 1.5948, + "nll_loss": 1.5551836490631104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14686310291290283, + "rewards/margins": 0.06029640883207321, + "rewards/rejected": -0.20715951919555664, + "step": 500 + }, + { + "epoch": 1.3810838798757334, + "grad_norm": 0.17545589804649353, + "learning_rate": 3.2801605241764432e-06, + "log_odds_chosen": 0.788885235786438, + "log_odds_ratio": -0.3837747573852539, + "logits/chosen": -0.3873632252216339, + "logits/rejected": -1.8536286354064941, + "logps/chosen": -1.4469109773635864, + "logps/rejected": -2.0962677001953125, + "loss": 1.5495, + "nll_loss": 1.5111393928527832, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1446911245584488, + "rewards/margins": 0.06493565440177917, + "rewards/rejected": -0.20962676405906677, + "step": 501 + }, + { + "epoch": 1.3838453572661373, + "grad_norm": 0.1889375001192093, + "learning_rate": 3.272542485937369e-06, + "log_odds_chosen": 0.7263686656951904, + "log_odds_ratio": -0.4058024287223816, + "logits/chosen": -0.37034204602241516, + "logits/rejected": -1.4220876693725586, + "logps/chosen": -1.4947741031646729, + "logps/rejected": -2.097472906112671, + "loss": 1.594, + "nll_loss": 1.553464651107788, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14947742223739624, + "rewards/margins": 0.06026986241340637, + "rewards/rejected": -0.2097472846508026, + "step": 502 + }, + { + "epoch": 1.3866068346565412, + "grad_norm": 0.1781754493713379, + "learning_rate": 3.2649165086289597e-06, + "log_odds_chosen": 0.7348980903625488, + "log_odds_ratio": -0.3946687579154968, + "logits/chosen": -0.4843774437904358, + "logits/rejected": -1.973357081413269, + "logps/chosen": -1.4816904067993164, + "logps/rejected": -2.091848373413086, + "loss": 1.5609, + "nll_loss": 1.5214791297912598, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14816902577877045, + "rewards/margins": 0.06101579964160919, + "rewards/rejected": -0.20918481051921844, + "step": 503 + }, + { + "epoch": 1.389368312046945, + "grad_norm": 0.1925283521413803, + "learning_rate": 3.2572826706199304e-06, + "log_odds_chosen": 0.9842827916145325, + "log_odds_ratio": -0.32510116696357727, + "logits/chosen": -0.4544733464717865, + "logits/rejected": -1.6470658779144287, + "logps/chosen": -1.365575909614563, + "logps/rejected": -2.175246238708496, + "loss": 1.4834, + "nll_loss": 1.4508426189422607, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13655760884284973, + "rewards/margins": 0.08096704632043839, + "rewards/rejected": -0.21752463281154633, + "step": 504 + }, + { + "epoch": 1.392129789437349, + "grad_norm": 0.18144164979457855, + "learning_rate": 3.249641050359779e-06, + "log_odds_chosen": 0.7838388085365295, + "log_odds_ratio": -0.3822609782218933, + "logits/chosen": -0.3912584185600281, + "logits/rejected": -1.5951449871063232, + "logps/chosen": -1.5115033388137817, + "logps/rejected": -2.16564679145813, + "loss": 1.5972, + "nll_loss": 1.5589358806610107, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15115033090114594, + "rewards/margins": 0.06541436165571213, + "rewards/rejected": -0.21656471490859985, + "step": 505 + }, + { + "epoch": 1.3948912668277529, + "grad_norm": 0.1973390281200409, + "learning_rate": 3.2419917263779765e-06, + "log_odds_chosen": 0.6922333836555481, + "log_odds_ratio": -0.4096466302871704, + "logits/chosen": -0.45658183097839355, + "logits/rejected": -1.6782792806625366, + "logps/chosen": -1.4805402755737305, + "logps/rejected": -2.0515551567077637, + "loss": 1.5777, + "nll_loss": 1.5367363691329956, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14805401861667633, + "rewards/margins": 0.05710148066282272, + "rewards/rejected": -0.20515549182891846, + "step": 506 + }, + { + "epoch": 1.3976527442181568, + "grad_norm": 0.17680558562278748, + "learning_rate": 3.234334777283162e-06, + "log_odds_chosen": 0.9198102355003357, + "log_odds_ratio": -0.34391117095947266, + "logits/chosen": -0.5111406445503235, + "logits/rejected": -1.8106690645217896, + "logps/chosen": -1.4262008666992188, + "logps/rejected": -2.1901590824127197, + "loss": 1.5103, + "nll_loss": 1.4758594036102295, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14262010157108307, + "rewards/margins": 0.07639580965042114, + "rewards/rejected": -0.21901589632034302, + "step": 507 + }, + { + "epoch": 1.4004142216085607, + "grad_norm": 0.19250674545764923, + "learning_rate": 3.2266702817623348e-06, + "log_odds_chosen": 0.6659774780273438, + "log_odds_ratio": -0.418069452047348, + "logits/chosen": -0.42007410526275635, + "logits/rejected": -1.4803179502487183, + "logps/chosen": -1.502759575843811, + "logps/rejected": -2.0552127361297607, + "loss": 1.6133, + "nll_loss": 1.5714445114135742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15027597546577454, + "rewards/margins": 0.05524532124400139, + "rewards/rejected": -0.20552130043506622, + "step": 508 + }, + { + "epoch": 1.4031756989989645, + "grad_norm": 0.1782425045967102, + "learning_rate": 3.218998318580043e-06, + "log_odds_chosen": 0.9149441719055176, + "log_odds_ratio": -0.34030070900917053, + "logits/chosen": -0.46657681465148926, + "logits/rejected": -1.7872267961502075, + "logps/chosen": -1.4161348342895508, + "logps/rejected": -2.172861099243164, + "loss": 1.5157, + "nll_loss": 1.4816545248031616, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14161348342895508, + "rewards/margins": 0.07567264139652252, + "rewards/rejected": -0.2172861099243164, + "step": 509 + }, + { + "epoch": 1.4059371763893682, + "grad_norm": 0.18463608622550964, + "learning_rate": 3.2113189665775812e-06, + "log_odds_chosen": 0.7678059339523315, + "log_odds_ratio": -0.391030490398407, + "logits/chosen": -0.3615942597389221, + "logits/rejected": -1.817467451095581, + "logps/chosen": -1.511760950088501, + "logps/rejected": -2.154329299926758, + "loss": 1.6013, + "nll_loss": 1.5622117519378662, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15117612481117249, + "rewards/margins": 0.06425681710243225, + "rewards/rejected": -0.21543294191360474, + "step": 510 + }, + { + "epoch": 1.408698653779772, + "grad_norm": 0.18353889882564545, + "learning_rate": 3.203632304672172e-06, + "log_odds_chosen": 0.8818862438201904, + "log_odds_ratio": -0.35136285424232483, + "logits/chosen": -0.3583129942417145, + "logits/rejected": -1.9124095439910889, + "logps/chosen": -1.446837306022644, + "logps/rejected": -2.177347183227539, + "loss": 1.5495, + "nll_loss": 1.5143190622329712, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14468373358249664, + "rewards/margins": 0.07305099815130234, + "rewards/rejected": -0.21773472428321838, + "step": 511 + }, + { + "epoch": 1.411460131170176, + "grad_norm": 0.175659641623497, + "learning_rate": 3.1959384118561587e-06, + "log_odds_chosen": 1.1884584426879883, + "log_odds_ratio": -0.302362859249115, + "logits/chosen": -0.30318501591682434, + "logits/rejected": -1.7051109075546265, + "logps/chosen": -1.3106746673583984, + "logps/rejected": -2.27245831489563, + "loss": 1.432, + "nll_loss": 1.4017256498336792, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13106746971607208, + "rewards/margins": 0.09617836773395538, + "rewards/rejected": -0.22724585235118866, + "step": 512 + }, + { + "epoch": 1.4142216085605799, + "grad_norm": 0.18372024595737457, + "learning_rate": 3.188237367196194e-06, + "log_odds_chosen": 0.8523167371749878, + "log_odds_ratio": -0.3591596186161041, + "logits/chosen": -0.4926550090312958, + "logits/rejected": -1.6857538223266602, + "logps/chosen": -1.4602830410003662, + "logps/rejected": -2.1705353260040283, + "loss": 1.5673, + "nll_loss": 1.5313630104064941, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1460283100605011, + "rewards/margins": 0.07102521508932114, + "rewards/rejected": -0.21705353260040283, + "step": 513 + }, + { + "epoch": 1.4169830859509838, + "grad_norm": 0.19358229637145996, + "learning_rate": 3.180529249832428e-06, + "log_odds_chosen": 0.8924736976623535, + "log_odds_ratio": -0.3460920751094818, + "logits/chosen": -0.40075597167015076, + "logits/rejected": -1.7352858781814575, + "logps/chosen": -1.4665447473526, + "logps/rejected": -2.211902141571045, + "loss": 1.5629, + "nll_loss": 1.528282880783081, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14665447175502777, + "rewards/margins": 0.07453573495149612, + "rewards/rejected": -0.2211902141571045, + "step": 514 + }, + { + "epoch": 1.4197445633413877, + "grad_norm": 0.18455541133880615, + "learning_rate": 3.172814138977692e-06, + "log_odds_chosen": 0.6485744118690491, + "log_odds_ratio": -0.4242747128009796, + "logits/chosen": -0.40753471851348877, + "logits/rejected": -1.4140284061431885, + "logps/chosen": -1.449316143989563, + "logps/rejected": -1.9814183712005615, + "loss": 1.5443, + "nll_loss": 1.5018267631530762, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1449316143989563, + "rewards/margins": 0.05321022868156433, + "rewards/rejected": -0.19814184308052063, + "step": 515 + }, + { + "epoch": 1.4225060407317915, + "grad_norm": 0.19965727627277374, + "learning_rate": 3.165092113916688e-06, + "log_odds_chosen": 0.9415782690048218, + "log_odds_ratio": -0.3400614857673645, + "logits/chosen": -0.37811753153800964, + "logits/rejected": -1.5702593326568604, + "logps/chosen": -1.4428731203079224, + "logps/rejected": -2.229719877243042, + "loss": 1.5402, + "nll_loss": 1.5062329769134521, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1442873179912567, + "rewards/margins": 0.07868468761444092, + "rewards/rejected": -0.22297202050685883, + "step": 516 + }, + { + "epoch": 1.4252675181221954, + "grad_norm": 0.2037183940410614, + "learning_rate": 3.1573632540051702e-06, + "log_odds_chosen": 0.8820104598999023, + "log_odds_ratio": -0.35525357723236084, + "logits/chosen": -0.3366633355617523, + "logits/rejected": -1.3813198804855347, + "logps/chosen": -1.4109978675842285, + "logps/rejected": -2.1356987953186035, + "loss": 1.5272, + "nll_loss": 1.4916839599609375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14109979569911957, + "rewards/margins": 0.07247008383274078, + "rewards/rejected": -0.21356987953186035, + "step": 517 + }, + { + "epoch": 1.4280289955125993, + "grad_norm": 0.18941880762577057, + "learning_rate": 3.1496276386691327e-06, + "log_odds_chosen": 0.6363694071769714, + "log_odds_ratio": -0.4300464987754822, + "logits/chosen": -0.34271934628486633, + "logits/rejected": -1.6024858951568604, + "logps/chosen": -1.5349292755126953, + "logps/rejected": -2.0657947063446045, + "loss": 1.6149, + "nll_loss": 1.5718772411346436, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15349292755126953, + "rewards/margins": 0.053086526691913605, + "rewards/rejected": -0.20657946169376373, + "step": 518 + }, + { + "epoch": 1.430790472903003, + "grad_norm": 0.19241963326931, + "learning_rate": 3.1418853474039913e-06, + "log_odds_chosen": 0.9222308993339539, + "log_odds_ratio": -0.3464682102203369, + "logits/chosen": -0.46934691071510315, + "logits/rejected": -1.862980604171753, + "logps/chosen": -1.4347989559173584, + "logps/rejected": -2.1988108158111572, + "loss": 1.51, + "nll_loss": 1.4753470420837402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14347989857196808, + "rewards/margins": 0.07640118896961212, + "rewards/rejected": -0.219881072640419, + "step": 519 + }, + { + "epoch": 1.433551950293407, + "grad_norm": 0.19518575072288513, + "learning_rate": 3.1341364597737684e-06, + "log_odds_chosen": 0.7466192245483398, + "log_odds_ratio": -0.3926132619380951, + "logits/chosen": -0.3003273606300354, + "logits/rejected": -1.4642976522445679, + "logps/chosen": -1.4427096843719482, + "logps/rejected": -2.0578322410583496, + "loss": 1.5336, + "nll_loss": 1.49433434009552, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14427097141742706, + "rewards/margins": 0.06151225417852402, + "rewards/rejected": -0.20578324794769287, + "step": 520 + }, + { + "epoch": 1.4363134276838108, + "grad_norm": 0.19285590946674347, + "learning_rate": 3.126381055410274e-06, + "log_odds_chosen": 0.8940811157226562, + "log_odds_ratio": -0.34637144207954407, + "logits/chosen": -0.4038392901420593, + "logits/rejected": -1.4152494668960571, + "logps/chosen": -1.4457933902740479, + "logps/rejected": -2.187410593032837, + "loss": 1.5584, + "nll_loss": 1.523801326751709, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14457935094833374, + "rewards/margins": 0.07416172325611115, + "rewards/rejected": -0.2187410593032837, + "step": 521 + }, + { + "epoch": 1.4390749050742146, + "grad_norm": 0.19499003887176514, + "learning_rate": 3.1186192140122863e-06, + "log_odds_chosen": 0.6345283389091492, + "log_odds_ratio": -0.43169543147087097, + "logits/chosen": -0.39155280590057373, + "logits/rejected": -1.2614598274230957, + "logps/chosen": -1.4829864501953125, + "logps/rejected": -2.0023984909057617, + "loss": 1.5886, + "nll_loss": 1.5454603433609009, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14829865097999573, + "rewards/margins": 0.05194120854139328, + "rewards/rejected": -0.20023983716964722, + "step": 522 + }, + { + "epoch": 1.4418363824646185, + "grad_norm": 0.18813124299049377, + "learning_rate": 3.1108510153447352e-06, + "log_odds_chosen": 0.6801859140396118, + "log_odds_ratio": -0.4208618998527527, + "logits/chosen": -0.4520847797393799, + "logits/rejected": -1.5769951343536377, + "logps/chosen": -1.4872287511825562, + "logps/rejected": -2.0551376342773438, + "loss": 1.5849, + "nll_loss": 1.5428580045700073, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14872288703918457, + "rewards/margins": 0.05679089203476906, + "rewards/rejected": -0.20551377534866333, + "step": 523 + }, + { + "epoch": 1.4445978598550224, + "grad_norm": 0.1975877583026886, + "learning_rate": 3.1030765392378813e-06, + "log_odds_chosen": 0.9773280620574951, + "log_odds_ratio": -0.321841299533844, + "logits/chosen": -0.4134189486503601, + "logits/rejected": -1.5019309520721436, + "logps/chosen": -1.4216653108596802, + "logps/rejected": -2.2362189292907715, + "loss": 1.4987, + "nll_loss": 1.4664944410324097, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14216652512550354, + "rewards/margins": 0.08145537227392197, + "rewards/rejected": -0.2236218899488449, + "step": 524 + }, + { + "epoch": 1.4473593372454263, + "grad_norm": 0.19157834351062775, + "learning_rate": 3.0952958655864957e-06, + "log_odds_chosen": 0.8533503413200378, + "log_odds_ratio": -0.3656317889690399, + "logits/chosen": -0.4119202494621277, + "logits/rejected": -1.60611093044281, + "logps/chosen": -1.4607584476470947, + "logps/rejected": -2.171645402908325, + "loss": 1.5549, + "nll_loss": 1.5183472633361816, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14607584476470947, + "rewards/margins": 0.07108870148658752, + "rewards/rejected": -0.2171645313501358, + "step": 525 + }, + { + "epoch": 1.4501208146358302, + "grad_norm": 0.18692122399806976, + "learning_rate": 3.0875090743490383e-06, + "log_odds_chosen": 1.008112907409668, + "log_odds_ratio": -0.31326454877853394, + "logits/chosen": -0.32677963376045227, + "logits/rejected": -1.9253056049346924, + "logps/chosen": -1.4773204326629639, + "logps/rejected": -2.3293113708496094, + "loss": 1.5654, + "nll_loss": 1.5340484380722046, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14773203432559967, + "rewards/margins": 0.08519911766052246, + "rewards/rejected": -0.23293115198612213, + "step": 526 + }, + { + "epoch": 1.452882292026234, + "grad_norm": 0.20491987466812134, + "learning_rate": 3.0797162455468367e-06, + "log_odds_chosen": 0.9280487298965454, + "log_odds_ratio": -0.33601540327072144, + "logits/chosen": -0.3736734688282013, + "logits/rejected": -1.8332206010818481, + "logps/chosen": -1.4430752992630005, + "logps/rejected": -2.2161500453948975, + "loss": 1.5345, + "nll_loss": 1.5009381771087646, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14430750906467438, + "rewards/margins": 0.07730748504400253, + "rewards/rejected": -0.2216150015592575, + "step": 527 + }, + { + "epoch": 1.455643769416638, + "grad_norm": 0.19581927359104156, + "learning_rate": 3.071917459263264e-06, + "log_odds_chosen": 0.8374117016792297, + "log_odds_ratio": -0.36380189657211304, + "logits/chosen": -0.40068864822387695, + "logits/rejected": -1.559683084487915, + "logps/chosen": -1.479543685913086, + "logps/rejected": -2.176377296447754, + "loss": 1.5825, + "nll_loss": 1.546140432357788, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14795437455177307, + "rewards/margins": 0.06968337297439575, + "rewards/rejected": -0.21763774752616882, + "step": 528 + }, + { + "epoch": 1.4584052468070419, + "grad_norm": 0.18251369893550873, + "learning_rate": 3.0641127956429157e-06, + "log_odds_chosen": 1.0565541982650757, + "log_odds_ratio": -0.31225889921188354, + "logits/chosen": -0.3141689896583557, + "logits/rejected": -1.755129337310791, + "logps/chosen": -1.4867286682128906, + "logps/rejected": -2.3898322582244873, + "loss": 1.5731, + "nll_loss": 1.541866660118103, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14867284893989563, + "rewards/margins": 0.09031039476394653, + "rewards/rejected": -0.23898322880268097, + "step": 529 + }, + { + "epoch": 1.4611667241974455, + "grad_norm": 0.18326324224472046, + "learning_rate": 3.056302334890786e-06, + "log_odds_chosen": 0.9464606046676636, + "log_odds_ratio": -0.3374951481819153, + "logits/chosen": -0.4323669373989105, + "logits/rejected": -1.7882404327392578, + "logps/chosen": -1.4064451456069946, + "logps/rejected": -2.193542003631592, + "loss": 1.4906, + "nll_loss": 1.4568413496017456, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14064452052116394, + "rewards/margins": 0.0787096917629242, + "rewards/rejected": -0.21935419738292694, + "step": 530 + }, + { + "epoch": 1.4639282015878494, + "grad_norm": 0.18796159327030182, + "learning_rate": 3.0484861572714446e-06, + "log_odds_chosen": 1.0237690210342407, + "log_odds_ratio": -0.30970078706741333, + "logits/chosen": -0.4086054563522339, + "logits/rejected": -1.8015973567962646, + "logps/chosen": -1.4367576837539673, + "logps/rejected": -2.2936997413635254, + "loss": 1.5186, + "nll_loss": 1.4875980615615845, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1436757743358612, + "rewards/margins": 0.08569422364234924, + "rewards/rejected": -0.22937001287937164, + "step": 531 + }, + { + "epoch": 1.4666896789782533, + "grad_norm": 0.22190004587173462, + "learning_rate": 3.0406643431082088e-06, + "log_odds_chosen": 1.0039647817611694, + "log_odds_ratio": -0.3233025372028351, + "logits/chosen": -0.4348459243774414, + "logits/rejected": -1.7563586235046387, + "logps/chosen": -1.418906331062317, + "logps/rejected": -2.259153366088867, + "loss": 1.5147, + "nll_loss": 1.4823333024978638, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14189063012599945, + "rewards/margins": 0.08402469754219055, + "rewards/rejected": -0.2259153425693512, + "step": 532 + }, + { + "epoch": 1.4694511563686572, + "grad_norm": 0.19412145018577576, + "learning_rate": 3.0328369727823216e-06, + "log_odds_chosen": 0.7487189173698425, + "log_odds_ratio": -0.3932346701622009, + "logits/chosen": -0.3478846549987793, + "logits/rejected": -1.6251699924468994, + "logps/chosen": -1.4815884828567505, + "logps/rejected": -2.1053826808929443, + "loss": 1.5689, + "nll_loss": 1.5295873880386353, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14815884828567505, + "rewards/margins": 0.06237940490245819, + "rewards/rejected": -0.21053825318813324, + "step": 533 + }, + { + "epoch": 1.472212633759061, + "grad_norm": 0.18353740870952606, + "learning_rate": 3.0250041267321234e-06, + "log_odds_chosen": 0.847536563873291, + "log_odds_ratio": -0.360176146030426, + "logits/chosen": -0.4132736623287201, + "logits/rejected": -1.6758484840393066, + "logps/chosen": -1.3828142881393433, + "logps/rejected": -2.0753824710845947, + "loss": 1.492, + "nll_loss": 1.4560058116912842, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1382814347743988, + "rewards/margins": 0.06925684213638306, + "rewards/rejected": -0.20753827691078186, + "step": 534 + }, + { + "epoch": 1.474974111149465, + "grad_norm": 0.19951318204402924, + "learning_rate": 3.0171658854522274e-06, + "log_odds_chosen": 0.9957120418548584, + "log_odds_ratio": -0.3261812925338745, + "logits/chosen": -0.435792475938797, + "logits/rejected": -1.6330525875091553, + "logps/chosen": -1.4285228252410889, + "logps/rejected": -2.266305446624756, + "loss": 1.516, + "nll_loss": 1.4834091663360596, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1428522914648056, + "rewards/margins": 0.0837782695889473, + "rewards/rejected": -0.2266305685043335, + "step": 535 + }, + { + "epoch": 1.4777355885398689, + "grad_norm": 0.1980207860469818, + "learning_rate": 3.009322329492689e-06, + "log_odds_chosen": 0.665631890296936, + "log_odds_ratio": -0.41715607047080994, + "logits/chosen": -0.46294739842414856, + "logits/rejected": -1.6968400478363037, + "logps/chosen": -1.5159837007522583, + "logps/rejected": -2.067196846008301, + "loss": 1.5875, + "nll_loss": 1.5457911491394043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15159837901592255, + "rewards/margins": 0.05512130260467529, + "rewards/rejected": -0.20671968162059784, + "step": 536 + }, + { + "epoch": 1.4804970659302727, + "grad_norm": 0.1900826096534729, + "learning_rate": 3.0014735394581824e-06, + "log_odds_chosen": 0.747369110584259, + "log_odds_ratio": -0.3997749388217926, + "logits/chosen": -0.4459250867366791, + "logits/rejected": -1.5584537982940674, + "logps/chosen": -1.4036555290222168, + "logps/rejected": -2.0093021392822266, + "loss": 1.5147, + "nll_loss": 1.4746928215026855, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14036555588245392, + "rewards/margins": 0.06056464836001396, + "rewards/rejected": -0.20093020796775818, + "step": 537 + }, + { + "epoch": 1.4832585433206766, + "grad_norm": 0.18820805847644806, + "learning_rate": 2.993619596007168e-06, + "log_odds_chosen": 0.852586567401886, + "log_odds_ratio": -0.36225834488868713, + "logits/chosen": -0.37023186683654785, + "logits/rejected": -1.8086068630218506, + "logps/chosen": -1.5031466484069824, + "logps/rejected": -2.218027114868164, + "loss": 1.5854, + "nll_loss": 1.5491464138031006, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15031468868255615, + "rewards/margins": 0.07148806005716324, + "rewards/rejected": -0.2218027263879776, + "step": 538 + }, + { + "epoch": 1.4860200207110803, + "grad_norm": 0.18898199498653412, + "learning_rate": 2.985760579851068e-06, + "log_odds_chosen": 1.0741615295410156, + "log_odds_ratio": -0.3005737364292145, + "logits/chosen": -0.3823893070220947, + "logits/rejected": -1.7826778888702393, + "logps/chosen": -1.5318934917449951, + "logps/rejected": -2.454892158508301, + "loss": 1.6065, + "nll_loss": 1.5764073133468628, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15318936109542847, + "rewards/margins": 0.09229984879493713, + "rewards/rejected": -0.2454892247915268, + "step": 539 + }, + { + "epoch": 1.4887814981014844, + "grad_norm": 0.18205980956554413, + "learning_rate": 2.9778965717534314e-06, + "log_odds_chosen": 0.8840954303741455, + "log_odds_ratio": -0.36301591992378235, + "logits/chosen": -0.48036321997642517, + "logits/rejected": -1.776585340499878, + "logps/chosen": -1.3918758630752563, + "logps/rejected": -2.1138408184051514, + "loss": 1.4757, + "nll_loss": 1.4393746852874756, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13918758928775787, + "rewards/margins": 0.07219650596380234, + "rewards/rejected": -0.21138408780097961, + "step": 540 + }, + { + "epoch": 1.491542975491888, + "grad_norm": 0.17650777101516724, + "learning_rate": 2.9700276525291096e-06, + "log_odds_chosen": 1.0778427124023438, + "log_odds_ratio": -0.30152907967567444, + "logits/chosen": -0.4420645833015442, + "logits/rejected": -1.7818214893341064, + "logps/chosen": -1.3953425884246826, + "logps/rejected": -2.2949678897857666, + "loss": 1.4933, + "nll_loss": 1.4631969928741455, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13953426480293274, + "rewards/margins": 0.08996252715587616, + "rewards/rejected": -0.2294968068599701, + "step": 541 + }, + { + "epoch": 1.494304452882292, + "grad_norm": 0.18586601316928864, + "learning_rate": 2.9621539030434223e-06, + "log_odds_chosen": 1.0328391790390015, + "log_odds_ratio": -0.3124905228614807, + "logits/chosen": -0.5735373497009277, + "logits/rejected": -1.8510777950286865, + "logps/chosen": -1.4249876737594604, + "logps/rejected": -2.2932486534118652, + "loss": 1.5165, + "nll_loss": 1.4852372407913208, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14249876141548157, + "rewards/margins": 0.08682610094547272, + "rewards/rejected": -0.22932487726211548, + "step": 542 + }, + { + "epoch": 1.4970659302726959, + "grad_norm": 0.18669597804546356, + "learning_rate": 2.954275404211328e-06, + "log_odds_chosen": 0.8634792566299438, + "log_odds_ratio": -0.36653023958206177, + "logits/chosen": -0.3841334581375122, + "logits/rejected": -1.634179711341858, + "logps/chosen": -1.3906664848327637, + "logps/rejected": -2.0937647819519043, + "loss": 1.4983, + "nll_loss": 1.4616230726242065, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1390666514635086, + "rewards/margins": 0.0703098252415657, + "rewards/rejected": -0.2093764841556549, + "step": 543 + }, + { + "epoch": 1.4998274076630997, + "grad_norm": 0.19772256910800934, + "learning_rate": 2.946392236996592e-06, + "log_odds_chosen": 1.0479884147644043, + "log_odds_ratio": -0.31629127264022827, + "logits/chosen": -0.41537678241729736, + "logits/rejected": -1.7293639183044434, + "logps/chosen": -1.4252593517303467, + "logps/rejected": -2.3072445392608643, + "loss": 1.5216, + "nll_loss": 1.4899686574935913, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14252594113349915, + "rewards/margins": 0.08819850534200668, + "rewards/rejected": -0.23072443902492523, + "step": 544 + }, + { + "epoch": 1.5025888850535036, + "grad_norm": 0.20116113126277924, + "learning_rate": 2.9385044824109544e-06, + "log_odds_chosen": 0.9986646175384521, + "log_odds_ratio": -0.32834985852241516, + "logits/chosen": -0.39595213532447815, + "logits/rejected": -1.64156973361969, + "logps/chosen": -1.4510791301727295, + "logps/rejected": -2.2933764457702637, + "loss": 1.5398, + "nll_loss": 1.5069797039031982, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1451079100370407, + "rewards/margins": 0.0842297375202179, + "rewards/rejected": -0.2293376624584198, + "step": 545 + }, + { + "epoch": 1.5053503624439075, + "grad_norm": 0.1882292777299881, + "learning_rate": 2.9306122215132974e-06, + "log_odds_chosen": 0.9424600601196289, + "log_odds_ratio": -0.33976250886917114, + "logits/chosen": -0.3478313982486725, + "logits/rejected": -1.7016404867172241, + "logps/chosen": -1.4425792694091797, + "logps/rejected": -2.232656478881836, + "loss": 1.5428, + "nll_loss": 1.5088294744491577, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14425793290138245, + "rewards/margins": 0.07900771498680115, + "rewards/rejected": -0.2232656478881836, + "step": 546 + }, + { + "epoch": 1.5081118398343114, + "grad_norm": 0.2000649869441986, + "learning_rate": 2.9227155354088134e-06, + "log_odds_chosen": 0.8382570743560791, + "log_odds_ratio": -0.3624340891838074, + "logits/chosen": -0.5286014080047607, + "logits/rejected": -1.5262038707733154, + "logps/chosen": -1.4778207540512085, + "logps/rejected": -2.178725004196167, + "loss": 1.5501, + "nll_loss": 1.5138760805130005, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1477820873260498, + "rewards/margins": 0.0700904130935669, + "rewards/rejected": -0.2178725004196167, + "step": 547 + }, + { + "epoch": 1.510873317224715, + "grad_norm": 0.19744566082954407, + "learning_rate": 2.91481450524817e-06, + "log_odds_chosen": 1.2025346755981445, + "log_odds_ratio": -0.2766777575016022, + "logits/chosen": -0.4703354239463806, + "logits/rejected": -1.5989863872528076, + "logps/chosen": -1.4106305837631226, + "logps/rejected": -2.429147243499756, + "loss": 1.4928, + "nll_loss": 1.4651353359222412, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14106306433677673, + "rewards/margins": 0.10185164213180542, + "rewards/rejected": -0.24291470646858215, + "step": 548 + }, + { + "epoch": 1.5136347946151192, + "grad_norm": 0.18259872496128082, + "learning_rate": 2.9069092122266758e-06, + "log_odds_chosen": 1.020290732383728, + "log_odds_ratio": -0.31102436780929565, + "logits/chosen": -0.34419721364974976, + "logits/rejected": -1.7346612215042114, + "logps/chosen": -1.4083847999572754, + "logps/rejected": -2.256067991256714, + "loss": 1.4963, + "nll_loss": 1.4652248620986938, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14083848893642426, + "rewards/margins": 0.08476831018924713, + "rewards/rejected": -0.22560681402683258, + "step": 549 + }, + { + "epoch": 1.5163962720055228, + "grad_norm": 0.18138575553894043, + "learning_rate": 2.8989997375834485e-06, + "log_odds_chosen": 1.027559518814087, + "log_odds_ratio": -0.3187801241874695, + "logits/chosen": -0.43066591024398804, + "logits/rejected": -1.62416410446167, + "logps/chosen": -1.412578821182251, + "logps/rejected": -2.271681070327759, + "loss": 1.5043, + "nll_loss": 1.4724640846252441, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1412578821182251, + "rewards/margins": 0.08591020107269287, + "rewards/rejected": -0.22716809809207916, + "step": 550 + }, + { + "epoch": 1.519157749395927, + "grad_norm": 0.18563157320022583, + "learning_rate": 2.8910861626005774e-06, + "log_odds_chosen": 0.9574222564697266, + "log_odds_ratio": -0.3370060920715332, + "logits/chosen": -0.4931245446205139, + "logits/rejected": -1.7520966529846191, + "logps/chosen": -1.4311273097991943, + "logps/rejected": -2.229990243911743, + "loss": 1.5245, + "nll_loss": 1.4907807111740112, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14311271905899048, + "rewards/margins": 0.07988631725311279, + "rewards/rejected": -0.22299903631210327, + "step": 551 + }, + { + "epoch": 1.5219192267863306, + "grad_norm": 0.20305076241493225, + "learning_rate": 2.8831685686022897e-06, + "log_odds_chosen": 0.9472601413726807, + "log_odds_ratio": -0.3367205262184143, + "logits/chosen": -0.3746846616268158, + "logits/rejected": -1.5865817070007324, + "logps/chosen": -1.46114182472229, + "logps/rejected": -2.255194664001465, + "loss": 1.5544, + "nll_loss": 1.5207619667053223, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14611420035362244, + "rewards/margins": 0.0794052854180336, + "rewards/rejected": -0.22551947832107544, + "step": 552 + }, + { + "epoch": 1.5246807041767345, + "grad_norm": 0.20478032529354095, + "learning_rate": 2.8752470369541152e-06, + "log_odds_chosen": 0.9435425400733948, + "log_odds_ratio": -0.341013640165329, + "logits/chosen": -0.47928714752197266, + "logits/rejected": -1.607470154762268, + "logps/chosen": -1.493147850036621, + "logps/rejected": -2.2932393550872803, + "loss": 1.585, + "nll_loss": 1.550898790359497, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14931480586528778, + "rewards/margins": 0.08000914007425308, + "rewards/rejected": -0.22932393848896027, + "step": 553 + }, + { + "epoch": 1.5274421815671384, + "grad_norm": 0.18675510585308075, + "learning_rate": 2.8673216490620453e-06, + "log_odds_chosen": 0.8560038208961487, + "log_odds_ratio": -0.3590989112854004, + "logits/chosen": -0.3732084035873413, + "logits/rejected": -1.5269240140914917, + "logps/chosen": -1.4434525966644287, + "logps/rejected": -2.1552090644836426, + "loss": 1.5403, + "nll_loss": 1.5043420791625977, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1443452537059784, + "rewards/margins": 0.07117564976215363, + "rewards/rejected": -0.21552088856697083, + "step": 554 + }, + { + "epoch": 1.5302036589575423, + "grad_norm": 0.19521136581897736, + "learning_rate": 2.859392486371705e-06, + "log_odds_chosen": 0.9180530905723572, + "log_odds_ratio": -0.3567126989364624, + "logits/chosen": -0.49841001629829407, + "logits/rejected": -1.4869251251220703, + "logps/chosen": -1.4458491802215576, + "logps/rejected": -2.2207460403442383, + "loss": 1.5457, + "nll_loss": 1.5100224018096924, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14458490908145905, + "rewards/margins": 0.07748968154191971, + "rewards/rejected": -0.22207459807395935, + "step": 555 + }, + { + "epoch": 1.5329651363479462, + "grad_norm": 0.20385955274105072, + "learning_rate": 2.8514596303675073e-06, + "log_odds_chosen": 1.151139497756958, + "log_odds_ratio": -0.2817905843257904, + "logits/chosen": -0.43373167514801025, + "logits/rejected": -1.7193479537963867, + "logps/chosen": -1.4189940690994263, + "logps/rejected": -2.3919098377227783, + "loss": 1.5087, + "nll_loss": 1.4805082082748413, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14189940690994263, + "rewards/margins": 0.09729158133268356, + "rewards/rejected": -0.2391909807920456, + "step": 556 + }, + { + "epoch": 1.53572661373835, + "grad_norm": 0.19316573441028595, + "learning_rate": 2.8435231625718242e-06, + "log_odds_chosen": 1.0120395421981812, + "log_odds_ratio": -0.3164248764514923, + "logits/chosen": -0.38262811303138733, + "logits/rejected": -1.806386113166809, + "logps/chosen": -1.4623085260391235, + "logps/rejected": -2.3146116733551025, + "loss": 1.5372, + "nll_loss": 1.5055807828903198, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14623086154460907, + "rewards/margins": 0.08523032069206238, + "rewards/rejected": -0.23146119713783264, + "step": 557 + }, + { + "epoch": 1.538488091128754, + "grad_norm": 2.1178505420684814, + "learning_rate": 2.835583164544139e-06, + "log_odds_chosen": 0.9513680934906006, + "log_odds_ratio": -0.33420196175575256, + "logits/chosen": -0.33462485671043396, + "logits/rejected": -1.6407803297042847, + "logps/chosen": -1.4331406354904175, + "logps/rejected": -2.228959798812866, + "loss": 1.5297, + "nll_loss": 1.4962306022644043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14331406354904175, + "rewards/margins": 0.07958193868398666, + "rewards/rejected": -0.22289599478244781, + "step": 558 + }, + { + "epoch": 1.5412495685191576, + "grad_norm": 0.19349446892738342, + "learning_rate": 2.827639717880218e-06, + "log_odds_chosen": 0.8153694272041321, + "log_odds_ratio": -0.3702642023563385, + "logits/chosen": -0.41866424679756165, + "logits/rejected": -1.768775463104248, + "logps/chosen": -1.4800782203674316, + "logps/rejected": -2.1566810607910156, + "loss": 1.552, + "nll_loss": 1.5150169134140015, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1480078101158142, + "rewards/margins": 0.06766027957201004, + "rewards/rejected": -0.21566811203956604, + "step": 559 + }, + { + "epoch": 1.5440110459095617, + "grad_norm": 0.20243015885353088, + "learning_rate": 2.8196929042112652e-06, + "log_odds_chosen": 0.7393404841423035, + "log_odds_ratio": -0.3941328823566437, + "logits/chosen": -0.3861313462257385, + "logits/rejected": -1.5598360300064087, + "logps/chosen": -1.514575481414795, + "logps/rejected": -2.1315693855285645, + "loss": 1.6067, + "nll_loss": 1.5672829151153564, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1514575481414795, + "rewards/margins": 0.06169939786195755, + "rewards/rejected": -0.21315695345401764, + "step": 560 + }, + { + "epoch": 1.5467725232999654, + "grad_norm": 0.18742065131664276, + "learning_rate": 2.811742805203087e-06, + "log_odds_chosen": 1.0400229692459106, + "log_odds_ratio": -0.3049144148826599, + "logits/chosen": -0.44754305481910706, + "logits/rejected": -1.7995396852493286, + "logps/chosen": -1.401616096496582, + "logps/rejected": -2.268768548965454, + "loss": 1.4907, + "nll_loss": 1.4601788520812988, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14016160368919373, + "rewards/margins": 0.0867152288556099, + "rewards/rejected": -0.22687683999538422, + "step": 561 + }, + { + "epoch": 1.5495340006903695, + "grad_norm": 0.1959799826145172, + "learning_rate": 2.8037895025552513e-06, + "log_odds_chosen": 0.782373309135437, + "log_odds_ratio": -0.38005301356315613, + "logits/chosen": -0.3823511302471161, + "logits/rejected": -1.534692645072937, + "logps/chosen": -1.5261653661727905, + "logps/rejected": -2.181563377380371, + "loss": 1.6025, + "nll_loss": 1.5644484758377075, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15261651575565338, + "rewards/margins": 0.06553982943296432, + "rewards/rejected": -0.2181563377380371, + "step": 562 + }, + { + "epoch": 1.5522954780807732, + "grad_norm": 0.18624603748321533, + "learning_rate": 2.7958330780002472e-06, + "log_odds_chosen": 1.0219098329544067, + "log_odds_ratio": -0.31670254468917847, + "logits/chosen": -0.48139262199401855, + "logits/rejected": -1.540400505065918, + "logps/chosen": -1.4481538534164429, + "logps/rejected": -2.306190013885498, + "loss": 1.5315, + "nll_loss": 1.4998620748519897, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14481540024280548, + "rewards/margins": 0.08580362796783447, + "rewards/rejected": -0.23061901330947876, + "step": 563 + }, + { + "epoch": 1.555056955471177, + "grad_norm": 0.21115663647651672, + "learning_rate": 2.787873613302649e-06, + "log_odds_chosen": 0.8930285573005676, + "log_odds_ratio": -0.34711483120918274, + "logits/chosen": -0.3769160807132721, + "logits/rejected": -1.5557960271835327, + "logps/chosen": -1.5821139812469482, + "logps/rejected": -2.338306427001953, + "loss": 1.6636, + "nll_loss": 1.628933310508728, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15821139514446259, + "rewards/margins": 0.07561925053596497, + "rewards/rejected": -0.23383064568042755, + "step": 564 + }, + { + "epoch": 1.557818432861581, + "grad_norm": 0.19150473177433014, + "learning_rate": 2.7799111902582697e-06, + "log_odds_chosen": 1.0769083499908447, + "log_odds_ratio": -0.29914119839668274, + "logits/chosen": -0.4121115207672119, + "logits/rejected": -1.5894091129302979, + "logps/chosen": -1.4135100841522217, + "logps/rejected": -2.3172075748443604, + "loss": 1.4981, + "nll_loss": 1.4682036638259888, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14135099947452545, + "rewards/margins": 0.09036976844072342, + "rewards/rejected": -0.23172077536582947, + "step": 565 + }, + { + "epoch": 1.5605799102519848, + "grad_norm": 0.20271562039852142, + "learning_rate": 2.7719458906933277e-06, + "log_odds_chosen": 0.9160252809524536, + "log_odds_ratio": -0.34139731526374817, + "logits/chosen": -0.4353451728820801, + "logits/rejected": -1.7503840923309326, + "logps/chosen": -1.4142577648162842, + "logps/rejected": -2.1746983528137207, + "loss": 1.5083, + "nll_loss": 1.4741257429122925, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14142578840255737, + "rewards/margins": 0.07604406028985977, + "rewards/rejected": -0.21746984124183655, + "step": 566 + }, + { + "epoch": 1.5633413876423887, + "grad_norm": 0.18848182260990143, + "learning_rate": 2.763977796463599e-06, + "log_odds_chosen": 0.9602777361869812, + "log_odds_ratio": -0.32593464851379395, + "logits/chosen": -0.49435821175575256, + "logits/rejected": -1.9882686138153076, + "logps/chosen": -1.4518954753875732, + "logps/rejected": -2.2552900314331055, + "loss": 1.5211, + "nll_loss": 1.488489031791687, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.145189568400383, + "rewards/margins": 0.08033942431211472, + "rewards/rejected": -0.22552898526191711, + "step": 567 + }, + { + "epoch": 1.5661028650327924, + "grad_norm": 0.21147844195365906, + "learning_rate": 2.7560069894535783e-06, + "log_odds_chosen": 0.8455137014389038, + "log_odds_ratio": -0.3622473478317261, + "logits/chosen": -0.3838009536266327, + "logits/rejected": -1.5061184167861938, + "logps/chosen": -1.4856162071228027, + "logps/rejected": -2.1930487155914307, + "loss": 1.5775, + "nll_loss": 1.5413126945495605, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14856162667274475, + "rewards/margins": 0.07074324786663055, + "rewards/rejected": -0.2193048745393753, + "step": 568 + }, + { + "epoch": 1.5688643424231965, + "grad_norm": 0.2110675871372223, + "learning_rate": 2.748033551575644e-06, + "log_odds_chosen": 0.7732433676719666, + "log_odds_ratio": -0.3829563856124878, + "logits/chosen": -0.5417189598083496, + "logits/rejected": -1.4143304824829102, + "logps/chosen": -1.4991943836212158, + "logps/rejected": -2.145965099334717, + "loss": 1.5934, + "nll_loss": 1.555059552192688, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14991945028305054, + "rewards/margins": 0.06467706710100174, + "rewards/rejected": -0.21459650993347168, + "step": 569 + }, + { + "epoch": 1.5716258198136002, + "grad_norm": 0.20080991089344025, + "learning_rate": 2.7400575647692046e-06, + "log_odds_chosen": 0.8514761924743652, + "log_odds_ratio": -0.3591695725917816, + "logits/chosen": -0.469234824180603, + "logits/rejected": -1.8506807088851929, + "logps/chosen": -1.5175936222076416, + "logps/rejected": -2.235103130340576, + "loss": 1.607, + "nll_loss": 1.5710902214050293, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15175937116146088, + "rewards/margins": 0.0717509537935257, + "rewards/rejected": -0.22351032495498657, + "step": 570 + }, + { + "epoch": 1.5743872972040043, + "grad_norm": 0.19513045251369476, + "learning_rate": 2.7320791109998655e-06, + "log_odds_chosen": 0.9315704107284546, + "log_odds_ratio": -0.3407260477542877, + "logits/chosen": -0.620419442653656, + "logits/rejected": -1.8977861404418945, + "logps/chosen": -1.4148740768432617, + "logps/rejected": -2.1817188262939453, + "loss": 1.495, + "nll_loss": 1.4609739780426025, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14148743450641632, + "rewards/margins": 0.07668447494506836, + "rewards/rejected": -0.2181718945503235, + "step": 571 + }, + { + "epoch": 1.577148774594408, + "grad_norm": 0.19687050580978394, + "learning_rate": 2.724098272258584e-06, + "log_odds_chosen": 0.9079073667526245, + "log_odds_ratio": -0.34860873222351074, + "logits/chosen": -0.48484548926353455, + "logits/rejected": -1.6853313446044922, + "logps/chosen": -1.4053500890731812, + "logps/rejected": -2.1579606533050537, + "loss": 1.4975, + "nll_loss": 1.4626667499542236, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14053499698638916, + "rewards/margins": 0.07526105642318726, + "rewards/rejected": -0.21579605340957642, + "step": 572 + }, + { + "epoch": 1.5799102519848118, + "grad_norm": 0.1954527497291565, + "learning_rate": 2.7161151305608258e-06, + "log_odds_chosen": 0.9750263690948486, + "log_odds_ratio": -0.3271198272705078, + "logits/chosen": -0.51035475730896, + "logits/rejected": -1.5407557487487793, + "logps/chosen": -1.3590997457504272, + "logps/rejected": -2.162524461746216, + "loss": 1.4519, + "nll_loss": 1.4191510677337646, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13590997457504272, + "rewards/margins": 0.08034246414899826, + "rewards/rejected": -0.2162524312734604, + "step": 573 + }, + { + "epoch": 1.5826717293752157, + "grad_norm": 0.19047684967517853, + "learning_rate": 2.7081297679457238e-06, + "log_odds_chosen": 0.9220460653305054, + "log_odds_ratio": -0.3471813499927521, + "logits/chosen": -0.5562020540237427, + "logits/rejected": -1.592578411102295, + "logps/chosen": -1.3342963457107544, + "logps/rejected": -2.0851826667785645, + "loss": 1.4261, + "nll_loss": 1.3913562297821045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1334296315908432, + "rewards/margins": 0.07508864253759384, + "rewards/rejected": -0.20851828157901764, + "step": 574 + }, + { + "epoch": 1.5854332067656196, + "grad_norm": 0.1876901239156723, + "learning_rate": 2.7001422664752338e-06, + "log_odds_chosen": 0.8865346908569336, + "log_odds_ratio": -0.353354275226593, + "logits/chosen": -0.46136918663978577, + "logits/rejected": -1.9794909954071045, + "logps/chosen": -1.4452863931655884, + "logps/rejected": -2.18290376663208, + "loss": 1.5254, + "nll_loss": 1.4900963306427002, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14452862739562988, + "rewards/margins": 0.07376174628734589, + "rewards/rejected": -0.21829037368297577, + "step": 575 + }, + { + "epoch": 1.5881946841560235, + "grad_norm": 0.21001578867435455, + "learning_rate": 2.692152708233292e-06, + "log_odds_chosen": 0.9853892922401428, + "log_odds_ratio": -0.32194337248802185, + "logits/chosen": -0.4604001045227051, + "logits/rejected": -1.8486415147781372, + "logps/chosen": -1.4580106735229492, + "logps/rejected": -2.287992477416992, + "loss": 1.5497, + "nll_loss": 1.5175025463104248, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14580106735229492, + "rewards/margins": 0.08299816399812698, + "rewards/rejected": -0.2287992388010025, + "step": 576 + }, + { + "epoch": 1.5909561615464274, + "grad_norm": 0.17950667440891266, + "learning_rate": 2.684161175324971e-06, + "log_odds_chosen": 1.089317798614502, + "log_odds_ratio": -0.3076907992362976, + "logits/chosen": -0.45762938261032104, + "logits/rejected": -1.8938159942626953, + "logps/chosen": -1.3867409229278564, + "logps/rejected": -2.298753261566162, + "loss": 1.4599, + "nll_loss": 1.4291636943817139, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13867409527301788, + "rewards/margins": 0.09120123088359833, + "rewards/rejected": -0.2298753261566162, + "step": 577 + }, + { + "epoch": 1.5937176389368313, + "grad_norm": 0.19735175371170044, + "learning_rate": 2.676167749875635e-06, + "log_odds_chosen": 0.8711864948272705, + "log_odds_ratio": -0.359083890914917, + "logits/chosen": -0.46582692861557007, + "logits/rejected": -1.7266442775726318, + "logps/chosen": -1.4628636837005615, + "logps/rejected": -2.1918888092041016, + "loss": 1.5503, + "nll_loss": 1.514415979385376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14628636837005615, + "rewards/margins": 0.07290251553058624, + "rewards/rejected": -0.2191888988018036, + "step": 578 + }, + { + "epoch": 1.596479116327235, + "grad_norm": 0.20369943976402283, + "learning_rate": 2.6681725140300995e-06, + "log_odds_chosen": 1.0683999061584473, + "log_odds_ratio": -0.3045715093612671, + "logits/chosen": -0.5854548215866089, + "logits/rejected": -1.6316256523132324, + "logps/chosen": -1.365673542022705, + "logps/rejected": -2.2548635005950928, + "loss": 1.4494, + "nll_loss": 1.4189079999923706, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1365673542022705, + "rewards/margins": 0.08891897648572922, + "rewards/rejected": -0.22548633813858032, + "step": 579 + }, + { + "epoch": 1.599240593717639, + "grad_norm": 0.19143253564834595, + "learning_rate": 2.6601755499517826e-06, + "log_odds_chosen": 0.8805921077728271, + "log_odds_ratio": -0.35098567605018616, + "logits/chosen": -0.46608656644821167, + "logits/rejected": -1.950951337814331, + "logps/chosen": -1.4723143577575684, + "logps/rejected": -2.210609197616577, + "loss": 1.5573, + "nll_loss": 1.5221997499465942, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14723142981529236, + "rewards/margins": 0.07382948696613312, + "rewards/rejected": -0.22106093168258667, + "step": 580 + }, + { + "epoch": 1.6020020711080427, + "grad_norm": 0.20576678216457367, + "learning_rate": 2.6521769398218635e-06, + "log_odds_chosen": 1.0742186307907104, + "log_odds_ratio": -0.2951492369174957, + "logits/chosen": -0.45335662364959717, + "logits/rejected": -1.8008517026901245, + "logps/chosen": -1.462270736694336, + "logps/rejected": -2.3689029216766357, + "loss": 1.5473, + "nll_loss": 1.5177663564682007, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14622707664966583, + "rewards/margins": 0.09066321700811386, + "rewards/rejected": -0.2368902862071991, + "step": 581 + }, + { + "epoch": 1.6047635484984468, + "grad_norm": 0.22372370958328247, + "learning_rate": 2.6441767658384363e-06, + "log_odds_chosen": 1.1760811805725098, + "log_odds_ratio": -0.28239208459854126, + "logits/chosen": -0.5110992789268494, + "logits/rejected": -1.7087138891220093, + "logps/chosen": -1.408003807067871, + "logps/rejected": -2.3959457874298096, + "loss": 1.4905, + "nll_loss": 1.4622644186019897, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1408003866672516, + "rewards/margins": 0.09879419207572937, + "rewards/rejected": -0.23959457874298096, + "step": 582 + }, + { + "epoch": 1.6075250258888505, + "grad_norm": 0.19179660081863403, + "learning_rate": 2.6361751102156673e-06, + "log_odds_chosen": 1.0498160123825073, + "log_odds_ratio": -0.3036152720451355, + "logits/chosen": -0.5851718783378601, + "logits/rejected": -1.7799510955810547, + "logps/chosen": -1.4760764837265015, + "logps/rejected": -2.3658981323242188, + "loss": 1.5528, + "nll_loss": 1.5223983526229858, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14760765433311462, + "rewards/margins": 0.08898216485977173, + "rewards/rejected": -0.23658981919288635, + "step": 583 + }, + { + "epoch": 1.6102865032792544, + "grad_norm": 0.19203205406665802, + "learning_rate": 2.628172055182948e-06, + "log_odds_chosen": 1.0640352964401245, + "log_odds_ratio": -0.3011913001537323, + "logits/chosen": -0.502800703048706, + "logits/rejected": -1.6559865474700928, + "logps/chosen": -1.411937952041626, + "logps/rejected": -2.296353340148926, + "loss": 1.4964, + "nll_loss": 1.4662748575210571, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14119380712509155, + "rewards/margins": 0.08844152092933655, + "rewards/rejected": -0.2296353280544281, + "step": 584 + }, + { + "epoch": 1.6130479806696583, + "grad_norm": 0.21048317849636078, + "learning_rate": 2.620167682984052e-06, + "log_odds_chosen": 1.0453581809997559, + "log_odds_ratio": -0.3030966818332672, + "logits/chosen": -0.44269949197769165, + "logits/rejected": -1.7648762464523315, + "logps/chosen": -1.5006442070007324, + "logps/rejected": -2.386173963546753, + "loss": 1.5706, + "nll_loss": 1.5402867794036865, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15006442368030548, + "rewards/margins": 0.08855296671390533, + "rewards/rejected": -0.23861739039421082, + "step": 585 + }, + { + "epoch": 1.6158094580600622, + "grad_norm": 0.202589213848114, + "learning_rate": 2.6121620758762877e-06, + "log_odds_chosen": 1.2401037216186523, + "log_odds_ratio": -0.26773184537887573, + "logits/chosen": -0.4954875409603119, + "logits/rejected": -1.7728123664855957, + "logps/chosen": -1.4029090404510498, + "logps/rejected": -2.4527475833892822, + "loss": 1.4836, + "nll_loss": 1.456812858581543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14029091596603394, + "rewards/margins": 0.10498383641242981, + "rewards/rejected": -0.24527473747730255, + "step": 586 + }, + { + "epoch": 1.618570935450466, + "grad_norm": 0.19367991387844086, + "learning_rate": 2.604155316129654e-06, + "log_odds_chosen": 1.1511355638504028, + "log_odds_ratio": -0.2958333492279053, + "logits/chosen": -0.41890662908554077, + "logits/rejected": -1.7125511169433594, + "logps/chosen": -1.370666742324829, + "logps/rejected": -2.336446762084961, + "loss": 1.4669, + "nll_loss": 1.4373064041137695, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13706667721271515, + "rewards/margins": 0.09657800942659378, + "rewards/rejected": -0.23364469408988953, + "step": 587 + }, + { + "epoch": 1.6213324128408697, + "grad_norm": 0.20538587868213654, + "learning_rate": 2.596147486025996e-06, + "log_odds_chosen": 0.998070240020752, + "log_odds_ratio": -0.32918909192085266, + "logits/chosen": -0.4908704161643982, + "logits/rejected": -1.7168567180633545, + "logps/chosen": -1.4522796869277954, + "logps/rejected": -2.2951455116271973, + "loss": 1.542, + "nll_loss": 1.5090749263763428, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14522796869277954, + "rewards/margins": 0.08428659290075302, + "rewards/rejected": -0.22951453924179077, + "step": 588 + }, + { + "epoch": 1.6240938902312738, + "grad_norm": 0.19525809586048126, + "learning_rate": 2.5881386678581587e-06, + "log_odds_chosen": 0.9631695747375488, + "log_odds_ratio": -0.33941492438316345, + "logits/chosen": -0.4752748906612396, + "logits/rejected": -1.823403239250183, + "logps/chosen": -1.4721821546554565, + "logps/rejected": -2.2869250774383545, + "loss": 1.5529, + "nll_loss": 1.5189189910888672, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14721821248531342, + "rewards/margins": 0.08147431910037994, + "rewards/rejected": -0.22869251668453217, + "step": 589 + }, + { + "epoch": 1.6268553676216775, + "grad_norm": 0.18992260098457336, + "learning_rate": 2.580128943929139e-06, + "log_odds_chosen": 1.0529534816741943, + "log_odds_ratio": -0.3118128478527069, + "logits/chosen": -0.5909528732299805, + "logits/rejected": -1.7769283056259155, + "logps/chosen": -1.482077956199646, + "logps/rejected": -2.3781533241271973, + "loss": 1.5494, + "nll_loss": 1.5182218551635742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14820778369903564, + "rewards/margins": 0.08960752934217453, + "rewards/rejected": -0.23781533539295197, + "step": 590 + }, + { + "epoch": 1.6296168450120816, + "grad_norm": 0.19947484135627747, + "learning_rate": 2.5721183965512424e-06, + "log_odds_chosen": 0.9602290391921997, + "log_odds_ratio": -0.33199790120124817, + "logits/chosen": -0.4957779049873352, + "logits/rejected": -1.8233221769332886, + "logps/chosen": -1.4387155771255493, + "logps/rejected": -2.243359327316284, + "loss": 1.5226, + "nll_loss": 1.4893605709075928, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14387157559394836, + "rewards/margins": 0.08046435564756393, + "rewards/rejected": -0.2243359237909317, + "step": 591 + }, + { + "epoch": 1.6323783224024853, + "grad_norm": 0.19560271501541138, + "learning_rate": 2.564107108045239e-06, + "log_odds_chosen": 0.9684425592422485, + "log_odds_ratio": -0.33081498742103577, + "logits/chosen": -0.390799880027771, + "logits/rejected": -1.8621724843978882, + "logps/chosen": -1.5033597946166992, + "logps/rejected": -2.32216215133667, + "loss": 1.5717, + "nll_loss": 1.5385994911193848, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15033598244190216, + "rewards/margins": 0.08188023418188095, + "rewards/rejected": -0.2322162240743637, + "step": 592 + }, + { + "epoch": 1.6351397997928891, + "grad_norm": 0.21181620657444, + "learning_rate": 2.556095160739513e-06, + "log_odds_chosen": 1.1007102727890015, + "log_odds_ratio": -0.29589590430259705, + "logits/chosen": -0.4980851411819458, + "logits/rejected": -1.648532748222351, + "logps/chosen": -1.4862791299819946, + "logps/rejected": -2.4272263050079346, + "loss": 1.5887, + "nll_loss": 1.5591251850128174, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14862790703773499, + "rewards/margins": 0.09409470856189728, + "rewards/rejected": -0.24272264540195465, + "step": 593 + }, + { + "epoch": 1.637901277183293, + "grad_norm": 0.20149773359298706, + "learning_rate": 2.5480826369692178e-06, + "log_odds_chosen": 0.94913649559021, + "log_odds_ratio": -0.35477903485298157, + "logits/chosen": -0.47597599029541016, + "logits/rejected": -1.6722043752670288, + "logps/chosen": -1.4268722534179688, + "logps/rejected": -2.231581211090088, + "loss": 1.521, + "nll_loss": 1.485567331314087, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14268723130226135, + "rewards/margins": 0.08047088980674744, + "rewards/rejected": -0.2231581062078476, + "step": 594 + }, + { + "epoch": 1.640662754573697, + "grad_norm": 0.19771575927734375, + "learning_rate": 2.5400696190754347e-06, + "log_odds_chosen": 1.0618021488189697, + "log_odds_ratio": -0.3072146475315094, + "logits/chosen": -0.5567762851715088, + "logits/rejected": -1.8684656620025635, + "logps/chosen": -1.4032098054885864, + "logps/rejected": -2.292144298553467, + "loss": 1.4948, + "nll_loss": 1.464051604270935, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14032098650932312, + "rewards/margins": 0.08889344334602356, + "rewards/rejected": -0.22921442985534668, + "step": 595 + }, + { + "epoch": 1.6434242319641008, + "grad_norm": 0.1921544373035431, + "learning_rate": 2.532056189404318e-06, + "log_odds_chosen": 1.0136642456054688, + "log_odds_ratio": -0.31946948170661926, + "logits/chosen": -0.5976563096046448, + "logits/rejected": -1.6406235694885254, + "logps/chosen": -1.3311948776245117, + "logps/rejected": -2.164565086364746, + "loss": 1.4056, + "nll_loss": 1.3736519813537598, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13311949372291565, + "rewards/margins": 0.08333701640367508, + "rewards/rejected": -0.21645651757717133, + "step": 596 + }, + { + "epoch": 1.6461857093545047, + "grad_norm": 0.20298127830028534, + "learning_rate": 2.524042430306258e-06, + "log_odds_chosen": 1.0893070697784424, + "log_odds_ratio": -0.29967525601387024, + "logits/chosen": -0.5517418384552002, + "logits/rejected": -1.6711257696151733, + "logps/chosen": -1.431249976158142, + "logps/rejected": -2.353149652481079, + "loss": 1.5109, + "nll_loss": 1.4809296131134033, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1431249976158142, + "rewards/margins": 0.0921899825334549, + "rewards/rejected": -0.2353149801492691, + "step": 597 + }, + { + "epoch": 1.6489471867449086, + "grad_norm": 0.2075520008802414, + "learning_rate": 2.516028424135028e-06, + "log_odds_chosen": 0.9435156583786011, + "log_odds_ratio": -0.331230103969574, + "logits/chosen": -0.47791504859924316, + "logits/rejected": -1.588948369026184, + "logps/chosen": -1.4235913753509521, + "logps/rejected": -2.206226110458374, + "loss": 1.5238, + "nll_loss": 1.490726351737976, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14235913753509521, + "rewards/margins": 0.07826346904039383, + "rewards/rejected": -0.22062261402606964, + "step": 598 + }, + { + "epoch": 1.6517086641353123, + "grad_norm": 0.3242340087890625, + "learning_rate": 2.5080142532469405e-06, + "log_odds_chosen": 0.8534775972366333, + "log_odds_ratio": -0.35872241854667664, + "logits/chosen": -0.5117841958999634, + "logits/rejected": -1.6608119010925293, + "logps/chosen": -1.383968710899353, + "logps/rejected": -2.084062099456787, + "loss": 1.4697, + "nll_loss": 1.4338546991348267, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13839687407016754, + "rewards/margins": 0.07000933587551117, + "rewards/rejected": -0.2084062248468399, + "step": 599 + }, + { + "epoch": 1.6544701415257164, + "grad_norm": 0.18982619047164917, + "learning_rate": 2.5e-06, + "log_odds_chosen": 1.0374600887298584, + "log_odds_ratio": -0.30684134364128113, + "logits/chosen": -0.47200512886047363, + "logits/rejected": -1.692277431488037, + "logps/chosen": -1.3962559700012207, + "logps/rejected": -2.260599374771118, + "loss": 1.4793, + "nll_loss": 1.448584794998169, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13962560892105103, + "rewards/margins": 0.08643434941768646, + "rewards/rejected": -0.2260599583387375, + "step": 600 + }, + { + "epoch": 1.65723161891612, + "grad_norm": 0.21345901489257812, + "learning_rate": 2.4919857467530608e-06, + "log_odds_chosen": 1.1221760511398315, + "log_odds_ratio": -0.2836793065071106, + "logits/chosen": -0.6063240766525269, + "logits/rejected": -1.7040760517120361, + "logps/chosen": -1.4183056354522705, + "logps/rejected": -2.36080265045166, + "loss": 1.5098, + "nll_loss": 1.4814752340316772, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14183056354522705, + "rewards/margins": 0.09424971044063568, + "rewards/rejected": -0.23608025908470154, + "step": 601 + }, + { + "epoch": 1.6599930963065241, + "grad_norm": 0.21028882265090942, + "learning_rate": 2.4839715758649724e-06, + "log_odds_chosen": 0.9369592070579529, + "log_odds_ratio": -0.3362049162387848, + "logits/chosen": -0.45639631152153015, + "logits/rejected": -1.7908605337142944, + "logps/chosen": -1.5038936138153076, + "logps/rejected": -2.2945337295532227, + "loss": 1.5641, + "nll_loss": 1.5304303169250488, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15038937330245972, + "rewards/margins": 0.07906400412321091, + "rewards/rejected": -0.22945338487625122, + "step": 602 + }, + { + "epoch": 1.6627545736969278, + "grad_norm": 0.20318543910980225, + "learning_rate": 2.475957569693742e-06, + "log_odds_chosen": 1.1263985633850098, + "log_odds_ratio": -0.28691041469573975, + "logits/chosen": -0.5157269835472107, + "logits/rejected": -1.812491774559021, + "logps/chosen": -1.4623472690582275, + "logps/rejected": -2.419386625289917, + "loss": 1.5437, + "nll_loss": 1.515026330947876, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14623473584651947, + "rewards/margins": 0.09570392221212387, + "rewards/rejected": -0.24193865060806274, + "step": 603 + }, + { + "epoch": 1.6655160510873317, + "grad_norm": 0.2132967710494995, + "learning_rate": 2.467943810595682e-06, + "log_odds_chosen": 0.7572492361068726, + "log_odds_ratio": -0.3912786543369293, + "logits/chosen": -0.6214781999588013, + "logits/rejected": -1.4917678833007812, + "logps/chosen": -1.4525686502456665, + "logps/rejected": -2.0807745456695557, + "loss": 1.5376, + "nll_loss": 1.4985002279281616, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1452568620443344, + "rewards/margins": 0.06282058358192444, + "rewards/rejected": -0.20807746052742004, + "step": 604 + }, + { + "epoch": 1.6682775284777356, + "grad_norm": 0.19940869510173798, + "learning_rate": 2.459930380924566e-06, + "log_odds_chosen": 1.0550998449325562, + "log_odds_ratio": -0.30236515402793884, + "logits/chosen": -0.44110098481178284, + "logits/rejected": -1.6045345067977905, + "logps/chosen": -1.4446752071380615, + "logps/rejected": -2.332336902618408, + "loss": 1.5361, + "nll_loss": 1.5058379173278809, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1444675326347351, + "rewards/margins": 0.0887661725282669, + "rewards/rejected": -0.23323370516300201, + "step": 605 + }, + { + "epoch": 1.6710390058681395, + "grad_norm": 0.2036145180463791, + "learning_rate": 2.4519173630307826e-06, + "log_odds_chosen": 0.9749938249588013, + "log_odds_ratio": -0.33162128925323486, + "logits/chosen": -0.5159994959831238, + "logits/rejected": -1.5376782417297363, + "logps/chosen": -1.3822424411773682, + "logps/rejected": -2.1887948513031006, + "loss": 1.463, + "nll_loss": 1.4298853874206543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13822424411773682, + "rewards/margins": 0.08065523952245712, + "rewards/rejected": -0.21887947618961334, + "step": 606 + }, + { + "epoch": 1.6738004832585434, + "grad_norm": 0.19285212457180023, + "learning_rate": 2.443904839260488e-06, + "log_odds_chosen": 1.0728142261505127, + "log_odds_ratio": -0.2983938455581665, + "logits/chosen": -0.48161622881889343, + "logits/rejected": -1.8781613111495972, + "logps/chosen": -1.4637552499771118, + "logps/rejected": -2.369638442993164, + "loss": 1.5399, + "nll_loss": 1.5100862979888916, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14637553691864014, + "rewards/margins": 0.09058833122253418, + "rewards/rejected": -0.23696383833885193, + "step": 607 + }, + { + "epoch": 1.676561960648947, + "grad_norm": 0.213767871260643, + "learning_rate": 2.4358928919547616e-06, + "log_odds_chosen": 1.0425572395324707, + "log_odds_ratio": -0.3122728168964386, + "logits/chosen": -0.621905505657196, + "logits/rejected": -1.3375020027160645, + "logps/chosen": -1.4448013305664062, + "logps/rejected": -2.3270554542541504, + "loss": 1.5444, + "nll_loss": 1.5131899118423462, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1444801241159439, + "rewards/margins": 0.08822542428970337, + "rewards/rejected": -0.23270554840564728, + "step": 608 + }, + { + "epoch": 1.6793234380393511, + "grad_norm": 0.20144887268543243, + "learning_rate": 2.427881603448758e-06, + "log_odds_chosen": 1.1905502080917358, + "log_odds_ratio": -0.27711763978004456, + "logits/chosen": -0.497713565826416, + "logits/rejected": -1.6608542203903198, + "logps/chosen": -1.3489593267440796, + "logps/rejected": -2.3394763469696045, + "loss": 1.4485, + "nll_loss": 1.4207406044006348, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1348959356546402, + "rewards/margins": 0.09905169904232025, + "rewards/rejected": -0.23394761979579926, + "step": 609 + }, + { + "epoch": 1.6820849154297548, + "grad_norm": 0.17932826280593872, + "learning_rate": 2.4198710560708623e-06, + "log_odds_chosen": 1.1837692260742188, + "log_odds_ratio": -0.2758199870586395, + "logits/chosen": -0.5171345472335815, + "logits/rejected": -1.9973483085632324, + "logps/chosen": -1.4259376525878906, + "logps/rejected": -2.424710512161255, + "loss": 1.5165, + "nll_loss": 1.4889655113220215, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14259377121925354, + "rewards/margins": 0.09987729787826538, + "rewards/rejected": -0.24247105419635773, + "step": 610 + }, + { + "epoch": 1.684846392820159, + "grad_norm": 0.2096761018037796, + "learning_rate": 2.411861332141842e-06, + "log_odds_chosen": 1.0929014682769775, + "log_odds_ratio": -0.29848530888557434, + "logits/chosen": -0.5515930652618408, + "logits/rejected": -1.6768569946289062, + "logps/chosen": -1.3467549085617065, + "logps/rejected": -2.250361680984497, + "loss": 1.4289, + "nll_loss": 1.3990368843078613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13467547297477722, + "rewards/margins": 0.09036068618297577, + "rewards/rejected": -0.22503617405891418, + "step": 611 + }, + { + "epoch": 1.6876078702105626, + "grad_norm": 0.23656167089939117, + "learning_rate": 2.403852513974004e-06, + "log_odds_chosen": 1.0888586044311523, + "log_odds_ratio": -0.3083086609840393, + "logits/chosen": -0.41853654384613037, + "logits/rejected": -1.3835935592651367, + "logps/chosen": -1.4978593587875366, + "logps/rejected": -2.4035394191741943, + "loss": 1.5535, + "nll_loss": 1.5226693153381348, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14978592097759247, + "rewards/margins": 0.09056802093982697, + "rewards/rejected": -0.24035394191741943, + "step": 612 + }, + { + "epoch": 1.6903693476009665, + "grad_norm": 0.19909033179283142, + "learning_rate": 2.3958446838703462e-06, + "log_odds_chosen": 1.2349250316619873, + "log_odds_ratio": -0.263581782579422, + "logits/chosen": -0.44553297758102417, + "logits/rejected": -1.8215895891189575, + "logps/chosen": -1.4487204551696777, + "logps/rejected": -2.501533031463623, + "loss": 1.5126, + "nll_loss": 1.4862422943115234, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14487206935882568, + "rewards/margins": 0.10528125613927841, + "rewards/rejected": -0.2501533329486847, + "step": 613 + }, + { + "epoch": 1.6931308249913704, + "grad_norm": 0.20478682219982147, + "learning_rate": 2.3878379241237136e-06, + "log_odds_chosen": 0.9483797550201416, + "log_odds_ratio": -0.33403685688972473, + "logits/chosen": -0.5537324547767639, + "logits/rejected": -1.3671543598175049, + "logps/chosen": -1.475398063659668, + "logps/rejected": -2.2737462520599365, + "loss": 1.5512, + "nll_loss": 1.5178141593933105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14753982424736023, + "rewards/margins": 0.07983479648828506, + "rewards/rejected": -0.2273746132850647, + "step": 614 + }, + { + "epoch": 1.6958923023817742, + "grad_norm": 0.19413712620735168, + "learning_rate": 2.3798323170159487e-06, + "log_odds_chosen": 0.9828821420669556, + "log_odds_ratio": -0.32647275924682617, + "logits/chosen": -0.5587603449821472, + "logits/rejected": -1.868147373199463, + "logps/chosen": -1.4277853965759277, + "logps/rejected": -2.252943992614746, + "loss": 1.5115, + "nll_loss": 1.4788599014282227, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14277854561805725, + "rewards/margins": 0.08251585066318512, + "rewards/rejected": -0.22529438138008118, + "step": 615 + }, + { + "epoch": 1.6986537797721781, + "grad_norm": 0.2020513415336609, + "learning_rate": 2.3718279448170527e-06, + "log_odds_chosen": 1.0720229148864746, + "log_odds_ratio": -0.3052593767642975, + "logits/chosen": -0.5639959573745728, + "logits/rejected": -1.528878927230835, + "logps/chosen": -1.4678881168365479, + "logps/rejected": -2.3781814575195312, + "loss": 1.5566, + "nll_loss": 1.526060938835144, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1467888206243515, + "rewards/margins": 0.0910293310880661, + "rewards/rejected": -0.2378181368112564, + "step": 616 + }, + { + "epoch": 1.701415257162582, + "grad_norm": 0.18654049932956696, + "learning_rate": 2.363824889784333e-06, + "log_odds_chosen": 1.1250965595245361, + "log_odds_ratio": -0.2894567847251892, + "logits/chosen": -0.4636651277542114, + "logits/rejected": -1.680492639541626, + "logps/chosen": -1.379407286643982, + "logps/rejected": -2.32011342048645, + "loss": 1.4648, + "nll_loss": 1.4358913898468018, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13794073462486267, + "rewards/margins": 0.09407063573598862, + "rewards/rejected": -0.2320113629102707, + "step": 617 + }, + { + "epoch": 1.704176734552986, + "grad_norm": 0.215307354927063, + "learning_rate": 2.3558232341615645e-06, + "log_odds_chosen": 0.9324149489402771, + "log_odds_ratio": -0.3408413231372833, + "logits/chosen": -0.4387376010417938, + "logits/rejected": -1.6170055866241455, + "logps/chosen": -1.4724270105361938, + "logps/rejected": -2.2553932666778564, + "loss": 1.569, + "nll_loss": 1.5349595546722412, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1472426950931549, + "rewards/margins": 0.07829662412405014, + "rewards/rejected": -0.22553932666778564, + "step": 618 + }, + { + "epoch": 1.7069382119433896, + "grad_norm": 0.20065483450889587, + "learning_rate": 2.3478230601781374e-06, + "log_odds_chosen": 1.0297596454620361, + "log_odds_ratio": -0.3112369775772095, + "logits/chosen": -0.6553104519844055, + "logits/rejected": -1.7943086624145508, + "logps/chosen": -1.4568157196044922, + "logps/rejected": -2.324397087097168, + "loss": 1.5444, + "nll_loss": 1.5132304430007935, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14568158984184265, + "rewards/margins": 0.08675812184810638, + "rewards/rejected": -0.23243969678878784, + "step": 619 + }, + { + "epoch": 1.7096996893337937, + "grad_norm": 0.19946393370628357, + "learning_rate": 2.339824450048218e-06, + "log_odds_chosen": 1.1597028970718384, + "log_odds_ratio": -0.27503636479377747, + "logits/chosen": -0.4450312554836273, + "logits/rejected": -1.8078252077102661, + "logps/chosen": -1.4679698944091797, + "logps/rejected": -2.453463077545166, + "loss": 1.5524, + "nll_loss": 1.5249354839324951, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14679698646068573, + "rewards/margins": 0.09854928404092789, + "rewards/rejected": -0.24534626305103302, + "step": 620 + }, + { + "epoch": 1.7124611667241973, + "grad_norm": 0.20729611814022064, + "learning_rate": 2.331827485969901e-06, + "log_odds_chosen": 1.084412932395935, + "log_odds_ratio": -0.3078622817993164, + "logits/chosen": -0.5357551574707031, + "logits/rejected": -1.7995785474777222, + "logps/chosen": -1.438812494277954, + "logps/rejected": -2.353656768798828, + "loss": 1.5192, + "nll_loss": 1.4883934259414673, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14388124644756317, + "rewards/margins": 0.0914844498038292, + "rewards/rejected": -0.23536570370197296, + "step": 621 + }, + { + "epoch": 1.7152226441146015, + "grad_norm": 0.21530677378177643, + "learning_rate": 2.323832250124365e-06, + "log_odds_chosen": 1.1834465265274048, + "log_odds_ratio": -0.2767082154750824, + "logits/chosen": -0.5363863110542297, + "logits/rejected": -1.811279535293579, + "logps/chosen": -1.4377635717391968, + "logps/rejected": -2.4433891773223877, + "loss": 1.5118, + "nll_loss": 1.4841231107711792, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14377637207508087, + "rewards/margins": 0.10056254267692566, + "rewards/rejected": -0.24433891475200653, + "step": 622 + }, + { + "epoch": 1.7179841215050051, + "grad_norm": 0.2050798088312149, + "learning_rate": 2.3158388246750308e-06, + "log_odds_chosen": 0.8693351745605469, + "log_odds_ratio": -0.3585469126701355, + "logits/chosen": -0.6070123314857483, + "logits/rejected": -1.4305416345596313, + "logps/chosen": -1.4386093616485596, + "logps/rejected": -2.1597909927368164, + "loss": 1.5264, + "nll_loss": 1.4905424118041992, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14386095106601715, + "rewards/margins": 0.07211817055940628, + "rewards/rejected": -0.21597909927368164, + "step": 623 + }, + { + "epoch": 1.720745598895409, + "grad_norm": 0.19685739278793335, + "learning_rate": 2.307847291766709e-06, + "log_odds_chosen": 1.0625098943710327, + "log_odds_ratio": -0.3041455149650574, + "logits/chosen": -0.5462653636932373, + "logits/rejected": -1.6785917282104492, + "logps/chosen": -1.4387760162353516, + "logps/rejected": -2.3338117599487305, + "loss": 1.5194, + "nll_loss": 1.4889512062072754, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14387759566307068, + "rewards/margins": 0.08950359374284744, + "rewards/rejected": -0.23338119685649872, + "step": 624 + }, + { + "epoch": 1.723507076285813, + "grad_norm": 0.19702772796154022, + "learning_rate": 2.299857733524767e-06, + "log_odds_chosen": 0.9161310195922852, + "log_odds_ratio": -0.3448503613471985, + "logits/chosen": -0.5154906511306763, + "logits/rejected": -1.6805267333984375, + "logps/chosen": -1.4076039791107178, + "logps/rejected": -2.1692450046539307, + "loss": 1.4916, + "nll_loss": 1.4570910930633545, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1407603770494461, + "rewards/margins": 0.076164111495018, + "rewards/rejected": -0.2169245034456253, + "step": 625 + }, + { + "epoch": 1.7262685536762168, + "grad_norm": 0.19595174491405487, + "learning_rate": 2.291870232054277e-06, + "log_odds_chosen": 0.9098183512687683, + "log_odds_ratio": -0.34384119510650635, + "logits/chosen": -0.4463910162448883, + "logits/rejected": -2.0581111907958984, + "logps/chosen": -1.4506475925445557, + "logps/rejected": -2.2079553604125977, + "loss": 1.5414, + "nll_loss": 1.5070030689239502, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14506477117538452, + "rewards/margins": 0.07573077827692032, + "rewards/rejected": -0.22079554200172424, + "step": 626 + }, + { + "epoch": 1.7290300310666207, + "grad_norm": 0.21787682175636292, + "learning_rate": 2.283884869439175e-06, + "log_odds_chosen": 1.1688117980957031, + "log_odds_ratio": -0.2737486958503723, + "logits/chosen": -0.5133581161499023, + "logits/rejected": -1.7837564945220947, + "logps/chosen": -1.5140570402145386, + "logps/rejected": -2.5165603160858154, + "loss": 1.5811, + "nll_loss": 1.5537470579147339, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1514057070016861, + "rewards/margins": 0.10025034844875336, + "rewards/rejected": -0.25165605545043945, + "step": 627 + }, + { + "epoch": 1.7317915084570243, + "grad_norm": 0.19975106418132782, + "learning_rate": 2.2759017277414165e-06, + "log_odds_chosen": 1.3159109354019165, + "log_odds_ratio": -0.24291640520095825, + "logits/chosen": -0.5234307646751404, + "logits/rejected": -1.8133102655410767, + "logps/chosen": -1.3741402626037598, + "logps/rejected": -2.484227418899536, + "loss": 1.4488, + "nll_loss": 1.424553632736206, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13741403818130493, + "rewards/margins": 0.11100868880748749, + "rewards/rejected": -0.24842271208763123, + "step": 628 + }, + { + "epoch": 1.7345529858474285, + "grad_norm": 0.20025824010372162, + "learning_rate": 2.267920889000135e-06, + "log_odds_chosen": 0.9150658845901489, + "log_odds_ratio": -0.3410889804363251, + "logits/chosen": -0.4876527786254883, + "logits/rejected": -1.7472199201583862, + "logps/chosen": -1.487480640411377, + "logps/rejected": -2.258146047592163, + "loss": 1.5642, + "nll_loss": 1.530122995376587, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14874805510044098, + "rewards/margins": 0.07706654071807861, + "rewards/rejected": -0.2258146107196808, + "step": 629 + }, + { + "epoch": 1.7373144632378321, + "grad_norm": 0.21012412011623383, + "learning_rate": 2.2599424352307958e-06, + "log_odds_chosen": 1.1698055267333984, + "log_odds_ratio": -0.2723722457885742, + "logits/chosen": -0.5588301420211792, + "logits/rejected": -1.7450660467147827, + "logps/chosen": -1.2947266101837158, + "logps/rejected": -2.2551028728485107, + "loss": 1.3905, + "nll_loss": 1.3633110523223877, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12947267293930054, + "rewards/margins": 0.09603763371706009, + "rewards/rejected": -0.22551029920578003, + "step": 630 + }, + { + "epoch": 1.7400759406282362, + "grad_norm": 0.20005680620670319, + "learning_rate": 2.2519664484243564e-06, + "log_odds_chosen": 1.027710199356079, + "log_odds_ratio": -0.3135443925857544, + "logits/chosen": -0.5663747191429138, + "logits/rejected": -1.6136528253555298, + "logps/chosen": -1.4354394674301147, + "logps/rejected": -2.2970330715179443, + "loss": 1.5057, + "nll_loss": 1.4743560552597046, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14354394376277924, + "rewards/margins": 0.0861593633890152, + "rewards/rejected": -0.22970330715179443, + "step": 631 + }, + { + "epoch": 1.74283741801864, + "grad_norm": 0.1957002431154251, + "learning_rate": 2.243993010546422e-06, + "log_odds_chosen": 1.0878336429595947, + "log_odds_ratio": -0.30391043424606323, + "logits/chosen": -0.5153028964996338, + "logits/rejected": -1.4119391441345215, + "logps/chosen": -1.4629807472229004, + "logps/rejected": -2.3840439319610596, + "loss": 1.5377, + "nll_loss": 1.507292628288269, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14629808068275452, + "rewards/margins": 0.09210632741451263, + "rewards/rejected": -0.23840439319610596, + "step": 632 + }, + { + "epoch": 1.745598895409044, + "grad_norm": 0.2143191397190094, + "learning_rate": 2.2360222035364027e-06, + "log_odds_chosen": 0.9642858505249023, + "log_odds_ratio": -0.32829856872558594, + "logits/chosen": -0.4845995604991913, + "logits/rejected": -1.453553318977356, + "logps/chosen": -1.440818190574646, + "logps/rejected": -2.2478103637695312, + "loss": 1.5264, + "nll_loss": 1.4935935735702515, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14408183097839355, + "rewards/margins": 0.08069923520088196, + "rewards/rejected": -0.22478105127811432, + "step": 633 + }, + { + "epoch": 1.7483603727994477, + "grad_norm": 0.19890277087688446, + "learning_rate": 2.228054109306673e-06, + "log_odds_chosen": 1.1955161094665527, + "log_odds_ratio": -0.27173271775245667, + "logits/chosen": -0.5415982604026794, + "logits/rejected": -2.0548179149627686, + "logps/chosen": -1.4267183542251587, + "logps/rejected": -2.4402973651885986, + "loss": 1.5018, + "nll_loss": 1.4746149778366089, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1426718533039093, + "rewards/margins": 0.10135789215564728, + "rewards/rejected": -0.24402973055839539, + "step": 634 + }, + { + "epoch": 1.7511218501898516, + "grad_norm": 0.21162277460098267, + "learning_rate": 2.2200888097417308e-06, + "log_odds_chosen": 1.1923596858978271, + "log_odds_ratio": -0.2712979316711426, + "logits/chosen": -0.5525784492492676, + "logits/rejected": -1.8263614177703857, + "logps/chosen": -1.4566607475280762, + "logps/rejected": -2.4698026180267334, + "loss": 1.5463, + "nll_loss": 1.5191999673843384, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14566607773303986, + "rewards/margins": 0.10131418704986572, + "rewards/rejected": -0.24698026478290558, + "step": 635 + }, + { + "epoch": 1.7538833275802554, + "grad_norm": 0.21881026029586792, + "learning_rate": 2.212126386697352e-06, + "log_odds_chosen": 0.9788854122161865, + "log_odds_ratio": -0.3270198404788971, + "logits/chosen": -0.5637598633766174, + "logits/rejected": -1.4680867195129395, + "logps/chosen": -1.486203670501709, + "logps/rejected": -2.3113152980804443, + "loss": 1.5654, + "nll_loss": 1.532711386680603, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1486203521490097, + "rewards/margins": 0.08251115679740906, + "rewards/rejected": -0.23113152384757996, + "step": 636 + }, + { + "epoch": 1.7566448049706593, + "grad_norm": 0.1946457475423813, + "learning_rate": 2.204166921999753e-06, + "log_odds_chosen": 1.10805082321167, + "log_odds_ratio": -0.28972306847572327, + "logits/chosen": -0.5027114748954773, + "logits/rejected": -1.73331618309021, + "logps/chosen": -1.4216092824935913, + "logps/rejected": -2.3447673320770264, + "loss": 1.492, + "nll_loss": 1.4630554914474487, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14216093719005585, + "rewards/margins": 0.09231581538915634, + "rewards/rejected": -0.2344767451286316, + "step": 637 + }, + { + "epoch": 1.7594062823610632, + "grad_norm": 0.2095964401960373, + "learning_rate": 2.196210497444749e-06, + "log_odds_chosen": 1.129142165184021, + "log_odds_ratio": -0.2840039134025574, + "logits/chosen": -0.6306648850440979, + "logits/rejected": -1.8879787921905518, + "logps/chosen": -1.4602231979370117, + "logps/rejected": -2.418236255645752, + "loss": 1.5277, + "nll_loss": 1.4992913007736206, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14602231979370117, + "rewards/margins": 0.09580130875110626, + "rewards/rejected": -0.24182362854480743, + "step": 638 + }, + { + "epoch": 1.7621677597514669, + "grad_norm": 0.2068668156862259, + "learning_rate": 2.1882571947969134e-06, + "log_odds_chosen": 0.9739767909049988, + "log_odds_ratio": -0.3297915458679199, + "logits/chosen": -0.5247660875320435, + "logits/rejected": -1.5939874649047852, + "logps/chosen": -1.424277901649475, + "logps/rejected": -2.237306833267212, + "loss": 1.5071, + "nll_loss": 1.4741605520248413, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14242780208587646, + "rewards/margins": 0.08130290359258652, + "rewards/rejected": -0.22373069822788239, + "step": 639 + }, + { + "epoch": 1.764929237141871, + "grad_norm": 0.2063615918159485, + "learning_rate": 2.1803070957887348e-06, + "log_odds_chosen": 0.9807276725769043, + "log_odds_ratio": -0.32462048530578613, + "logits/chosen": -0.5069320201873779, + "logits/rejected": -1.6134123802185059, + "logps/chosen": -1.391251802444458, + "logps/rejected": -2.201903820037842, + "loss": 1.4684, + "nll_loss": 1.4358959197998047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13912518322467804, + "rewards/margins": 0.08106519281864166, + "rewards/rejected": -0.2201903760433197, + "step": 640 + }, + { + "epoch": 1.7676907145322747, + "grad_norm": 0.20232875645160675, + "learning_rate": 2.1723602821197835e-06, + "log_odds_chosen": 1.0409268140792847, + "log_odds_ratio": -0.310039222240448, + "logits/chosen": -0.5773876905441284, + "logits/rejected": -1.7172613143920898, + "logps/chosen": -1.43693208694458, + "logps/rejected": -2.3079919815063477, + "loss": 1.5217, + "nll_loss": 1.4906917810440063, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.143693208694458, + "rewards/margins": 0.08710600435733795, + "rewards/rejected": -0.23079921305179596, + "step": 641 + }, + { + "epoch": 1.7704521919226788, + "grad_norm": 0.22946524620056152, + "learning_rate": 2.1644168354558623e-06, + "log_odds_chosen": 1.0303146839141846, + "log_odds_ratio": -0.31098419427871704, + "logits/chosen": -0.5952135324478149, + "logits/rejected": -1.6176903247833252, + "logps/chosen": -1.502160906791687, + "logps/rejected": -2.3773584365844727, + "loss": 1.5832, + "nll_loss": 1.5521173477172852, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15021610260009766, + "rewards/margins": 0.08751977235078812, + "rewards/rejected": -0.2377358376979828, + "step": 642 + }, + { + "epoch": 1.7732136693130824, + "grad_norm": 0.21178148686885834, + "learning_rate": 2.1564768374281774e-06, + "log_odds_chosen": 1.2231894731521606, + "log_odds_ratio": -0.2707389295101166, + "logits/chosen": -0.6701139807701111, + "logits/rejected": -2.0402708053588867, + "logps/chosen": -1.4372367858886719, + "logps/rejected": -2.47658371925354, + "loss": 1.5106, + "nll_loss": 1.4835582971572876, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14372369647026062, + "rewards/margins": 0.10393467545509338, + "rewards/rejected": -0.2476583570241928, + "step": 643 + }, + { + "epoch": 1.7759751467034863, + "grad_norm": 0.20058633387088776, + "learning_rate": 2.1485403696324935e-06, + "log_odds_chosen": 0.9557737708091736, + "log_odds_ratio": -0.3273327946662903, + "logits/chosen": -0.5809505581855774, + "logits/rejected": -1.6531572341918945, + "logps/chosen": -1.4407601356506348, + "logps/rejected": -2.23899245262146, + "loss": 1.5311, + "nll_loss": 1.4983248710632324, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14407603442668915, + "rewards/margins": 0.07982321083545685, + "rewards/rejected": -0.2238992303609848, + "step": 644 + }, + { + "epoch": 1.7787366240938902, + "grad_norm": 0.22395431995391846, + "learning_rate": 2.140607513628296e-06, + "log_odds_chosen": 1.0929580926895142, + "log_odds_ratio": -0.3005385994911194, + "logits/chosen": -0.5677430033683777, + "logits/rejected": -1.6676310300827026, + "logps/chosen": -1.491722583770752, + "logps/rejected": -2.4220542907714844, + "loss": 1.5717, + "nll_loss": 1.541687250137329, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14917226135730743, + "rewards/margins": 0.09303317964076996, + "rewards/rejected": -0.2422054558992386, + "step": 645 + }, + { + "epoch": 1.781498101484294, + "grad_norm": 0.19423379004001617, + "learning_rate": 2.1326783509379555e-06, + "log_odds_chosen": 1.230663537979126, + "log_odds_ratio": -0.27645426988601685, + "logits/chosen": -0.6330420970916748, + "logits/rejected": -1.7341974973678589, + "logps/chosen": -1.33867347240448, + "logps/rejected": -2.3714494705200195, + "loss": 1.4287, + "nll_loss": 1.401092290878296, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13386735320091248, + "rewards/margins": 0.10327759385108948, + "rewards/rejected": -0.23714493215084076, + "step": 646 + }, + { + "epoch": 1.784259578874698, + "grad_norm": 0.21576644480228424, + "learning_rate": 2.1247529630458856e-06, + "log_odds_chosen": 0.9605802297592163, + "log_odds_ratio": -0.33712220191955566, + "logits/chosen": -0.5994086265563965, + "logits/rejected": -1.5157111883163452, + "logps/chosen": -1.4738686084747314, + "logps/rejected": -2.2864460945129395, + "loss": 1.5531, + "nll_loss": 1.5193928480148315, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14738686382770538, + "rewards/margins": 0.08125776797533035, + "rewards/rejected": -0.22864462435245514, + "step": 647 + }, + { + "epoch": 1.7870210562651019, + "grad_norm": 0.2162182778120041, + "learning_rate": 2.11683143139771e-06, + "log_odds_chosen": 1.0968060493469238, + "log_odds_ratio": -0.3004920184612274, + "logits/chosen": -0.5496225357055664, + "logits/rejected": -1.5592927932739258, + "logps/chosen": -1.4152264595031738, + "logps/rejected": -2.3394081592559814, + "loss": 1.5072, + "nll_loss": 1.4771496057510376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14152266085147858, + "rewards/margins": 0.09241816401481628, + "rewards/rejected": -0.23394080996513367, + "step": 648 + }, + { + "epoch": 1.7897825336555058, + "grad_norm": 0.20863664150238037, + "learning_rate": 2.1089138373994226e-06, + "log_odds_chosen": 1.085922122001648, + "log_odds_ratio": -0.2969967722892761, + "logits/chosen": -0.589851975440979, + "logits/rejected": -1.8017020225524902, + "logps/chosen": -1.4874114990234375, + "logps/rejected": -2.4139440059661865, + "loss": 1.5819, + "nll_loss": 1.552212119102478, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14874114096164703, + "rewards/margins": 0.09265326708555222, + "rewards/rejected": -0.24139441549777985, + "step": 649 + }, + { + "epoch": 1.7925440110459094, + "grad_norm": 0.2144482284784317, + "learning_rate": 2.1010002624165528e-06, + "log_odds_chosen": 0.886896550655365, + "log_odds_ratio": -0.3648533225059509, + "logits/chosen": -0.5143107175827026, + "logits/rejected": -1.4646867513656616, + "logps/chosen": -1.5178825855255127, + "logps/rejected": -2.2712976932525635, + "loss": 1.5934, + "nll_loss": 1.5568923950195312, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15178826451301575, + "rewards/margins": 0.07534152269363403, + "rewards/rejected": -0.2271297723054886, + "step": 650 + }, + { + "epoch": 1.7953054884363135, + "grad_norm": 0.19179891049861908, + "learning_rate": 2.0930907877733255e-06, + "log_odds_chosen": 1.1237578392028809, + "log_odds_ratio": -0.2867814004421234, + "logits/chosen": -0.43964463472366333, + "logits/rejected": -1.8275145292282104, + "logps/chosen": -1.4230607748031616, + "logps/rejected": -2.369021415710449, + "loss": 1.4899, + "nll_loss": 1.4611868858337402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14230608940124512, + "rewards/margins": 0.09459604322910309, + "rewards/rejected": -0.2369021326303482, + "step": 651 + }, + { + "epoch": 1.7980669658267172, + "grad_norm": 0.19488249719142914, + "learning_rate": 2.085185494751831e-06, + "log_odds_chosen": 1.1252477169036865, + "log_odds_ratio": -0.2942362427711487, + "logits/chosen": -0.526394248008728, + "logits/rejected": -1.643446683883667, + "logps/chosen": -1.3340688943862915, + "logps/rejected": -2.267882823944092, + "loss": 1.4353, + "nll_loss": 1.4058688879013062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1334068924188614, + "rewards/margins": 0.09338139742612839, + "rewards/rejected": -0.22678828239440918, + "step": 652 + }, + { + "epoch": 1.8008284432171213, + "grad_norm": 0.20605449378490448, + "learning_rate": 2.077284464591187e-06, + "log_odds_chosen": 1.0873126983642578, + "log_odds_ratio": -0.3010372817516327, + "logits/chosen": -0.47243842482566833, + "logits/rejected": -1.7291725873947144, + "logps/chosen": -1.4694960117340088, + "logps/rejected": -2.3931758403778076, + "loss": 1.5427, + "nll_loss": 1.5125980377197266, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14694960415363312, + "rewards/margins": 0.092367984354496, + "rewards/rejected": -0.23931758105754852, + "step": 653 + }, + { + "epoch": 1.803589920607525, + "grad_norm": 0.201577290892601, + "learning_rate": 2.069387778486703e-06, + "log_odds_chosen": 1.1571605205535889, + "log_odds_ratio": -0.28402575850486755, + "logits/chosen": -0.5961681008338928, + "logits/rejected": -1.9189343452453613, + "logps/chosen": -1.5506150722503662, + "logps/rejected": -2.54967999458313, + "loss": 1.6219, + "nll_loss": 1.5935449600219727, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1550614982843399, + "rewards/margins": 0.09990648925304413, + "rewards/rejected": -0.25496798753738403, + "step": 654 + }, + { + "epoch": 1.8063513979979289, + "grad_norm": 0.21721962094306946, + "learning_rate": 2.0614955175890464e-06, + "log_odds_chosen": 1.1083087921142578, + "log_odds_ratio": -0.2993934750556946, + "logits/chosen": -0.6008000373840332, + "logits/rejected": -1.8069735765457153, + "logps/chosen": -1.4827308654785156, + "logps/rejected": -2.4276866912841797, + "loss": 1.5456, + "nll_loss": 1.515679955482483, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14827308058738708, + "rewards/margins": 0.09449561685323715, + "rewards/rejected": -0.24276868999004364, + "step": 655 + }, + { + "epoch": 1.8091128753883328, + "grad_norm": 0.19219523668289185, + "learning_rate": 2.053607763003409e-06, + "log_odds_chosen": 1.2417378425598145, + "log_odds_ratio": -0.2833240032196045, + "logits/chosen": -0.5437658429145813, + "logits/rejected": -1.7058812379837036, + "logps/chosen": -1.4310146570205688, + "logps/rejected": -2.4930710792541504, + "loss": 1.5176, + "nll_loss": 1.4892582893371582, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14310148358345032, + "rewards/margins": 0.10620564222335815, + "rewards/rejected": -0.24930711090564728, + "step": 656 + }, + { + "epoch": 1.8118743527787367, + "grad_norm": 0.1976533681154251, + "learning_rate": 2.045724595788673e-06, + "log_odds_chosen": 1.2406878471374512, + "log_odds_ratio": -0.2589613199234009, + "logits/chosen": -0.5714414119720459, + "logits/rejected": -1.9800139665603638, + "logps/chosen": -1.4148036241531372, + "logps/rejected": -2.4669976234436035, + "loss": 1.4927, + "nll_loss": 1.4667580127716064, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14148037135601044, + "rewards/margins": 0.10521939396858215, + "rewards/rejected": -0.2466997504234314, + "step": 657 + }, + { + "epoch": 1.8146358301691405, + "grad_norm": 0.2066175788640976, + "learning_rate": 2.037846096956578e-06, + "log_odds_chosen": 1.1821521520614624, + "log_odds_ratio": -0.27946531772613525, + "logits/chosen": -0.5792191028594971, + "logits/rejected": -1.5947271585464478, + "logps/chosen": -1.4168436527252197, + "logps/rejected": -2.4209742546081543, + "loss": 1.5035, + "nll_loss": 1.4755176305770874, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1416843682527542, + "rewards/margins": 0.10041306912899017, + "rewards/rejected": -0.24209743738174438, + "step": 658 + }, + { + "epoch": 1.8173973075595442, + "grad_norm": 0.19449689984321594, + "learning_rate": 2.0299723474708917e-06, + "log_odds_chosen": 1.0754530429840088, + "log_odds_ratio": -0.30575478076934814, + "logits/chosen": -0.522871732711792, + "logits/rejected": -1.6677809953689575, + "logps/chosen": -1.405239224433899, + "logps/rejected": -2.306657075881958, + "loss": 1.4803, + "nll_loss": 1.4497466087341309, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14052392542362213, + "rewards/margins": 0.09014178067445755, + "rewards/rejected": -0.23066571354866028, + "step": 659 + }, + { + "epoch": 1.8201587849499483, + "grad_norm": 0.20105119049549103, + "learning_rate": 2.02210342824657e-06, + "log_odds_chosen": 1.2487144470214844, + "log_odds_ratio": -0.26469725370407104, + "logits/chosen": -0.6605066061019897, + "logits/rejected": -1.9393763542175293, + "logps/chosen": -1.4192674160003662, + "logps/rejected": -2.4731574058532715, + "loss": 1.5006, + "nll_loss": 1.4741599559783936, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14192675054073334, + "rewards/margins": 0.10538898408412933, + "rewards/rejected": -0.24731573462486267, + "step": 660 + }, + { + "epoch": 1.822920262340352, + "grad_norm": 0.20803654193878174, + "learning_rate": 2.0142394201489334e-06, + "log_odds_chosen": 1.2532217502593994, + "log_odds_ratio": -0.2540588676929474, + "logits/chosen": -0.5191413164138794, + "logits/rejected": -1.8599536418914795, + "logps/chosen": -1.4484375715255737, + "logps/rejected": -2.5184717178344727, + "loss": 1.513, + "nll_loss": 1.4876114130020142, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14484377205371857, + "rewards/margins": 0.10700343549251556, + "rewards/rejected": -0.25184717774391174, + "step": 661 + }, + { + "epoch": 1.825681739730756, + "grad_norm": 0.2015330046415329, + "learning_rate": 2.0063804039928324e-06, + "log_odds_chosen": 1.2455298900604248, + "log_odds_ratio": -0.26516517996788025, + "logits/chosen": -0.5362703204154968, + "logits/rejected": -1.6895490884780884, + "logps/chosen": -1.4490387439727783, + "logps/rejected": -2.5178382396698, + "loss": 1.5259, + "nll_loss": 1.49942946434021, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14490386843681335, + "rewards/margins": 0.10687996447086334, + "rewards/rejected": -0.2517838180065155, + "step": 662 + }, + { + "epoch": 1.8284432171211598, + "grad_norm": 0.21101270616054535, + "learning_rate": 1.9985264605418185e-06, + "log_odds_chosen": 1.265969157218933, + "log_odds_ratio": -0.26632457971572876, + "logits/chosen": -0.624642550945282, + "logits/rejected": -1.8359934091567993, + "logps/chosen": -1.4265787601470947, + "logps/rejected": -2.5020813941955566, + "loss": 1.5126, + "nll_loss": 1.4859540462493896, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14265787601470947, + "rewards/margins": 0.10755026340484619, + "rewards/rejected": -0.25020813941955566, + "step": 663 + }, + { + "epoch": 1.8312046945115636, + "grad_norm": 0.21531936526298523, + "learning_rate": 1.9906776705073113e-06, + "log_odds_chosen": 1.3287895917892456, + "log_odds_ratio": -0.24583633244037628, + "logits/chosen": -0.6203726530075073, + "logits/rejected": -1.9401627779006958, + "logps/chosen": -1.4545714855194092, + "logps/rejected": -2.5981836318969727, + "loss": 1.5272, + "nll_loss": 1.5025845766067505, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14545713365077972, + "rewards/margins": 0.1143612340092659, + "rewards/rejected": -0.2598183751106262, + "step": 664 + }, + { + "epoch": 1.8339661719019675, + "grad_norm": 0.20089717209339142, + "learning_rate": 1.982834114547773e-06, + "log_odds_chosen": 1.1791274547576904, + "log_odds_ratio": -0.27245157957077026, + "logits/chosen": -0.6062506437301636, + "logits/rejected": -1.8836218118667603, + "logps/chosen": -1.440775752067566, + "logps/rejected": -2.4392642974853516, + "loss": 1.524, + "nll_loss": 1.4968045949935913, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1440775841474533, + "rewards/margins": 0.09984884411096573, + "rewards/rejected": -0.24392642080783844, + "step": 665 + }, + { + "epoch": 1.8367276492923714, + "grad_norm": 0.21053946018218994, + "learning_rate": 1.9749958732678766e-06, + "log_odds_chosen": 1.1956483125686646, + "log_odds_ratio": -0.2698996663093567, + "logits/chosen": -0.6107760071754456, + "logits/rejected": -1.7991489171981812, + "logps/chosen": -1.4191172122955322, + "logps/rejected": -2.4300425052642822, + "loss": 1.5046, + "nll_loss": 1.4775981903076172, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14191171526908875, + "rewards/margins": 0.10109255462884903, + "rewards/rejected": -0.24300426244735718, + "step": 666 + }, + { + "epoch": 1.8394891266827753, + "grad_norm": 0.23768320679664612, + "learning_rate": 1.967163027217679e-06, + "log_odds_chosen": 1.1234877109527588, + "log_odds_ratio": -0.29781588912010193, + "logits/chosen": -0.6074894666671753, + "logits/rejected": -1.5648796558380127, + "logps/chosen": -1.4478806257247925, + "logps/rejected": -2.397280216217041, + "loss": 1.5416, + "nll_loss": 1.5118470191955566, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14478805661201477, + "rewards/margins": 0.0949399471282959, + "rewards/rejected": -0.23972800374031067, + "step": 667 + }, + { + "epoch": 1.8422506040731792, + "grad_norm": 0.22009487450122833, + "learning_rate": 1.9593356568917916e-06, + "log_odds_chosen": 1.0102051496505737, + "log_odds_ratio": -0.3210904896259308, + "logits/chosen": -0.5804159641265869, + "logits/rejected": -1.5628560781478882, + "logps/chosen": -1.5433801412582397, + "logps/rejected": -2.4113738536834717, + "loss": 1.6194, + "nll_loss": 1.5873008966445923, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1543380171060562, + "rewards/margins": 0.08679936826229095, + "rewards/rejected": -0.24113738536834717, + "step": 668 + }, + { + "epoch": 1.845012081463583, + "grad_norm": 0.20616894960403442, + "learning_rate": 1.951513842728556e-06, + "log_odds_chosen": 1.106505036354065, + "log_odds_ratio": -0.2891699969768524, + "logits/chosen": -0.5833257436752319, + "logits/rejected": -1.9531350135803223, + "logps/chosen": -1.4294037818908691, + "logps/rejected": -2.3592071533203125, + "loss": 1.5097, + "nll_loss": 1.4808082580566406, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14294040203094482, + "rewards/margins": 0.09298031032085419, + "rewards/rejected": -0.235920712351799, + "step": 669 + }, + { + "epoch": 1.8477735588539868, + "grad_norm": 0.21930097043514252, + "learning_rate": 1.9436976651092143e-06, + "log_odds_chosen": 1.1857033967971802, + "log_odds_ratio": -0.27113771438598633, + "logits/chosen": -0.6338096261024475, + "logits/rejected": -1.7122553586959839, + "logps/chosen": -1.424892544746399, + "logps/rejected": -2.4289939403533936, + "loss": 1.5164, + "nll_loss": 1.4892981052398682, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1424892544746399, + "rewards/margins": 0.10041015595197678, + "rewards/rejected": -0.24289940297603607, + "step": 670 + }, + { + "epoch": 1.8505350362443909, + "grad_norm": 0.20231999456882477, + "learning_rate": 1.935887204357085e-06, + "log_odds_chosen": 1.3218573331832886, + "log_odds_ratio": -0.25989198684692383, + "logits/chosen": -0.5880028009414673, + "logits/rejected": -1.7775429487228394, + "logps/chosen": -1.4524977207183838, + "logps/rejected": -2.591127395629883, + "loss": 1.5228, + "nll_loss": 1.4968382120132446, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14524978399276733, + "rewards/margins": 0.11386297643184662, + "rewards/rejected": -0.25911277532577515, + "step": 671 + }, + { + "epoch": 1.8532965136347945, + "grad_norm": 0.2004518359899521, + "learning_rate": 1.928082540736737e-06, + "log_odds_chosen": 1.3334486484527588, + "log_odds_ratio": -0.2543681859970093, + "logits/chosen": -0.5984243750572205, + "logits/rejected": -1.6906664371490479, + "logps/chosen": -1.4101768732070923, + "logps/rejected": -2.5547547340393066, + "loss": 1.5006, + "nll_loss": 1.4751571416854858, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14101769030094147, + "rewards/margins": 0.11445777118206024, + "rewards/rejected": -0.2554754912853241, + "step": 672 + }, + { + "epoch": 1.8560579910251986, + "grad_norm": 0.21709613502025604, + "learning_rate": 1.920283754453164e-06, + "log_odds_chosen": 1.3128209114074707, + "log_odds_ratio": -0.2452673465013504, + "logits/chosen": -0.5661304593086243, + "logits/rejected": -1.7389955520629883, + "logps/chosen": -1.3728487491607666, + "logps/rejected": -2.478579044342041, + "loss": 1.4466, + "nll_loss": 1.422105312347412, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13728488981723785, + "rewards/margins": 0.11057302355766296, + "rewards/rejected": -0.24785789847373962, + "step": 673 + }, + { + "epoch": 1.8588194684156023, + "grad_norm": 0.2150663137435913, + "learning_rate": 1.912490925650962e-06, + "log_odds_chosen": 1.348259687423706, + "log_odds_ratio": -0.23330648243427277, + "logits/chosen": -0.6710218191146851, + "logits/rejected": -1.9455617666244507, + "logps/chosen": -1.4661921262741089, + "logps/rejected": -2.6251721382141113, + "loss": 1.5275, + "nll_loss": 1.5041943788528442, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14661921560764313, + "rewards/margins": 0.1158980131149292, + "rewards/rejected": -0.26251721382141113, + "step": 674 + }, + { + "epoch": 1.8615809458060062, + "grad_norm": 0.2174362689256668, + "learning_rate": 1.9047041344135045e-06, + "log_odds_chosen": 1.003183126449585, + "log_odds_ratio": -0.3182450830936432, + "logits/chosen": -0.6303192973136902, + "logits/rejected": -1.8845601081848145, + "logps/chosen": -1.5408192873001099, + "logps/rejected": -2.400413990020752, + "loss": 1.6063, + "nll_loss": 1.5744513273239136, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15408192574977875, + "rewards/margins": 0.08595946431159973, + "rewards/rejected": -0.24004140496253967, + "step": 675 + }, + { + "epoch": 1.86434242319641, + "grad_norm": 0.1918780356645584, + "learning_rate": 1.8969234607621187e-06, + "log_odds_chosen": 1.1956127882003784, + "log_odds_ratio": -0.2732754349708557, + "logits/chosen": -0.6226102113723755, + "logits/rejected": -1.8827693462371826, + "logps/chosen": -1.3975963592529297, + "logps/rejected": -2.3993749618530273, + "loss": 1.4784, + "nll_loss": 1.4511014223098755, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1397596299648285, + "rewards/margins": 0.10017785429954529, + "rewards/rejected": -0.23993748426437378, + "step": 676 + }, + { + "epoch": 1.867103900586814, + "grad_norm": 0.19374889135360718, + "learning_rate": 1.8891489846552645e-06, + "log_odds_chosen": 1.1678223609924316, + "log_odds_ratio": -0.2740553915500641, + "logits/chosen": -0.5673727989196777, + "logits/rejected": -1.970065951347351, + "logps/chosen": -1.3401387929916382, + "logps/rejected": -2.310602903366089, + "loss": 1.4337, + "nll_loss": 1.4062803983688354, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13401387631893158, + "rewards/margins": 0.09704641252756119, + "rewards/rejected": -0.23106031119823456, + "step": 677 + }, + { + "epoch": 1.8698653779772179, + "grad_norm": 0.20325230062007904, + "learning_rate": 1.8813807859877147e-06, + "log_odds_chosen": 1.0683099031448364, + "log_odds_ratio": -0.30433735251426697, + "logits/chosen": -0.5034862756729126, + "logits/rejected": -1.5476434230804443, + "logps/chosen": -1.4672025442123413, + "logps/rejected": -2.3724498748779297, + "loss": 1.549, + "nll_loss": 1.5185987949371338, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1467202603816986, + "rewards/margins": 0.09052471071481705, + "rewards/rejected": -0.23724496364593506, + "step": 678 + }, + { + "epoch": 1.8726268553676215, + "grad_norm": 0.19582170248031616, + "learning_rate": 1.8736189445897268e-06, + "log_odds_chosen": 1.1498327255249023, + "log_odds_ratio": -0.28108832240104675, + "logits/chosen": -0.6858939528465271, + "logits/rejected": -1.8512457609176636, + "logps/chosen": -1.3199814558029175, + "logps/rejected": -2.2606868743896484, + "loss": 1.4063, + "nll_loss": 1.3781564235687256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13199815154075623, + "rewards/margins": 0.09407053887844086, + "rewards/rejected": -0.22606870532035828, + "step": 679 + }, + { + "epoch": 1.8753883327580256, + "grad_norm": 0.19537782669067383, + "learning_rate": 1.865863540226232e-06, + "log_odds_chosen": 1.2928959131240845, + "log_odds_ratio": -0.24565257132053375, + "logits/chosen": -0.5815032124519348, + "logits/rejected": -1.7621716260910034, + "logps/chosen": -1.3540948629379272, + "logps/rejected": -2.4354796409606934, + "loss": 1.4379, + "nll_loss": 1.4132860898971558, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13540948927402496, + "rewards/margins": 0.10813847184181213, + "rewards/rejected": -0.2435479760169983, + "step": 680 + }, + { + "epoch": 1.8781498101484293, + "grad_norm": 0.20229066908359528, + "learning_rate": 1.8581146525960093e-06, + "log_odds_chosen": 1.1232048273086548, + "log_odds_ratio": -0.2937353551387787, + "logits/chosen": -0.637298583984375, + "logits/rejected": -1.6448088884353638, + "logps/chosen": -1.3534826040267944, + "logps/rejected": -2.2935564517974854, + "loss": 1.4484, + "nll_loss": 1.4190561771392822, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13534826040267944, + "rewards/margins": 0.09400738775730133, + "rewards/rejected": -0.22935563325881958, + "step": 681 + }, + { + "epoch": 1.8809112875388334, + "grad_norm": 0.2125195562839508, + "learning_rate": 1.8503723613308683e-06, + "log_odds_chosen": 1.274790644645691, + "log_odds_ratio": -0.26285573840141296, + "logits/chosen": -0.5560697317123413, + "logits/rejected": -1.9034526348114014, + "logps/chosen": -1.442688226699829, + "logps/rejected": -2.5351686477661133, + "loss": 1.5013, + "nll_loss": 1.4750303030014038, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14426881074905396, + "rewards/margins": 0.1092480719089508, + "rewards/rejected": -0.25351691246032715, + "step": 682 + }, + { + "epoch": 1.883672764929237, + "grad_norm": 0.20290443301200867, + "learning_rate": 1.8426367459948306e-06, + "log_odds_chosen": 1.2297403812408447, + "log_odds_ratio": -0.2659325897693634, + "logits/chosen": -0.5875911712646484, + "logits/rejected": -1.781616449356079, + "logps/chosen": -1.4287062883377075, + "logps/rejected": -2.475780487060547, + "loss": 1.507, + "nll_loss": 1.4804446697235107, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14287063479423523, + "rewards/margins": 0.10470742732286453, + "rewards/rejected": -0.24757805466651917, + "step": 683 + }, + { + "epoch": 1.886434242319641, + "grad_norm": 0.22338153421878815, + "learning_rate": 1.8349078860833125e-06, + "log_odds_chosen": 1.1323693990707397, + "log_odds_ratio": -0.28367865085601807, + "logits/chosen": -0.6488045454025269, + "logits/rejected": -1.7948299646377563, + "logps/chosen": -1.4490909576416016, + "logps/rejected": -2.4043657779693604, + "loss": 1.5199, + "nll_loss": 1.491485595703125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1449090987443924, + "rewards/margins": 0.09552749991416931, + "rewards/rejected": -0.2404365837574005, + "step": 684 + }, + { + "epoch": 1.8891957197100449, + "grad_norm": 0.21708610653877258, + "learning_rate": 1.827185861022308e-06, + "log_odds_chosen": 1.321728229522705, + "log_odds_ratio": -0.2453889697790146, + "logits/chosen": -0.5850996375083923, + "logits/rejected": -1.7692553997039795, + "logps/chosen": -1.4336585998535156, + "logps/rejected": -2.565013885498047, + "loss": 1.4925, + "nll_loss": 1.467954158782959, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14336584508419037, + "rewards/margins": 0.11313553899526596, + "rewards/rejected": -0.2565014064311981, + "step": 685 + }, + { + "epoch": 1.8919571971004487, + "grad_norm": 0.20613998174667358, + "learning_rate": 1.8194707501675724e-06, + "log_odds_chosen": 1.4058480262756348, + "log_odds_ratio": -0.2284797877073288, + "logits/chosen": -0.688378632068634, + "logits/rejected": -1.8188039064407349, + "logps/chosen": -1.377896785736084, + "logps/rejected": -2.5687503814697266, + "loss": 1.4495, + "nll_loss": 1.4266870021820068, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13778966665267944, + "rewards/margins": 0.119085393846035, + "rewards/rejected": -0.25687506794929504, + "step": 686 + }, + { + "epoch": 1.8947186744908526, + "grad_norm": 0.22078323364257812, + "learning_rate": 1.8117626328038069e-06, + "log_odds_chosen": 1.1295044422149658, + "log_odds_ratio": -0.28533411026000977, + "logits/chosen": -0.5322979688644409, + "logits/rejected": -1.77653968334198, + "logps/chosen": -1.4119873046875, + "logps/rejected": -2.3596487045288086, + "loss": 1.4847, + "nll_loss": 1.4561336040496826, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14119873940944672, + "rewards/margins": 0.09476611763238907, + "rewards/rejected": -0.23596486449241638, + "step": 687 + }, + { + "epoch": 1.8974801518812565, + "grad_norm": 0.20481730997562408, + "learning_rate": 1.8040615881438425e-06, + "log_odds_chosen": 1.02912175655365, + "log_odds_ratio": -0.3101516664028168, + "logits/chosen": -0.6844128370285034, + "logits/rejected": -1.748498797416687, + "logps/chosen": -1.4500445127487183, + "logps/rejected": -2.318002700805664, + "loss": 1.5398, + "nll_loss": 1.5087554454803467, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14500445127487183, + "rewards/margins": 0.08679580688476562, + "rewards/rejected": -0.23180025815963745, + "step": 688 + }, + { + "epoch": 1.9002416292716604, + "grad_norm": 0.21134057641029358, + "learning_rate": 1.7963676953278292e-06, + "log_odds_chosen": 1.2188888788223267, + "log_odds_ratio": -0.26270338892936707, + "logits/chosen": -0.6463155150413513, + "logits/rejected": -1.7863259315490723, + "logps/chosen": -1.4500489234924316, + "logps/rejected": -2.4897103309631348, + "loss": 1.5305, + "nll_loss": 1.5042182207107544, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14500491321086884, + "rewards/margins": 0.1039661318063736, + "rewards/rejected": -0.24897105991840363, + "step": 689 + }, + { + "epoch": 1.903003106662064, + "grad_norm": 0.21317671239376068, + "learning_rate": 1.7886810334224192e-06, + "log_odds_chosen": 1.2021043300628662, + "log_odds_ratio": -0.2723836600780487, + "logits/chosen": -0.6922492980957031, + "logits/rejected": -1.89365816116333, + "logps/chosen": -1.414392352104187, + "logps/rejected": -2.432486057281494, + "loss": 1.4898, + "nll_loss": 1.4626052379608154, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14143922924995422, + "rewards/margins": 0.10180937498807907, + "rewards/rejected": -0.2432485967874527, + "step": 690 + }, + { + "epoch": 1.9057645840524682, + "grad_norm": 0.1928180307149887, + "learning_rate": 1.781001681419957e-06, + "log_odds_chosen": 1.4509155750274658, + "log_odds_ratio": -0.24097022414207458, + "logits/chosen": -0.5879383087158203, + "logits/rejected": -1.8242533206939697, + "logps/chosen": -1.3567006587982178, + "logps/rejected": -2.600719451904297, + "loss": 1.4364, + "nll_loss": 1.4122999906539917, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13567008078098297, + "rewards/margins": 0.12440188229084015, + "rewards/rejected": -0.2600719630718231, + "step": 691 + }, + { + "epoch": 1.9085260614428718, + "grad_norm": 0.2123909443616867, + "learning_rate": 1.7733297182376663e-06, + "log_odds_chosen": 1.194056510925293, + "log_odds_ratio": -0.2803875803947449, + "logits/chosen": -0.7072063088417053, + "logits/rejected": -1.670470118522644, + "logps/chosen": -1.3613109588623047, + "logps/rejected": -2.3637614250183105, + "loss": 1.4595, + "nll_loss": 1.431486964225769, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13613110780715942, + "rewards/margins": 0.10024504363536835, + "rewards/rejected": -0.23637613654136658, + "step": 692 + }, + { + "epoch": 1.911287538833276, + "grad_norm": 0.2346637099981308, + "learning_rate": 1.7656652227168384e-06, + "log_odds_chosen": 1.2246406078338623, + "log_odds_ratio": -0.2636195421218872, + "logits/chosen": -0.6246435046195984, + "logits/rejected": -1.6411041021347046, + "logps/chosen": -1.4360047578811646, + "logps/rejected": -2.4795889854431152, + "loss": 1.5046, + "nll_loss": 1.4782655239105225, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1436004787683487, + "rewards/margins": 0.10435841977596283, + "rewards/rejected": -0.24795889854431152, + "step": 693 + }, + { + "epoch": 1.9140490162236796, + "grad_norm": 0.21563811600208282, + "learning_rate": 1.7580082736220235e-06, + "log_odds_chosen": 1.351541519165039, + "log_odds_ratio": -0.24304015934467316, + "logits/chosen": -0.6352071762084961, + "logits/rejected": -1.7275444269180298, + "logps/chosen": -1.4102025032043457, + "logps/rejected": -2.5561411380767822, + "loss": 1.4874, + "nll_loss": 1.4630484580993652, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1410202533006668, + "rewards/margins": 0.11459386348724365, + "rewards/rejected": -0.25561410188674927, + "step": 694 + }, + { + "epoch": 1.9168104936140835, + "grad_norm": 0.23292036354541779, + "learning_rate": 1.750358949640221e-06, + "log_odds_chosen": 1.0779083967208862, + "log_odds_ratio": -0.3038214147090912, + "logits/chosen": -0.6787149906158447, + "logits/rejected": -1.33951997756958, + "logps/chosen": -1.389949083328247, + "logps/rejected": -2.295311212539673, + "loss": 1.4843, + "nll_loss": 1.453916072845459, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13899490237236023, + "rewards/margins": 0.0905362069606781, + "rewards/rejected": -0.22953110933303833, + "step": 695 + }, + { + "epoch": 1.9195719710044874, + "grad_norm": 0.20314742624759674, + "learning_rate": 1.74271732938007e-06, + "log_odds_chosen": 1.121297836303711, + "log_odds_ratio": -0.2867622375488281, + "logits/chosen": -0.6550911068916321, + "logits/rejected": -1.6976155042648315, + "logps/chosen": -1.3656046390533447, + "logps/rejected": -2.2984426021575928, + "loss": 1.4571, + "nll_loss": 1.428386926651001, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13656045496463776, + "rewards/margins": 0.09328380227088928, + "rewards/rejected": -0.22984427213668823, + "step": 696 + }, + { + "epoch": 1.9223334483948913, + "grad_norm": 0.22091306746006012, + "learning_rate": 1.7350834913710414e-06, + "log_odds_chosen": 1.366137146949768, + "log_odds_ratio": -0.23213300108909607, + "logits/chosen": -0.7029794454574585, + "logits/rejected": -1.7516682147979736, + "logps/chosen": -1.3843116760253906, + "logps/rejected": -2.5412652492523193, + "loss": 1.4653, + "nll_loss": 1.4420487880706787, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13843117654323578, + "rewards/margins": 0.11569535732269287, + "rewards/rejected": -0.25412654876708984, + "step": 697 + }, + { + "epoch": 1.9250949257852952, + "grad_norm": 0.21280437707901, + "learning_rate": 1.7274575140626318e-06, + "log_odds_chosen": 1.4392701387405396, + "log_odds_ratio": -0.21989382803440094, + "logits/chosen": -0.6601370573043823, + "logits/rejected": -1.899329423904419, + "logps/chosen": -1.3539754152297974, + "logps/rejected": -2.572890281677246, + "loss": 1.4257, + "nll_loss": 1.403670072555542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1353975534439087, + "rewards/margins": 0.12189144641160965, + "rewards/rejected": -0.25728899240493774, + "step": 698 + }, + { + "epoch": 1.9278564031756988, + "grad_norm": 0.22868065536022186, + "learning_rate": 1.7198394758235576e-06, + "log_odds_chosen": 1.0325840711593628, + "log_odds_ratio": -0.3077024519443512, + "logits/chosen": -0.7413440346717834, + "logits/rejected": -1.7055625915527344, + "logps/chosen": -1.4271225929260254, + "logps/rejected": -2.2928249835968018, + "loss": 1.5059, + "nll_loss": 1.4751147031784058, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1427122801542282, + "rewards/margins": 0.08657021820545197, + "rewards/rejected": -0.2292824685573578, + "step": 699 + }, + { + "epoch": 1.930617880566103, + "grad_norm": 0.1975845843553543, + "learning_rate": 1.7122294549409486e-06, + "log_odds_chosen": 1.1576435565948486, + "log_odds_ratio": -0.2774282693862915, + "logits/chosen": -0.6693485975265503, + "logits/rejected": -1.845128059387207, + "logps/chosen": -1.4216663837432861, + "logps/rejected": -2.3981292247772217, + "loss": 1.4897, + "nll_loss": 1.4619686603546143, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1421666294336319, + "rewards/margins": 0.0976463034749031, + "rewards/rejected": -0.2398129254579544, + "step": 700 + }, + { + "epoch": 1.9333793579565066, + "grad_norm": 0.2156144380569458, + "learning_rate": 1.704627529619543e-06, + "log_odds_chosen": 1.0374083518981934, + "log_odds_ratio": -0.313325971364975, + "logits/chosen": -0.6931014060974121, + "logits/rejected": -1.77117919921875, + "logps/chosen": -1.4780950546264648, + "logps/rejected": -2.358729839324951, + "loss": 1.5628, + "nll_loss": 1.531511902809143, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14780952036380768, + "rewards/margins": 0.08806345611810684, + "rewards/rejected": -0.23587298393249512, + "step": 701 + }, + { + "epoch": 1.9361408353469107, + "grad_norm": 0.20054619014263153, + "learning_rate": 1.6970337779808862e-06, + "log_odds_chosen": 1.087822437286377, + "log_odds_ratio": -0.2988903224468231, + "logits/chosen": -0.6482985615730286, + "logits/rejected": -1.7110782861709595, + "logps/chosen": -1.4709477424621582, + "logps/rejected": -2.3889198303222656, + "loss": 1.5488, + "nll_loss": 1.5189554691314697, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14709478616714478, + "rewards/margins": 0.09179720282554626, + "rewards/rejected": -0.23889197409152985, + "step": 702 + }, + { + "epoch": 1.9389023127373144, + "grad_norm": 0.2022159993648529, + "learning_rate": 1.689448278062525e-06, + "log_odds_chosen": 1.2595505714416504, + "log_odds_ratio": -0.25268253684043884, + "logits/chosen": -0.6973377466201782, + "logits/rejected": -1.8542219400405884, + "logps/chosen": -1.4063862562179565, + "logps/rejected": -2.4727578163146973, + "loss": 1.4933, + "nll_loss": 1.4680652618408203, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14063863456249237, + "rewards/margins": 0.10663717240095139, + "rewards/rejected": -0.24727579951286316, + "step": 703 + }, + { + "epoch": 1.9416637901277183, + "grad_norm": 0.768374502658844, + "learning_rate": 1.681871107817208e-06, + "log_odds_chosen": 1.2967448234558105, + "log_odds_ratio": -0.2469194382429123, + "logits/chosen": -0.6639657616615295, + "logits/rejected": -1.750392198562622, + "logps/chosen": -1.3959019184112549, + "logps/rejected": -2.4921858310699463, + "loss": 1.4819, + "nll_loss": 1.4571783542633057, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13959020376205444, + "rewards/margins": 0.10962840914726257, + "rewards/rejected": -0.24921859800815582, + "step": 704 + }, + { + "epoch": 1.9444252675181222, + "grad_norm": 0.21738582849502563, + "learning_rate": 1.6743023451120831e-06, + "log_odds_chosen": 1.179488182067871, + "log_odds_ratio": -0.2750599980354309, + "logits/chosen": -0.5516219139099121, + "logits/rejected": -1.8831249475479126, + "logps/chosen": -1.524298071861267, + "logps/rejected": -2.542219638824463, + "loss": 1.5971, + "nll_loss": 1.569570541381836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15242981910705566, + "rewards/margins": 0.10179214924573898, + "rewards/rejected": -0.25422194600105286, + "step": 705 + }, + { + "epoch": 1.947186744908526, + "grad_norm": 0.1983586996793747, + "learning_rate": 1.6667420677278962e-06, + "log_odds_chosen": 1.2816970348358154, + "log_odds_ratio": -0.2563754618167877, + "logits/chosen": -0.5835578441619873, + "logits/rejected": -1.8975870609283447, + "logps/chosen": -1.4354478120803833, + "logps/rejected": -2.5307862758636475, + "loss": 1.5051, + "nll_loss": 1.4794516563415527, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14354479312896729, + "rewards/margins": 0.10953383892774582, + "rewards/rejected": -0.2530786395072937, + "step": 706 + }, + { + "epoch": 1.94994822229893, + "grad_norm": 0.21526382863521576, + "learning_rate": 1.659190353358196e-06, + "log_odds_chosen": 1.4398778676986694, + "log_odds_ratio": -0.22124187648296356, + "logits/chosen": -0.6108973026275635, + "logits/rejected": -1.833961844444275, + "logps/chosen": -1.3309154510498047, + "logps/rejected": -2.544623851776123, + "loss": 1.4061, + "nll_loss": 1.383970022201538, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13309155404567719, + "rewards/margins": 0.1213708445429802, + "rewards/rejected": -0.2544623911380768, + "step": 707 + }, + { + "epoch": 1.9527096996893338, + "grad_norm": 0.21251551806926727, + "learning_rate": 1.6516472796085314e-06, + "log_odds_chosen": 1.4610146284103394, + "log_odds_ratio": -0.2106262743473053, + "logits/chosen": -0.716134786605835, + "logits/rejected": -2.0480661392211914, + "logps/chosen": -1.3701903820037842, + "logps/rejected": -2.611903190612793, + "loss": 1.4403, + "nll_loss": 1.419216275215149, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13701903820037842, + "rewards/margins": 0.12417127192020416, + "rewards/rejected": -0.2611902952194214, + "step": 708 + }, + { + "epoch": 1.9554711770797377, + "grad_norm": 0.21623145043849945, + "learning_rate": 1.644112923995656e-06, + "log_odds_chosen": 1.4195404052734375, + "log_odds_ratio": -0.2321530133485794, + "logits/chosen": -0.6861832737922668, + "logits/rejected": -1.6703221797943115, + "logps/chosen": -1.414649248123169, + "logps/rejected": -2.63264536857605, + "loss": 1.4826, + "nll_loss": 1.4593451023101807, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1414649337530136, + "rewards/margins": 0.12179961055517197, + "rewards/rejected": -0.263264536857605, + "step": 709 + }, + { + "epoch": 1.9582326544701414, + "grad_norm": 0.19821126759052277, + "learning_rate": 1.6365873639467314e-06, + "log_odds_chosen": 1.1184158325195312, + "log_odds_ratio": -0.2991623878479004, + "logits/chosen": -0.5886229276657104, + "logits/rejected": -1.7395761013031006, + "logps/chosen": -1.3078004121780396, + "logps/rejected": -2.224254608154297, + "loss": 1.401, + "nll_loss": 1.371075987815857, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13078004121780396, + "rewards/margins": 0.09164540469646454, + "rewards/rejected": -0.2224254608154297, + "step": 710 + }, + { + "epoch": 1.9609941318605455, + "grad_norm": 0.20421788096427917, + "learning_rate": 1.6290706767985299e-06, + "log_odds_chosen": 1.2795296907424927, + "log_odds_ratio": -0.2593962252140045, + "logits/chosen": -0.6316720843315125, + "logits/rejected": -1.9121975898742676, + "logps/chosen": -1.3871182203292847, + "logps/rejected": -2.4652185440063477, + "loss": 1.4544, + "nll_loss": 1.4284642934799194, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1387118101119995, + "rewards/margins": 0.10781005024909973, + "rewards/rejected": -0.24652186036109924, + "step": 711 + }, + { + "epoch": 1.9637556092509492, + "grad_norm": 0.23917905986309052, + "learning_rate": 1.6215629397966432e-06, + "log_odds_chosen": 1.2976970672607422, + "log_odds_ratio": -0.24504350125789642, + "logits/chosen": -0.6498063802719116, + "logits/rejected": -1.8320040702819824, + "logps/chosen": -1.3817360401153564, + "logps/rejected": -2.479696273803711, + "loss": 1.461, + "nll_loss": 1.436539888381958, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13817362487316132, + "rewards/margins": 0.10979600250720978, + "rewards/rejected": -0.2479696273803711, + "step": 712 + }, + { + "epoch": 1.9665170866413533, + "grad_norm": 0.21537664532661438, + "learning_rate": 1.614064230094684e-06, + "log_odds_chosen": 1.2931197881698608, + "log_odds_ratio": -0.24594642221927643, + "logits/chosen": -0.7107862830162048, + "logits/rejected": -1.7718734741210938, + "logps/chosen": -1.439354658126831, + "logps/rejected": -2.5453765392303467, + "loss": 1.5165, + "nll_loss": 1.4919335842132568, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14393548667430878, + "rewards/margins": 0.11060215532779694, + "rewards/rejected": -0.2545376420021057, + "step": 713 + }, + { + "epoch": 1.969278564031757, + "grad_norm": 0.20869500935077667, + "learning_rate": 1.6065746247534986e-06, + "log_odds_chosen": 1.3867168426513672, + "log_odds_ratio": -0.23219045996665955, + "logits/chosen": -0.6156042218208313, + "logits/rejected": -1.8871080875396729, + "logps/chosen": -1.3930268287658691, + "logps/rejected": -2.572889804840088, + "loss": 1.4852, + "nll_loss": 1.4620187282562256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13930267095565796, + "rewards/margins": 0.11798630654811859, + "rewards/rejected": -0.25728896260261536, + "step": 714 + }, + { + "epoch": 1.9720400414221608, + "grad_norm": 0.22620144486427307, + "learning_rate": 1.599094200740367e-06, + "log_odds_chosen": 1.1833250522613525, + "log_odds_ratio": -0.2710687816143036, + "logits/chosen": -0.6892810463905334, + "logits/rejected": -1.728080153465271, + "logps/chosen": -1.4800595045089722, + "logps/rejected": -2.4893577098846436, + "loss": 1.557, + "nll_loss": 1.5298923254013062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14800597727298737, + "rewards/margins": 0.10092981159687042, + "rewards/rejected": -0.2489357888698578, + "step": 715 + }, + { + "epoch": 1.9748015188125647, + "grad_norm": 0.2466319352388382, + "learning_rate": 1.5916230349282215e-06, + "log_odds_chosen": 1.4189105033874512, + "log_odds_ratio": -0.23354926705360413, + "logits/chosen": -0.759655237197876, + "logits/rejected": -1.570073127746582, + "logps/chosen": -1.3400826454162598, + "logps/rejected": -2.537966728210449, + "loss": 1.431, + "nll_loss": 1.407649278640747, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1340082734823227, + "rewards/margins": 0.11978840827941895, + "rewards/rejected": -0.25379669666290283, + "step": 716 + }, + { + "epoch": 1.9775629962029686, + "grad_norm": 0.21501441299915314, + "learning_rate": 1.58416120409485e-06, + "log_odds_chosen": 1.2824525833129883, + "log_odds_ratio": -0.24649572372436523, + "logits/chosen": -0.6352999806404114, + "logits/rejected": -2.0760064125061035, + "logps/chosen": -1.4632048606872559, + "logps/rejected": -2.5624969005584717, + "loss": 1.5135, + "nll_loss": 1.4888546466827393, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14632049202919006, + "rewards/margins": 0.10992921143770218, + "rewards/rejected": -0.25624969601631165, + "step": 717 + }, + { + "epoch": 1.9803244735933725, + "grad_norm": 0.20214088261127472, + "learning_rate": 1.5767087849221097e-06, + "log_odds_chosen": 1.2076809406280518, + "log_odds_ratio": -0.2692188620567322, + "logits/chosen": -0.6633406281471252, + "logits/rejected": -1.7296428680419922, + "logps/chosen": -1.385692834854126, + "logps/rejected": -2.4033093452453613, + "loss": 1.4713, + "nll_loss": 1.4444183111190796, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13856931030750275, + "rewards/margins": 0.10176165401935577, + "rewards/rejected": -0.24033094942569733, + "step": 718 + }, + { + "epoch": 1.9830859509837762, + "grad_norm": 0.21768531203269958, + "learning_rate": 1.5692658539951371e-06, + "log_odds_chosen": 1.3379472494125366, + "log_odds_ratio": -0.23937927186489105, + "logits/chosen": -0.7326225638389587, + "logits/rejected": -1.9709231853485107, + "logps/chosen": -1.4037171602249146, + "logps/rejected": -2.5365142822265625, + "loss": 1.4892, + "nll_loss": 1.4652434587478638, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14037172496318817, + "rewards/margins": 0.11327968537807465, + "rewards/rejected": -0.2536514103412628, + "step": 719 + }, + { + "epoch": 1.9858474283741803, + "grad_norm": 0.21629256010055542, + "learning_rate": 1.561832487801565e-06, + "log_odds_chosen": 1.3326436281204224, + "log_odds_ratio": -0.2489023208618164, + "logits/chosen": -0.6759160161018372, + "logits/rejected": -1.9447509050369263, + "logps/chosen": -1.413867473602295, + "logps/rejected": -2.552180528640747, + "loss": 1.4818, + "nll_loss": 1.456876277923584, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1413867473602295, + "rewards/margins": 0.1138312891125679, + "rewards/rejected": -0.2552180588245392, + "step": 720 + }, + { + "epoch": 1.988608905764584, + "grad_norm": 0.2140151709318161, + "learning_rate": 1.5544087627307308e-06, + "log_odds_chosen": 1.4864577054977417, + "log_odds_ratio": -0.20958803594112396, + "logits/chosen": -0.7287192940711975, + "logits/rejected": -2.011051893234253, + "logps/chosen": -1.403037190437317, + "logps/rejected": -2.680103302001953, + "loss": 1.4696, + "nll_loss": 1.4486826658248901, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14030373096466064, + "rewards/margins": 0.1277066171169281, + "rewards/rejected": -0.26801031827926636, + "step": 721 + }, + { + "epoch": 1.991370383154988, + "grad_norm": 0.21654756367206573, + "learning_rate": 1.546994755072896e-06, + "log_odds_chosen": 1.4832570552825928, + "log_odds_ratio": -0.21186988055706024, + "logits/chosen": -0.6306319832801819, + "logits/rejected": -1.948667287826538, + "logps/chosen": -1.3662739992141724, + "logps/rejected": -2.6316754817962646, + "loss": 1.4518, + "nll_loss": 1.4306564331054688, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13662739098072052, + "rewards/margins": 0.1265401393175125, + "rewards/rejected": -0.2631675601005554, + "step": 722 + }, + { + "epoch": 1.9941318605453917, + "grad_norm": 0.2074047029018402, + "learning_rate": 1.539590541018461e-06, + "log_odds_chosen": 1.355461597442627, + "log_odds_ratio": -0.23559004068374634, + "logits/chosen": -0.7348309755325317, + "logits/rejected": -1.9341561794281006, + "logps/chosen": -1.4475115537643433, + "logps/rejected": -2.612541675567627, + "loss": 1.5228, + "nll_loss": 1.4992461204528809, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1447511613368988, + "rewards/margins": 0.1165030226111412, + "rewards/rejected": -0.2612541913986206, + "step": 723 + }, + { + "epoch": 1.9968933379357956, + "grad_norm": 0.21759307384490967, + "learning_rate": 1.5321961966571801e-06, + "log_odds_chosen": 1.2848310470581055, + "log_odds_ratio": -0.2530062794685364, + "logits/chosen": -0.7775071859359741, + "logits/rejected": -1.8053133487701416, + "logps/chosen": -1.4230960607528687, + "logps/rejected": -2.5227210521698, + "loss": 1.4948, + "nll_loss": 1.469494104385376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14230960607528687, + "rewards/margins": 0.10996250063180923, + "rewards/rejected": -0.2522720992565155, + "step": 724 + }, + { + "epoch": 1.9996548153261995, + "grad_norm": 0.20666009187698364, + "learning_rate": 1.524811797977383e-06, + "log_odds_chosen": 1.1944769620895386, + "log_odds_ratio": -0.2727535367012024, + "logits/chosen": -0.7024763822555542, + "logits/rejected": -1.7682558298110962, + "logps/chosen": -1.462043285369873, + "logps/rejected": -2.4807724952697754, + "loss": 1.5248, + "nll_loss": 1.4975111484527588, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14620435237884521, + "rewards/margins": 0.10187289118766785, + "rewards/rejected": -0.24807721376419067, + "step": 725 + }, + { + "epoch": 2.0, + "grad_norm": 0.49442198872566223, + "learning_rate": 1.5174374208651913e-06, + "log_odds_chosen": 1.194680094718933, + "log_odds_ratio": -0.2645164132118225, + "logits/chosen": -0.6112655997276306, + "logits/rejected": -2.1095049381256104, + "logps/chosen": -1.4013067483901978, + "logps/rejected": -2.4076037406921387, + "loss": 1.4474, + "nll_loss": 1.4209479093551636, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1401306837797165, + "rewards/margins": 0.10062968730926514, + "rewards/rejected": -0.24076037108898163, + "step": 726 + }, + { + "epoch": 2.0027614773904037, + "grad_norm": 0.19985070824623108, + "learning_rate": 1.5100731411037395e-06, + "log_odds_chosen": 1.524064064025879, + "log_odds_ratio": -0.20543928444385529, + "logits/chosen": -0.7049272656440735, + "logits/rejected": -1.9933799505233765, + "logps/chosen": -1.4320381879806519, + "logps/rejected": -2.751387119293213, + "loss": 1.5037, + "nll_loss": 1.4831523895263672, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14320380985736847, + "rewards/margins": 0.13193488121032715, + "rewards/rejected": -0.2751386761665344, + "step": 727 + }, + { + "epoch": 2.0055229547808078, + "grad_norm": 0.1995333880186081, + "learning_rate": 1.502719034372396e-06, + "log_odds_chosen": 1.1505608558654785, + "log_odds_ratio": -0.28334516286849976, + "logits/chosen": -0.78338623046875, + "logits/rejected": -1.6854302883148193, + "logps/chosen": -1.3915482759475708, + "logps/rejected": -2.359539270401001, + "loss": 1.457, + "nll_loss": 1.4286986589431763, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1391548365354538, + "rewards/margins": 0.0967991054058075, + "rewards/rejected": -0.2359539419412613, + "step": 728 + }, + { + "epoch": 2.0082844321712114, + "grad_norm": 0.2035675048828125, + "learning_rate": 1.4953751762459867e-06, + "log_odds_chosen": 1.4010441303253174, + "log_odds_ratio": -0.24087637662887573, + "logits/chosen": -0.675167977809906, + "logits/rejected": -1.913004994392395, + "logps/chosen": -1.3706873655319214, + "logps/rejected": -2.5642473697662354, + "loss": 1.4511, + "nll_loss": 1.4269742965698242, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1370687186717987, + "rewards/margins": 0.11935600638389587, + "rewards/rejected": -0.2564247250556946, + "step": 729 + }, + { + "epoch": 2.0110459095616156, + "grad_norm": 0.2269647717475891, + "learning_rate": 1.4880416421940155e-06, + "log_odds_chosen": 1.3267796039581299, + "log_odds_ratio": -0.24957221746444702, + "logits/chosen": -0.7462670803070068, + "logits/rejected": -1.7765371799468994, + "logps/chosen": -1.4251582622528076, + "logps/rejected": -2.561957836151123, + "loss": 1.4963, + "nll_loss": 1.471299409866333, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14251583814620972, + "rewards/margins": 0.11367997527122498, + "rewards/rejected": -0.2561958134174347, + "step": 730 + }, + { + "epoch": 2.013807386952019, + "grad_norm": 0.21582657098770142, + "learning_rate": 1.4807185075798919e-06, + "log_odds_chosen": 1.455092430114746, + "log_odds_ratio": -0.2169358879327774, + "logits/chosen": -0.8274435997009277, + "logits/rejected": -2.024759292602539, + "logps/chosen": -1.4748046398162842, + "logps/rejected": -2.735257148742676, + "loss": 1.5375, + "nll_loss": 1.5158442258834839, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14748047292232513, + "rewards/margins": 0.12604522705078125, + "rewards/rejected": -0.2735256850719452, + "step": 731 + }, + { + "epoch": 2.0165688643424233, + "grad_norm": 0.20380431413650513, + "learning_rate": 1.4734058476601553e-06, + "log_odds_chosen": 1.4209482669830322, + "log_odds_ratio": -0.2209295928478241, + "logits/chosen": -0.7545549869537354, + "logits/rejected": -2.0098748207092285, + "logps/chosen": -1.4165632724761963, + "logps/rejected": -2.6308608055114746, + "loss": 1.4862, + "nll_loss": 1.4640896320343018, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14165633916854858, + "rewards/margins": 0.12142977118492126, + "rewards/rejected": -0.26308611035346985, + "step": 732 + }, + { + "epoch": 2.019330341732827, + "grad_norm": 0.20928171277046204, + "learning_rate": 1.466103737583699e-06, + "log_odds_chosen": 1.3327419757843018, + "log_odds_ratio": -0.24103975296020508, + "logits/chosen": -0.7420281767845154, + "logits/rejected": -1.969616413116455, + "logps/chosen": -1.4328577518463135, + "logps/rejected": -2.5726540088653564, + "loss": 1.5061, + "nll_loss": 1.481981873512268, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14328575134277344, + "rewards/margins": 0.11397963762283325, + "rewards/rejected": -0.2572654187679291, + "step": 733 + }, + { + "epoch": 2.022091819123231, + "grad_norm": 0.20876739919185638, + "learning_rate": 1.4588122523910032e-06, + "log_odds_chosen": 1.093473196029663, + "log_odds_ratio": -0.29052767157554626, + "logits/chosen": -0.7385894656181335, + "logits/rejected": -1.8053125143051147, + "logps/chosen": -1.456524133682251, + "logps/rejected": -2.3806233406066895, + "loss": 1.5197, + "nll_loss": 1.4906771183013916, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1456524282693863, + "rewards/margins": 0.09240993857383728, + "rewards/rejected": -0.23806238174438477, + "step": 734 + }, + { + "epoch": 2.0248532965136348, + "grad_norm": 0.2023276835680008, + "learning_rate": 1.4515314670133582e-06, + "log_odds_chosen": 1.535997986793518, + "log_odds_ratio": -0.21157340705394745, + "logits/chosen": -0.7386196851730347, + "logits/rejected": -1.8687727451324463, + "logps/chosen": -1.419616937637329, + "logps/rejected": -2.751965284347534, + "loss": 1.478, + "nll_loss": 1.456872820854187, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1419617086648941, + "rewards/margins": 0.13323484361171722, + "rewards/rejected": -0.27519655227661133, + "step": 735 + }, + { + "epoch": 2.027614773904039, + "grad_norm": 0.21986712515354156, + "learning_rate": 1.4442614562721011e-06, + "log_odds_chosen": 1.2944157123565674, + "log_odds_ratio": -0.25431200861930847, + "logits/chosen": -0.6299113035202026, + "logits/rejected": -1.713247537612915, + "logps/chosen": -1.3927693367004395, + "logps/rejected": -2.483180046081543, + "loss": 1.4661, + "nll_loss": 1.440664291381836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.139276921749115, + "rewards/margins": 0.10904109477996826, + "rewards/rejected": -0.24831801652908325, + "step": 736 + }, + { + "epoch": 2.0303762512944425, + "grad_norm": 0.2185211479663849, + "learning_rate": 1.4370022948778383e-06, + "log_odds_chosen": 1.4415481090545654, + "log_odds_ratio": -0.2162938416004181, + "logits/chosen": -0.6487110257148743, + "logits/rejected": -2.1030993461608887, + "logps/chosen": -1.4767889976501465, + "logps/rejected": -2.7231407165527344, + "loss": 1.5402, + "nll_loss": 1.5185596942901611, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1476789265871048, + "rewards/margins": 0.12463513016700745, + "rewards/rejected": -0.27231404185295105, + "step": 737 + }, + { + "epoch": 2.033137728684846, + "grad_norm": 0.22171227633953094, + "learning_rate": 1.429754057429687e-06, + "log_odds_chosen": 1.3265342712402344, + "log_odds_ratio": -0.2393992841243744, + "logits/chosen": -0.8432047963142395, + "logits/rejected": -1.8966618776321411, + "logps/chosen": -1.3895668983459473, + "logps/rejected": -2.5131988525390625, + "loss": 1.4717, + "nll_loss": 1.4477918148040771, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1389566957950592, + "rewards/margins": 0.11236318945884705, + "rewards/rejected": -0.25131988525390625, + "step": 738 + }, + { + "epoch": 2.0358992060752503, + "grad_norm": 0.42570745944976807, + "learning_rate": 1.422516818414499e-06, + "log_odds_chosen": 1.2274531126022339, + "log_odds_ratio": -0.26237213611602783, + "logits/chosen": -0.754129946231842, + "logits/rejected": -1.742024540901184, + "logps/chosen": -1.3718271255493164, + "logps/rejected": -2.3997960090637207, + "loss": 1.4529, + "nll_loss": 1.4266401529312134, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13718271255493164, + "rewards/margins": 0.10279688239097595, + "rewards/rejected": -0.2399796098470688, + "step": 739 + }, + { + "epoch": 2.038660683465654, + "grad_norm": 0.24944612383842468, + "learning_rate": 1.415290652206105e-06, + "log_odds_chosen": 1.3752034902572632, + "log_odds_ratio": -0.2288976013660431, + "logits/chosen": -0.784325361251831, + "logits/rejected": -1.8980276584625244, + "logps/chosen": -1.4716408252716064, + "logps/rejected": -2.658496379852295, + "loss": 1.5268, + "nll_loss": 1.503926157951355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14716407656669617, + "rewards/margins": 0.11868555843830109, + "rewards/rejected": -0.26584964990615845, + "step": 740 + }, + { + "epoch": 2.041422160856058, + "grad_norm": 0.19353602826595306, + "learning_rate": 1.4080756330645424e-06, + "log_odds_chosen": 1.3971836566925049, + "log_odds_ratio": -0.23170241713523865, + "logits/chosen": -0.757472038269043, + "logits/rejected": -1.8959999084472656, + "logps/chosen": -1.3243657350540161, + "logps/rejected": -2.496757745742798, + "loss": 1.3909, + "nll_loss": 1.3677642345428467, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1324365735054016, + "rewards/margins": 0.11723920702934265, + "rewards/rejected": -0.24967579543590546, + "step": 741 + }, + { + "epoch": 2.0441836382464618, + "grad_norm": 0.2381175011396408, + "learning_rate": 1.400871835135295e-06, + "log_odds_chosen": 1.488401174545288, + "log_odds_ratio": -0.2107824981212616, + "logits/chosen": -0.8121135830879211, + "logits/rejected": -1.801040768623352, + "logps/chosen": -1.395693063735962, + "logps/rejected": -2.6747279167175293, + "loss": 1.4644, + "nll_loss": 1.443336844444275, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13956931233406067, + "rewards/margins": 0.12790347635746002, + "rewards/rejected": -0.2674728035926819, + "step": 742 + }, + { + "epoch": 2.046945115636866, + "grad_norm": 0.21528197824954987, + "learning_rate": 1.3936793324485343e-06, + "log_odds_chosen": 1.426561713218689, + "log_odds_ratio": -0.2208237648010254, + "logits/chosen": -0.7020534873008728, + "logits/rejected": -1.9243749380111694, + "logps/chosen": -1.377315878868103, + "logps/rejected": -2.5889315605163574, + "loss": 1.4521, + "nll_loss": 1.4300365447998047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13773158192634583, + "rewards/margins": 0.12116158753633499, + "rewards/rejected": -0.2588931918144226, + "step": 743 + }, + { + "epoch": 2.0497065930272695, + "grad_norm": 0.21400389075279236, + "learning_rate": 1.386498198918352e-06, + "log_odds_chosen": 1.3467150926589966, + "log_odds_ratio": -0.23828084766864777, + "logits/chosen": -0.7199838161468506, + "logits/rejected": -2.0278215408325195, + "logps/chosen": -1.480491042137146, + "logps/rejected": -2.645315408706665, + "loss": 1.5512, + "nll_loss": 1.527329683303833, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14804911613464355, + "rewards/margins": 0.11648242175579071, + "rewards/rejected": -0.26453155279159546, + "step": 744 + }, + { + "epoch": 2.0524680704176737, + "grad_norm": 0.2323625087738037, + "learning_rate": 1.3793285083420077e-06, + "log_odds_chosen": 1.2312310934066772, + "log_odds_ratio": -0.26046109199523926, + "logits/chosen": -0.6492730379104614, + "logits/rejected": -1.7875938415527344, + "logps/chosen": -1.5298579931259155, + "logps/rejected": -2.5959219932556152, + "loss": 1.5921, + "nll_loss": 1.5660476684570312, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15298578143119812, + "rewards/margins": 0.10660640150308609, + "rewards/rejected": -0.259592205286026, + "step": 745 + }, + { + "epoch": 2.0552295478080773, + "grad_norm": 0.21785420179367065, + "learning_rate": 1.3721703343991634e-06, + "log_odds_chosen": 1.5281133651733398, + "log_odds_ratio": -0.20278286933898926, + "logits/chosen": -0.7953973412513733, + "logits/rejected": -2.098724603652954, + "logps/chosen": -1.403007984161377, + "logps/rejected": -2.718156337738037, + "loss": 1.4625, + "nll_loss": 1.4422367811203003, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14030081033706665, + "rewards/margins": 0.13151484727859497, + "rewards/rejected": -0.2718156576156616, + "step": 746 + }, + { + "epoch": 2.057991025198481, + "grad_norm": 0.2271818220615387, + "learning_rate": 1.3650237506511333e-06, + "log_odds_chosen": 1.3591433763504028, + "log_odds_ratio": -0.24462954699993134, + "logits/chosen": -0.7451946139335632, + "logits/rejected": -1.7923208475112915, + "logps/chosen": -1.3873848915100098, + "logps/rejected": -2.5456383228302, + "loss": 1.4824, + "nll_loss": 1.4579014778137207, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1387384980916977, + "rewards/margins": 0.1158253476023674, + "rewards/rejected": -0.2545638382434845, + "step": 747 + }, + { + "epoch": 2.060752502588885, + "grad_norm": 0.22212862968444824, + "learning_rate": 1.3578888305401208e-06, + "log_odds_chosen": 1.3162353038787842, + "log_odds_ratio": -0.2563769817352295, + "logits/chosen": -0.7208426594734192, + "logits/rejected": -1.8948148488998413, + "logps/chosen": -1.389916181564331, + "logps/rejected": -2.5131213665008545, + "loss": 1.4683, + "nll_loss": 1.4426243305206299, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13899163901805878, + "rewards/margins": 0.11232049018144608, + "rewards/rejected": -0.25131210684776306, + "step": 748 + }, + { + "epoch": 2.0635139799792888, + "grad_norm": 0.21235612034797668, + "learning_rate": 1.3507656473884718e-06, + "log_odds_chosen": 1.3354262113571167, + "log_odds_ratio": -0.24460864067077637, + "logits/chosen": -0.6753593683242798, + "logits/rejected": -1.8687498569488525, + "logps/chosen": -1.3753629922866821, + "logps/rejected": -2.499833583831787, + "loss": 1.4357, + "nll_loss": 1.4112131595611572, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13753631711006165, + "rewards/margins": 0.11244705319404602, + "rewards/rejected": -0.24998337030410767, + "step": 749 + }, + { + "epoch": 2.066275457369693, + "grad_norm": 0.2222043126821518, + "learning_rate": 1.3436542743979125e-06, + "log_odds_chosen": 1.1222965717315674, + "log_odds_ratio": -0.29325950145721436, + "logits/chosen": -0.7638905644416809, + "logits/rejected": -1.7164831161499023, + "logps/chosen": -1.3756606578826904, + "logps/rejected": -2.3173580169677734, + "loss": 1.4687, + "nll_loss": 1.4393634796142578, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13756605982780457, + "rewards/margins": 0.0941697359085083, + "rewards/rejected": -0.23173581063747406, + "step": 750 + }, + { + "epoch": 2.0690369347600965, + "grad_norm": 0.2122756689786911, + "learning_rate": 1.3365547846488037e-06, + "log_odds_chosen": 1.5225597620010376, + "log_odds_ratio": -0.20094801485538483, + "logits/chosen": -0.7458276152610779, + "logits/rejected": -1.9331128597259521, + "logps/chosen": -1.4181265830993652, + "logps/rejected": -2.7289392948150635, + "loss": 1.4843, + "nll_loss": 1.4641839265823364, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14181266725063324, + "rewards/margins": 0.1310812532901764, + "rewards/rejected": -0.2728939354419708, + "step": 751 + }, + { + "epoch": 2.0717984121505006, + "grad_norm": 0.2157318890094757, + "learning_rate": 1.3294672510993862e-06, + "log_odds_chosen": 1.4378488063812256, + "log_odds_ratio": -0.22265379130840302, + "logits/chosen": -0.7862150073051453, + "logits/rejected": -1.94209885597229, + "logps/chosen": -1.4849308729171753, + "logps/rejected": -2.7356879711151123, + "loss": 1.5487, + "nll_loss": 1.5264191627502441, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14849308133125305, + "rewards/margins": 0.12507571280002594, + "rewards/rejected": -0.2735688090324402, + "step": 752 + }, + { + "epoch": 2.0745598895409043, + "grad_norm": 0.21287965774536133, + "learning_rate": 1.3223917465850344e-06, + "log_odds_chosen": 1.291487216949463, + "log_odds_ratio": -0.25558802485466003, + "logits/chosen": -0.6685554385185242, + "logits/rejected": -1.8840442895889282, + "logps/chosen": -1.4667881727218628, + "logps/rejected": -2.579162836074829, + "loss": 1.543, + "nll_loss": 1.5174570083618164, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14667882025241852, + "rewards/margins": 0.11123748123645782, + "rewards/rejected": -0.25791627168655396, + "step": 753 + }, + { + "epoch": 2.0773213669313084, + "grad_norm": 0.23124369978904724, + "learning_rate": 1.3153283438175036e-06, + "log_odds_chosen": 1.5643819570541382, + "log_odds_ratio": -0.2072456032037735, + "logits/chosen": -0.7434122562408447, + "logits/rejected": -1.8903828859329224, + "logps/chosen": -1.498293161392212, + "logps/rejected": -2.8726089000701904, + "loss": 1.5474, + "nll_loss": 1.526721715927124, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14982932806015015, + "rewards/margins": 0.1374315768480301, + "rewards/rejected": -0.28726089000701904, + "step": 754 + }, + { + "epoch": 2.080082844321712, + "grad_norm": 0.24547821283340454, + "learning_rate": 1.3082771153841872e-06, + "log_odds_chosen": 1.359768271446228, + "log_odds_ratio": -0.2351943403482437, + "logits/chosen": -0.786487877368927, + "logits/rejected": -1.7432615756988525, + "logps/chosen": -1.4537529945373535, + "logps/rejected": -2.622997760772705, + "loss": 1.5433, + "nll_loss": 1.5197895765304565, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14537528157234192, + "rewards/margins": 0.1169244721531868, + "rewards/rejected": -0.2622997760772705, + "step": 755 + }, + { + "epoch": 2.082844321712116, + "grad_norm": 0.22222405672073364, + "learning_rate": 1.3012381337473656e-06, + "log_odds_chosen": 1.737757682800293, + "log_odds_ratio": -0.18532472848892212, + "logits/chosen": -0.6934542059898376, + "logits/rejected": -2.0250802040100098, + "logps/chosen": -1.2511564493179321, + "logps/rejected": -2.703065872192383, + "loss": 1.3362, + "nll_loss": 1.3176548480987549, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12511564791202545, + "rewards/margins": 0.14519096910953522, + "rewards/rejected": -0.2703065872192383, + "step": 756 + }, + { + "epoch": 2.08560579910252, + "grad_norm": 0.21016424894332886, + "learning_rate": 1.294211471243466e-06, + "log_odds_chosen": 1.409480094909668, + "log_odds_ratio": -0.22224737703800201, + "logits/chosen": -0.7898975610733032, + "logits/rejected": -2.0010173320770264, + "logps/chosen": -1.36336088180542, + "logps/rejected": -2.5560050010681152, + "loss": 1.4278, + "nll_loss": 1.4055848121643066, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.136336088180542, + "rewards/margins": 0.1192643940448761, + "rewards/rejected": -0.2556004822254181, + "step": 757 + }, + { + "epoch": 2.0883672764929235, + "grad_norm": 0.20899075269699097, + "learning_rate": 1.2871972000823197e-06, + "log_odds_chosen": 1.3560638427734375, + "log_odds_ratio": -0.23246008157730103, + "logits/chosen": -0.7394353151321411, + "logits/rejected": -2.0328736305236816, + "logps/chosen": -1.3651076555252075, + "logps/rejected": -2.5123071670532227, + "loss": 1.4393, + "nll_loss": 1.4160305261611938, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13651075959205627, + "rewards/margins": 0.11471996456384659, + "rewards/rejected": -0.25123071670532227, + "step": 758 + }, + { + "epoch": 2.0911287538833276, + "grad_norm": 0.20122100412845612, + "learning_rate": 1.2801953923464139e-06, + "log_odds_chosen": 1.322190761566162, + "log_odds_ratio": -0.2632856070995331, + "logits/chosen": -0.7781534790992737, + "logits/rejected": -1.7706246376037598, + "logps/chosen": -1.3630403280258179, + "logps/rejected": -2.4846913814544678, + "loss": 1.4516, + "nll_loss": 1.4252920150756836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13630403578281403, + "rewards/margins": 0.11216511577367783, + "rewards/rejected": -0.24846915900707245, + "step": 759 + }, + { + "epoch": 2.0938902312737313, + "grad_norm": 0.22142185270786285, + "learning_rate": 1.2732061199901563e-06, + "log_odds_chosen": 1.2743151187896729, + "log_odds_ratio": -0.2580106258392334, + "logits/chosen": -0.7141470313072205, + "logits/rejected": -1.5875985622406006, + "logps/chosen": -1.4468305110931396, + "logps/rejected": -2.5337939262390137, + "loss": 1.5244, + "nll_loss": 1.4985558986663818, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14468304812908173, + "rewards/margins": 0.108696348965168, + "rewards/rejected": -0.2533794045448303, + "step": 760 + }, + { + "epoch": 2.0966517086641354, + "grad_norm": 0.2264997810125351, + "learning_rate": 1.266229454839133e-06, + "log_odds_chosen": 1.2582509517669678, + "log_odds_ratio": -0.2561207711696625, + "logits/chosen": -0.8011865615844727, + "logits/rejected": -1.6508357524871826, + "logps/chosen": -1.3973218202590942, + "logps/rejected": -2.462202310562134, + "loss": 1.4649, + "nll_loss": 1.4393376111984253, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13973219692707062, + "rewards/margins": 0.10648804903030396, + "rewards/rejected": -0.24622023105621338, + "step": 761 + }, + { + "epoch": 2.099413186054539, + "grad_norm": 0.23098450899124146, + "learning_rate": 1.2592654685893757e-06, + "log_odds_chosen": 1.5264986753463745, + "log_odds_ratio": -0.20155426859855652, + "logits/chosen": -0.66983962059021, + "logits/rejected": -1.9511102437973022, + "logps/chosen": -1.396782398223877, + "logps/rejected": -2.706751585006714, + "loss": 1.4673, + "nll_loss": 1.4471567869186401, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1396782398223877, + "rewards/margins": 0.1309969276189804, + "rewards/rejected": -0.2706751525402069, + "step": 762 + }, + { + "epoch": 2.102174663444943, + "grad_norm": 0.23991060256958008, + "learning_rate": 1.252314232806615e-06, + "log_odds_chosen": 1.432591438293457, + "log_odds_ratio": -0.23098701238632202, + "logits/chosen": -0.8026631474494934, + "logits/rejected": -1.9127060174942017, + "logps/chosen": -1.4595669507980347, + "logps/rejected": -2.698004722595215, + "loss": 1.5126, + "nll_loss": 1.4895089864730835, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14595669507980347, + "rewards/margins": 0.12384375929832458, + "rewards/rejected": -0.26980048418045044, + "step": 763 + }, + { + "epoch": 2.104936140835347, + "grad_norm": 0.21347320079803467, + "learning_rate": 1.2453758189255568e-06, + "log_odds_chosen": 1.3525452613830566, + "log_odds_ratio": -0.2315671443939209, + "logits/chosen": -0.7290445566177368, + "logits/rejected": -2.068817138671875, + "logps/chosen": -1.3774614334106445, + "logps/rejected": -2.521169900894165, + "loss": 1.4489, + "nll_loss": 1.4257827997207642, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13774614036083221, + "rewards/margins": 0.11437083780765533, + "rewards/rejected": -0.25211697816848755, + "step": 764 + }, + { + "epoch": 2.107697618225751, + "grad_norm": 0.22012005746364594, + "learning_rate": 1.2384502982491359e-06, + "log_odds_chosen": 1.4865221977233887, + "log_odds_ratio": -0.21373732388019562, + "logits/chosen": -0.7536633610725403, + "logits/rejected": -2.040149688720703, + "logps/chosen": -1.5073720216751099, + "logps/rejected": -2.8081774711608887, + "loss": 1.578, + "nll_loss": 1.5566452741622925, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1507372111082077, + "rewards/margins": 0.13008056581020355, + "rewards/rejected": -0.28081777691841125, + "step": 765 + }, + { + "epoch": 2.1104590956161546, + "grad_norm": 0.22354905307292938, + "learning_rate": 1.231537741947795e-06, + "log_odds_chosen": 1.2573996782302856, + "log_odds_ratio": -0.26209187507629395, + "logits/chosen": -0.7578604817390442, + "logits/rejected": -1.8252947330474854, + "logps/chosen": -1.4263148307800293, + "logps/rejected": -2.4963927268981934, + "loss": 1.501, + "nll_loss": 1.4747966527938843, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14263150095939636, + "rewards/margins": 0.10700778663158417, + "rewards/rejected": -0.24963928759098053, + "step": 766 + }, + { + "epoch": 2.1132205730065583, + "grad_norm": 0.2208777815103531, + "learning_rate": 1.2246382210587432e-06, + "log_odds_chosen": 1.5075188875198364, + "log_odds_ratio": -0.20569130778312683, + "logits/chosen": -0.7639518976211548, + "logits/rejected": -1.967806100845337, + "logps/chosen": -1.4122600555419922, + "logps/rejected": -2.710855722427368, + "loss": 1.4812, + "nll_loss": 1.4606313705444336, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14122600853443146, + "rewards/margins": 0.1298595815896988, + "rewards/rejected": -0.27108559012413025, + "step": 767 + }, + { + "epoch": 2.1159820503969624, + "grad_norm": 0.20796267688274384, + "learning_rate": 1.217751806485235e-06, + "log_odds_chosen": 1.4210944175720215, + "log_odds_ratio": -0.22676415741443634, + "logits/chosen": -0.7732008099555969, + "logits/rejected": -1.9926378726959229, + "logps/chosen": -1.3832429647445679, + "logps/rejected": -2.59248423576355, + "loss": 1.4545, + "nll_loss": 1.43184494972229, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1383242905139923, + "rewards/margins": 0.12092413753271103, + "rewards/rejected": -0.25924843549728394, + "step": 768 + }, + { + "epoch": 2.118743527787366, + "grad_norm": 0.21591438353061676, + "learning_rate": 1.2108785689958337e-06, + "log_odds_chosen": 1.2549666166305542, + "log_odds_ratio": -0.2598121464252472, + "logits/chosen": -0.6248151659965515, + "logits/rejected": -1.732608437538147, + "logps/chosen": -1.4587433338165283, + "logps/rejected": -2.5326249599456787, + "loss": 1.5275, + "nll_loss": 1.5015511512756348, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14587435126304626, + "rewards/margins": 0.10738813877105713, + "rewards/rejected": -0.253262460231781, + "step": 769 + }, + { + "epoch": 2.12150500517777, + "grad_norm": 0.21577268838882446, + "learning_rate": 1.2040185792236874e-06, + "log_odds_chosen": 1.386768102645874, + "log_odds_ratio": -0.22949083149433136, + "logits/chosen": -0.7459929585456848, + "logits/rejected": -1.90436589717865, + "logps/chosen": -1.3482396602630615, + "logps/rejected": -2.517188310623169, + "loss": 1.4235, + "nll_loss": 1.400557279586792, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1348239779472351, + "rewards/margins": 0.11689485609531403, + "rewards/rejected": -0.2517188489437103, + "step": 770 + }, + { + "epoch": 2.124266482568174, + "grad_norm": 0.23260287940502167, + "learning_rate": 1.197171907665808e-06, + "log_odds_chosen": 1.4571518898010254, + "log_odds_ratio": -0.21860212087631226, + "logits/chosen": -0.8009171485900879, + "logits/rejected": -1.845001459121704, + "logps/chosen": -1.393100380897522, + "logps/rejected": -2.638636589050293, + "loss": 1.4644, + "nll_loss": 1.4425642490386963, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1393100470304489, + "rewards/margins": 0.1245536133646965, + "rewards/rejected": -0.2638636827468872, + "step": 771 + }, + { + "epoch": 2.127027959958578, + "grad_norm": 0.21608783304691315, + "learning_rate": 1.1903386246823363e-06, + "log_odds_chosen": 1.6979553699493408, + "log_odds_ratio": -0.1744653582572937, + "logits/chosen": -0.7080166935920715, + "logits/rejected": -2.05287504196167, + "logps/chosen": -1.4491580724716187, + "logps/rejected": -2.9308226108551025, + "loss": 1.5145, + "nll_loss": 1.4970835447311401, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14491581916809082, + "rewards/margins": 0.1481664478778839, + "rewards/rejected": -0.2930822968482971, + "step": 772 + }, + { + "epoch": 2.1297894373489816, + "grad_norm": 0.21926312148571014, + "learning_rate": 1.1835188004958298e-06, + "log_odds_chosen": 1.411818265914917, + "log_odds_ratio": -0.22714708745479584, + "logits/chosen": -0.8942671418190002, + "logits/rejected": -2.0131399631500244, + "logps/chosen": -1.3782057762145996, + "logps/rejected": -2.5774431228637695, + "loss": 1.4443, + "nll_loss": 1.4216082096099854, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13782057166099548, + "rewards/margins": 0.11992374807596207, + "rewards/rejected": -0.25774431228637695, + "step": 773 + }, + { + "epoch": 2.1325509147393857, + "grad_norm": 0.22599764168262482, + "learning_rate": 1.1767125051905314e-06, + "log_odds_chosen": 1.4329699277877808, + "log_odds_ratio": -0.2236868441104889, + "logits/chosen": -0.8578775525093079, + "logits/rejected": -1.9920375347137451, + "logps/chosen": -1.3404793739318848, + "logps/rejected": -2.5507805347442627, + "loss": 1.4275, + "nll_loss": 1.4050943851470947, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13404794037342072, + "rewards/margins": 0.12103011459112167, + "rewards/rejected": -0.2550780475139618, + "step": 774 + }, + { + "epoch": 2.1353123921297894, + "grad_norm": 0.20496487617492676, + "learning_rate": 1.169919808711659e-06, + "log_odds_chosen": 1.492319941520691, + "log_odds_ratio": -0.20643500983715057, + "logits/chosen": -0.7351235747337341, + "logits/rejected": -2.068713665008545, + "logps/chosen": -1.4658230543136597, + "logps/rejected": -2.7593133449554443, + "loss": 1.5291, + "nll_loss": 1.5084741115570068, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14658233523368835, + "rewards/margins": 0.1293489933013916, + "rewards/rejected": -0.27593132853507996, + "step": 775 + }, + { + "epoch": 2.138073869520193, + "grad_norm": 0.227569580078125, + "learning_rate": 1.1631407808646758e-06, + "log_odds_chosen": 1.5084763765335083, + "log_odds_ratio": -0.20716872811317444, + "logits/chosen": -0.8234575390815735, + "logits/rejected": -1.9566833972930908, + "logps/chosen": -1.480001449584961, + "logps/rejected": -2.793423891067505, + "loss": 1.5407, + "nll_loss": 1.5199940204620361, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14800015091896057, + "rewards/margins": 0.13134226202964783, + "rewards/rejected": -0.2793424129486084, + "step": 776 + }, + { + "epoch": 2.140835346910597, + "grad_norm": 0.22688134014606476, + "learning_rate": 1.156375491314585e-06, + "log_odds_chosen": 1.4476473331451416, + "log_odds_ratio": -0.21755696833133698, + "logits/chosen": -0.8190490007400513, + "logits/rejected": -2.084383964538574, + "logps/chosen": -1.4168970584869385, + "logps/rejected": -2.659780979156494, + "loss": 1.4805, + "nll_loss": 1.4587358236312866, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1416897177696228, + "rewards/margins": 0.12428837269544601, + "rewards/rejected": -0.265978068113327, + "step": 777 + }, + { + "epoch": 2.143596824301001, + "grad_norm": 0.21955221891403198, + "learning_rate": 1.1496240095852002e-06, + "log_odds_chosen": 1.4811240434646606, + "log_odds_ratio": -0.2159479707479477, + "logits/chosen": -0.7700929045677185, + "logits/rejected": -2.0682449340820312, + "logps/chosen": -1.4933757781982422, + "logps/rejected": -2.7856602668762207, + "loss": 1.5621, + "nll_loss": 1.5405230522155762, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14933758974075317, + "rewards/margins": 0.12922842800617218, + "rewards/rejected": -0.27856600284576416, + "step": 778 + }, + { + "epoch": 2.146358301691405, + "grad_norm": 0.19441139698028564, + "learning_rate": 1.142886405058445e-06, + "log_odds_chosen": 1.5811694860458374, + "log_odds_ratio": -0.20655685663223267, + "logits/chosen": -0.7932131290435791, + "logits/rejected": -2.209730863571167, + "logps/chosen": -1.414037823677063, + "logps/rejected": -2.7863986492156982, + "loss": 1.4757, + "nll_loss": 1.4550689458847046, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14140377938747406, + "rewards/margins": 0.137236088514328, + "rewards/rejected": -0.27863985300064087, + "step": 779 + }, + { + "epoch": 2.1491197790818086, + "grad_norm": 0.21649286150932312, + "learning_rate": 1.1361627469736286e-06, + "log_odds_chosen": 1.2342243194580078, + "log_odds_ratio": -0.26484403014183044, + "logits/chosen": -0.7019641995429993, + "logits/rejected": -2.029801368713379, + "logps/chosen": -1.4683226346969604, + "logps/rejected": -2.5286829471588135, + "loss": 1.5529, + "nll_loss": 1.5264643430709839, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14683228731155396, + "rewards/margins": 0.10603601485490799, + "rewards/rejected": -0.25286829471588135, + "step": 780 + }, + { + "epoch": 2.1518812564722127, + "grad_norm": 0.21707890927791595, + "learning_rate": 1.1294531044267418e-06, + "log_odds_chosen": 1.2695517539978027, + "log_odds_ratio": -0.2593621015548706, + "logits/chosen": -0.7232626676559448, + "logits/rejected": -1.7367572784423828, + "logps/chosen": -1.4411171674728394, + "logps/rejected": -2.5253968238830566, + "loss": 1.5015, + "nll_loss": 1.4755988121032715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1441117227077484, + "rewards/margins": 0.1084279716014862, + "rewards/rejected": -0.2525396943092346, + "step": 781 + }, + { + "epoch": 2.1546427338626164, + "grad_norm": 0.21778395771980286, + "learning_rate": 1.122757546369744e-06, + "log_odds_chosen": 1.3463826179504395, + "log_odds_ratio": -0.24496321380138397, + "logits/chosen": -0.7988300323486328, + "logits/rejected": -1.561537504196167, + "logps/chosen": -1.4459059238433838, + "logps/rejected": -2.5993716716766357, + "loss": 1.5201, + "nll_loss": 1.49556565284729, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1445905864238739, + "rewards/margins": 0.11534658074378967, + "rewards/rejected": -0.2599371671676636, + "step": 782 + }, + { + "epoch": 2.1574042112530205, + "grad_norm": 0.24371539056301117, + "learning_rate": 1.1160761416098518e-06, + "log_odds_chosen": 1.4046707153320312, + "log_odds_ratio": -0.2345598042011261, + "logits/chosen": -0.7829396724700928, + "logits/rejected": -1.7336349487304688, + "logps/chosen": -1.466633915901184, + "logps/rejected": -2.6856021881103516, + "loss": 1.5183, + "nll_loss": 1.494824767112732, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14666339755058289, + "rewards/margins": 0.12189682573080063, + "rewards/rejected": -0.2685602307319641, + "step": 783 + }, + { + "epoch": 2.160165688643424, + "grad_norm": 0.2371114045381546, + "learning_rate": 1.1094089588088383e-06, + "log_odds_chosen": 1.6812859773635864, + "log_odds_ratio": -0.18432164192199707, + "logits/chosen": -0.7592156529426575, + "logits/rejected": -1.956417202949524, + "logps/chosen": -1.4686094522476196, + "logps/rejected": -2.9425501823425293, + "loss": 1.5348, + "nll_loss": 1.5163860321044922, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14686094224452972, + "rewards/margins": 0.1473940908908844, + "rewards/rejected": -0.2942550480365753, + "step": 784 + }, + { + "epoch": 2.1629271660338283, + "grad_norm": 0.20977692306041718, + "learning_rate": 1.1027560664823208e-06, + "log_odds_chosen": 1.4073948860168457, + "log_odds_ratio": -0.2274792492389679, + "logits/chosen": -0.7849516272544861, + "logits/rejected": -2.003600597381592, + "logps/chosen": -1.4130662679672241, + "logps/rejected": -2.6155569553375244, + "loss": 1.4807, + "nll_loss": 1.4579068422317505, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14130662381649017, + "rewards/margins": 0.12024906277656555, + "rewards/rejected": -0.2615557014942169, + "step": 785 + }, + { + "epoch": 2.165688643424232, + "grad_norm": 0.21764487028121948, + "learning_rate": 1.0961175329990631e-06, + "log_odds_chosen": 1.6485837697982788, + "log_odds_ratio": -0.1814139187335968, + "logits/chosen": -0.7036556005477905, + "logits/rejected": -2.169128179550171, + "logps/chosen": -1.463451862335205, + "logps/rejected": -2.9016788005828857, + "loss": 1.5204, + "nll_loss": 1.5022188425064087, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14634516835212708, + "rewards/margins": 0.14382271468639374, + "rewards/rejected": -0.290167897939682, + "step": 786 + }, + { + "epoch": 2.1684501208146356, + "grad_norm": 0.20861107110977173, + "learning_rate": 1.0894934265802675e-06, + "log_odds_chosen": 1.4223312139511108, + "log_odds_ratio": -0.2421904057264328, + "logits/chosen": -0.7688660621643066, + "logits/rejected": -1.7660201787948608, + "logps/chosen": -1.4704207181930542, + "logps/rejected": -2.7102015018463135, + "loss": 1.5322, + "nll_loss": 1.507992148399353, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14704206585884094, + "rewards/margins": 0.12397809326648712, + "rewards/rejected": -0.27102014422416687, + "step": 787 + }, + { + "epoch": 2.1712115982050397, + "grad_norm": 0.20427848398685455, + "learning_rate": 1.082883815298876e-06, + "log_odds_chosen": 1.28458571434021, + "log_odds_ratio": -0.2563609480857849, + "logits/chosen": -0.8109991550445557, + "logits/rejected": -1.8216277360916138, + "logps/chosen": -1.3573540449142456, + "logps/rejected": -2.4361014366149902, + "loss": 1.434, + "nll_loss": 1.4083268642425537, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.135735422372818, + "rewards/margins": 0.10787472128868103, + "rewards/rejected": -0.24361014366149902, + "step": 788 + }, + { + "epoch": 2.1739730755954434, + "grad_norm": 0.21633489429950714, + "learning_rate": 1.0762887670788702e-06, + "log_odds_chosen": 1.3372876644134521, + "log_odds_ratio": -0.24584585428237915, + "logits/chosen": -0.7131776809692383, + "logits/rejected": -1.6811692714691162, + "logps/chosen": -1.4008711576461792, + "logps/rejected": -2.5390570163726807, + "loss": 1.4978, + "nll_loss": 1.4731900691986084, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14008711278438568, + "rewards/margins": 0.11381859332323074, + "rewards/rejected": -0.25390568375587463, + "step": 789 + }, + { + "epoch": 2.1767345529858475, + "grad_norm": 0.22346127033233643, + "learning_rate": 1.0697083496945766e-06, + "log_odds_chosen": 1.3910284042358398, + "log_odds_ratio": -0.22719833254814148, + "logits/chosen": -0.6821406483650208, + "logits/rejected": -1.8485900163650513, + "logps/chosen": -1.376333475112915, + "logps/rejected": -2.557509422302246, + "loss": 1.4589, + "nll_loss": 1.4361891746520996, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13763336837291718, + "rewards/margins": 0.1181175708770752, + "rewards/rejected": -0.25575095415115356, + "step": 790 + }, + { + "epoch": 2.179496030376251, + "grad_norm": 0.22128628194332123, + "learning_rate": 1.063142630769963e-06, + "log_odds_chosen": 1.2716386318206787, + "log_odds_ratio": -0.2559214234352112, + "logits/chosen": -0.7769445776939392, + "logits/rejected": -1.9463646411895752, + "logps/chosen": -1.3476147651672363, + "logps/rejected": -2.411182403564453, + "loss": 1.4282, + "nll_loss": 1.4026559591293335, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1347614824771881, + "rewards/margins": 0.10635677725076675, + "rewards/rejected": -0.24111826717853546, + "step": 791 + }, + { + "epoch": 2.1822575077666553, + "grad_norm": 0.21540698409080505, + "learning_rate": 1.0565916777779519e-06, + "log_odds_chosen": 1.4014261960983276, + "log_odds_ratio": -0.2265271246433258, + "logits/chosen": -0.7393918633460999, + "logits/rejected": -1.8673772811889648, + "logps/chosen": -1.4762778282165527, + "logps/rejected": -2.6873135566711426, + "loss": 1.5394, + "nll_loss": 1.5167272090911865, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1476277858018875, + "rewards/margins": 0.12110358476638794, + "rewards/rejected": -0.26873135566711426, + "step": 792 + }, + { + "epoch": 2.185018985157059, + "grad_norm": 0.21713268756866455, + "learning_rate": 1.0500555580397193e-06, + "log_odds_chosen": 1.5545601844787598, + "log_odds_ratio": -0.20979206264019012, + "logits/chosen": -0.7159947752952576, + "logits/rejected": -1.8904709815979004, + "logps/chosen": -1.4165940284729004, + "logps/rejected": -2.763000965118408, + "loss": 1.483, + "nll_loss": 1.4620165824890137, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14165939390659332, + "rewards/margins": 0.13464069366455078, + "rewards/rejected": -0.2763000726699829, + "step": 793 + }, + { + "epoch": 2.187780462547463, + "grad_norm": 0.22529271245002747, + "learning_rate": 1.04353433872401e-06, + "log_odds_chosen": 1.3307311534881592, + "log_odds_ratio": -0.238363116979599, + "logits/chosen": -0.7927432060241699, + "logits/rejected": -1.904407262802124, + "logps/chosen": -1.4591866731643677, + "logps/rejected": -2.602426290512085, + "loss": 1.5187, + "nll_loss": 1.4948735237121582, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14591866731643677, + "rewards/margins": 0.11432396620512009, + "rewards/rejected": -0.26024264097213745, + "step": 794 + }, + { + "epoch": 2.1905419399378667, + "grad_norm": 0.20850500464439392, + "learning_rate": 1.0370280868464405e-06, + "log_odds_chosen": 1.777038335800171, + "log_odds_ratio": -0.17781738936901093, + "logits/chosen": -0.7591485977172852, + "logits/rejected": -2.2028274536132812, + "logps/chosen": -1.424153208732605, + "logps/rejected": -2.9798388481140137, + "loss": 1.4756, + "nll_loss": 1.4577784538269043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14241531491279602, + "rewards/margins": 0.15556861460208893, + "rewards/rejected": -0.29798394441604614, + "step": 795 + }, + { + "epoch": 2.193303417328271, + "grad_norm": 0.22121626138687134, + "learning_rate": 1.0305368692688175e-06, + "log_odds_chosen": 1.6756552457809448, + "log_odds_ratio": -0.180609792470932, + "logits/chosen": -0.7418386936187744, + "logits/rejected": -1.983974814414978, + "logps/chosen": -1.327376365661621, + "logps/rejected": -2.759260654449463, + "loss": 1.3908, + "nll_loss": 1.3727234601974487, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1327376365661621, + "rewards/margins": 0.14318843185901642, + "rewards/rejected": -0.2759261131286621, + "step": 796 + }, + { + "epoch": 2.1960648947186745, + "grad_norm": 0.2660160958766937, + "learning_rate": 1.0240607526984437e-06, + "log_odds_chosen": 1.6831055879592896, + "log_odds_ratio": -0.18040511012077332, + "logits/chosen": -0.8147282004356384, + "logits/rejected": -1.9211832284927368, + "logps/chosen": -1.4013046026229858, + "logps/rejected": -2.8633060455322266, + "loss": 1.4634, + "nll_loss": 1.4453954696655273, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14013046026229858, + "rewards/margins": 0.14620015025138855, + "rewards/rejected": -0.28633058071136475, + "step": 797 + }, + { + "epoch": 2.198826372109078, + "grad_norm": 0.2278819978237152, + "learning_rate": 1.0175998036874357e-06, + "log_odds_chosen": 1.3868612051010132, + "log_odds_ratio": -0.22797061502933502, + "logits/chosen": -0.8481411933898926, + "logits/rejected": -1.837113380432129, + "logps/chosen": -1.3668131828308105, + "logps/rejected": -2.541306257247925, + "loss": 1.4386, + "nll_loss": 1.415839433670044, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13668133318424225, + "rewards/margins": 0.11744929850101471, + "rewards/rejected": -0.25413063168525696, + "step": 798 + }, + { + "epoch": 2.2015878494994823, + "grad_norm": 0.22600935399532318, + "learning_rate": 1.0111540886320433e-06, + "log_odds_chosen": 1.3912237882614136, + "log_odds_ratio": -0.22471024096012115, + "logits/chosen": -0.7584688663482666, + "logits/rejected": -1.8486192226409912, + "logps/chosen": -1.4408618211746216, + "logps/rejected": -2.633699893951416, + "loss": 1.5085, + "nll_loss": 1.4860626459121704, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14408619701862335, + "rewards/margins": 0.11928381770849228, + "rewards/rejected": -0.2633700370788574, + "step": 799 + }, + { + "epoch": 2.204349326889886, + "grad_norm": 0.21976998448371887, + "learning_rate": 1.00472367377196e-06, + "log_odds_chosen": 1.375922441482544, + "log_odds_ratio": -0.23655349016189575, + "logits/chosen": -0.8409130573272705, + "logits/rejected": -1.8534971475601196, + "logps/chosen": -1.428146481513977, + "logps/rejected": -2.6036124229431152, + "loss": 1.4798, + "nll_loss": 1.4561710357666016, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14281465113162994, + "rewards/margins": 0.11754661053419113, + "rewards/rejected": -0.2603612542152405, + "step": 800 + }, + { + "epoch": 2.20711080428029, + "grad_norm": 0.2096586525440216, + "learning_rate": 9.983086251896493e-07, + "log_odds_chosen": 1.4792360067367554, + "log_odds_ratio": -0.2164044976234436, + "logits/chosen": -0.8477017879486084, + "logits/rejected": -2.007514476776123, + "logps/chosen": -1.4218276739120483, + "logps/rejected": -2.697108030319214, + "loss": 1.4754, + "nll_loss": 1.453714370727539, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14218276739120483, + "rewards/margins": 0.12752802670001984, + "rewards/rejected": -0.2697107791900635, + "step": 801 + }, + { + "epoch": 2.2098722816706937, + "grad_norm": 0.31831514835357666, + "learning_rate": 9.91909008809659e-07, + "log_odds_chosen": 1.3559105396270752, + "log_odds_ratio": -0.24869945645332336, + "logits/chosen": -0.6960320472717285, + "logits/rejected": -1.9635218381881714, + "logps/chosen": -1.4762076139450073, + "logps/rejected": -2.649646043777466, + "loss": 1.5351, + "nll_loss": 1.5102181434631348, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1476207673549652, + "rewards/margins": 0.11734385043382645, + "rewards/rejected": -0.26496461033821106, + "step": 802 + }, + { + "epoch": 2.212633759061098, + "grad_norm": 0.23091106116771698, + "learning_rate": 9.855248903979505e-07, + "log_odds_chosen": 1.436722993850708, + "log_odds_ratio": -0.21449066698551178, + "logits/chosen": -0.8235598802566528, + "logits/rejected": -2.0638046264648438, + "logps/chosen": -1.463809847831726, + "logps/rejected": -2.703613519668579, + "loss": 1.5394, + "nll_loss": 1.517941951751709, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14638099074363708, + "rewards/margins": 0.12398036569356918, + "rewards/rejected": -0.27036136388778687, + "step": 803 + }, + { + "epoch": 2.2153952364515015, + "grad_norm": 0.2459157109260559, + "learning_rate": 9.791563355612171e-07, + "log_odds_chosen": 1.2237358093261719, + "log_odds_ratio": -0.2616325318813324, + "logits/chosen": -0.7013297080993652, + "logits/rejected": -1.9213218688964844, + "logps/chosen": -1.5507513284683228, + "logps/rejected": -2.6120126247406006, + "loss": 1.613, + "nll_loss": 1.586790919303894, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15507514774799347, + "rewards/margins": 0.10612612217664719, + "rewards/rejected": -0.26120126247406006, + "step": 804 + }, + { + "epoch": 2.2181567138419056, + "grad_norm": 0.2175196260213852, + "learning_rate": 9.728034097462144e-07, + "log_odds_chosen": 1.8408254384994507, + "log_odds_ratio": -0.16706471145153046, + "logits/chosen": -0.7350028157234192, + "logits/rejected": -2.162047863006592, + "logps/chosen": -1.3898506164550781, + "logps/rejected": -2.9982106685638428, + "loss": 1.4664, + "nll_loss": 1.4496983289718628, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1389850676059723, + "rewards/margins": 0.16083598136901855, + "rewards/rejected": -0.29982104897499084, + "step": 805 + }, + { + "epoch": 2.2209181912323093, + "grad_norm": 0.22365769743919373, + "learning_rate": 9.66466178239084e-07, + "log_odds_chosen": 1.4642393589019775, + "log_odds_ratio": -0.23663140833377838, + "logits/chosen": -0.7512733340263367, + "logits/rejected": -2.0906615257263184, + "logps/chosen": -1.4405597448349, + "logps/rejected": -2.709528923034668, + "loss": 1.4955, + "nll_loss": 1.4718680381774902, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14405599236488342, + "rewards/margins": 0.1268969029188156, + "rewards/rejected": -0.27095291018486023, + "step": 806 + }, + { + "epoch": 2.2236796686227134, + "grad_norm": 0.2207762897014618, + "learning_rate": 9.601447061646838e-07, + "log_odds_chosen": 1.2826145887374878, + "log_odds_ratio": -0.247898131608963, + "logits/chosen": -0.9267134666442871, + "logits/rejected": -1.7989857196807861, + "logps/chosen": -1.3823951482772827, + "logps/rejected": -2.466029644012451, + "loss": 1.4549, + "nll_loss": 1.4300906658172607, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1382395178079605, + "rewards/margins": 0.10836344957351685, + "rewards/rejected": -0.24660295248031616, + "step": 807 + }, + { + "epoch": 2.226441146013117, + "grad_norm": 0.22479695081710815, + "learning_rate": 9.538390584859213e-07, + "log_odds_chosen": 1.5794352293014526, + "log_odds_ratio": -0.1960069239139557, + "logits/chosen": -0.7440246939659119, + "logits/rejected": -1.8689384460449219, + "logps/chosen": -1.3518363237380981, + "logps/rejected": -2.699951171875, + "loss": 1.4346, + "nll_loss": 1.4149811267852783, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13518363237380981, + "rewards/margins": 0.13481149077415466, + "rewards/rejected": -0.2699951231479645, + "step": 808 + }, + { + "epoch": 2.2292026234035207, + "grad_norm": 0.2177855521440506, + "learning_rate": 9.47549300003082e-07, + "log_odds_chosen": 1.5709489583969116, + "log_odds_ratio": -0.19270090758800507, + "logits/chosen": -0.8938512802124023, + "logits/rejected": -1.9860432147979736, + "logps/chosen": -1.3834973573684692, + "logps/rejected": -2.7306644916534424, + "loss": 1.4517, + "nll_loss": 1.432417869567871, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1383497416973114, + "rewards/margins": 0.1347167193889618, + "rewards/rejected": -0.2730664610862732, + "step": 809 + }, + { + "epoch": 2.231964100793925, + "grad_norm": 0.24030561745166779, + "learning_rate": 9.412754953531664e-07, + "log_odds_chosen": 1.81654691696167, + "log_odds_ratio": -0.1660957783460617, + "logits/chosen": -0.7898241877555847, + "logits/rejected": -2.001150369644165, + "logps/chosen": -1.365299940109253, + "logps/rejected": -2.937218189239502, + "loss": 1.4319, + "nll_loss": 1.4152500629425049, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13652999699115753, + "rewards/margins": 0.15719182789325714, + "rewards/rejected": -0.2937218248844147, + "step": 810 + }, + { + "epoch": 2.2347255781843285, + "grad_norm": 0.21561409533023834, + "learning_rate": 9.350177090092233e-07, + "log_odds_chosen": 1.6017990112304688, + "log_odds_ratio": -0.19290661811828613, + "logits/chosen": -0.663612425327301, + "logits/rejected": -2.0358691215515137, + "logps/chosen": -1.4089775085449219, + "logps/rejected": -2.790071487426758, + "loss": 1.4723, + "nll_loss": 1.453056812286377, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1408977508544922, + "rewards/margins": 0.13810938596725464, + "rewards/rejected": -0.2790071368217468, + "step": 811 + }, + { + "epoch": 2.2374870555747326, + "grad_norm": 0.22939537465572357, + "learning_rate": 9.28776005279691e-07, + "log_odds_chosen": 1.6415590047836304, + "log_odds_ratio": -0.18874233961105347, + "logits/chosen": -0.7984322905540466, + "logits/rejected": -2.1387336254119873, + "logps/chosen": -1.4797344207763672, + "logps/rejected": -2.917440414428711, + "loss": 1.5297, + "nll_loss": 1.5108166933059692, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1479734182357788, + "rewards/margins": 0.14377063512802124, + "rewards/rejected": -0.29174405336380005, + "step": 812 + }, + { + "epoch": 2.2402485329651363, + "grad_norm": 0.21815143525600433, + "learning_rate": 9.225504483077311e-07, + "log_odds_chosen": 1.6695032119750977, + "log_odds_ratio": -0.1855509877204895, + "logits/chosen": -0.8288291692733765, + "logits/rejected": -2.1964502334594727, + "logps/chosen": -1.3856902122497559, + "logps/rejected": -2.8318369388580322, + "loss": 1.4392, + "nll_loss": 1.420630931854248, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13856902718544006, + "rewards/margins": 0.14461466670036316, + "rewards/rejected": -0.28318366408348083, + "step": 813 + }, + { + "epoch": 2.2430100103555404, + "grad_norm": 0.2161535620689392, + "learning_rate": 9.163411020705762e-07, + "log_odds_chosen": 1.3703440427780151, + "log_odds_ratio": -0.24051517248153687, + "logits/chosen": -0.7353973984718323, + "logits/rejected": -2.010030746459961, + "logps/chosen": -1.3800489902496338, + "logps/rejected": -2.546851873397827, + "loss": 1.4496, + "nll_loss": 1.4255536794662476, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13800489902496338, + "rewards/margins": 0.11668029427528381, + "rewards/rejected": -0.2546851933002472, + "step": 814 + }, + { + "epoch": 2.245771487745944, + "grad_norm": 0.22893516719341278, + "learning_rate": 9.101480303788623e-07, + "log_odds_chosen": 1.2863377332687378, + "log_odds_ratio": -0.24850118160247803, + "logits/chosen": -0.7366392016410828, + "logits/rejected": -1.659240484237671, + "logps/chosen": -1.3816016912460327, + "logps/rejected": -2.464106798171997, + "loss": 1.457, + "nll_loss": 1.4321224689483643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13816016912460327, + "rewards/margins": 0.10825051367282867, + "rewards/rejected": -0.24641066789627075, + "step": 815 + }, + { + "epoch": 2.248532965136348, + "grad_norm": 0.21422047913074493, + "learning_rate": 9.039712968759864e-07, + "log_odds_chosen": 1.591874361038208, + "log_odds_ratio": -0.19570983946323395, + "logits/chosen": -0.7051729559898376, + "logits/rejected": -2.0805304050445557, + "logps/chosen": -1.4287737607955933, + "logps/rejected": -2.8088440895080566, + "loss": 1.5089, + "nll_loss": 1.4893687963485718, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14287738502025604, + "rewards/margins": 0.1380070298910141, + "rewards/rejected": -0.28088441491127014, + "step": 816 + }, + { + "epoch": 2.251294442526752, + "grad_norm": 0.20570261776447296, + "learning_rate": 8.978109650374398e-07, + "log_odds_chosen": 1.599745273590088, + "log_odds_ratio": -0.1869451403617859, + "logits/chosen": -0.7604836821556091, + "logits/rejected": -2.0949907302856445, + "logps/chosen": -1.4347954988479614, + "logps/rejected": -2.8207342624664307, + "loss": 1.5008, + "nll_loss": 1.482066035270691, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14347955584526062, + "rewards/margins": 0.1385938823223114, + "rewards/rejected": -0.28207340836524963, + "step": 817 + }, + { + "epoch": 2.254055919917156, + "grad_norm": 0.24999411404132843, + "learning_rate": 8.916670981701656e-07, + "log_odds_chosen": 1.4131107330322266, + "log_odds_ratio": -0.22106236219406128, + "logits/chosen": -0.794218122959137, + "logits/rejected": -2.1165714263916016, + "logps/chosen": -1.517124056816101, + "logps/rejected": -2.7476296424865723, + "loss": 1.5648, + "nll_loss": 1.5427284240722656, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15171240270137787, + "rewards/margins": 0.12305056303739548, + "rewards/rejected": -0.27476298809051514, + "step": 818 + }, + { + "epoch": 2.2568173973075596, + "grad_norm": 0.21286356449127197, + "learning_rate": 8.855397594118995e-07, + "log_odds_chosen": 1.3657292127609253, + "log_odds_ratio": -0.24322526156902313, + "logits/chosen": -0.7253862619400024, + "logits/rejected": -1.8961949348449707, + "logps/chosen": -1.3823702335357666, + "logps/rejected": -2.545137882232666, + "loss": 1.4589, + "nll_loss": 1.4345393180847168, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13823701441287994, + "rewards/margins": 0.11627674847841263, + "rewards/rejected": -0.25451380014419556, + "step": 819 + }, + { + "epoch": 2.2595788746979633, + "grad_norm": 0.21439580619335175, + "learning_rate": 8.794290117305296e-07, + "log_odds_chosen": 1.5430362224578857, + "log_odds_ratio": -0.19931325316429138, + "logits/chosen": -0.8306599259376526, + "logits/rejected": -2.151496171951294, + "logps/chosen": -1.3482749462127686, + "logps/rejected": -2.660677433013916, + "loss": 1.4238, + "nll_loss": 1.40384042263031, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13482749462127686, + "rewards/margins": 0.13124024868011475, + "rewards/rejected": -0.2660677433013916, + "step": 820 + }, + { + "epoch": 2.2623403520883674, + "grad_norm": 0.21566608548164368, + "learning_rate": 8.733349179234412e-07, + "log_odds_chosen": 1.4808435440063477, + "log_odds_ratio": -0.22421996295452118, + "logits/chosen": -0.8796311616897583, + "logits/rejected": -1.8572511672973633, + "logps/chosen": -1.3629071712493896, + "logps/rejected": -2.6273839473724365, + "loss": 1.4427, + "nll_loss": 1.4202845096588135, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13629071414470673, + "rewards/margins": 0.12644769251346588, + "rewards/rejected": -0.2627384066581726, + "step": 821 + }, + { + "epoch": 2.265101829478771, + "grad_norm": 0.2407940924167633, + "learning_rate": 8.672575406168782e-07, + "log_odds_chosen": 1.3748390674591064, + "log_odds_ratio": -0.2400941550731659, + "logits/chosen": -0.7804903984069824, + "logits/rejected": -1.6939703226089478, + "logps/chosen": -1.4525799751281738, + "logps/rejected": -2.6338768005371094, + "loss": 1.5231, + "nll_loss": 1.4990655183792114, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14525800943374634, + "rewards/margins": 0.1181296706199646, + "rewards/rejected": -0.26338768005371094, + "step": 822 + }, + { + "epoch": 2.267863306869175, + "grad_norm": 0.21523721516132355, + "learning_rate": 8.611969422652966e-07, + "log_odds_chosen": 1.5803931951522827, + "log_odds_ratio": -0.20858606696128845, + "logits/chosen": -0.7921524047851562, + "logits/rejected": -1.8999464511871338, + "logps/chosen": -1.3710713386535645, + "logps/rejected": -2.7342100143432617, + "loss": 1.4605, + "nll_loss": 1.4396417140960693, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13710711896419525, + "rewards/margins": 0.13631387054920197, + "rewards/rejected": -0.2734209895133972, + "step": 823 + }, + { + "epoch": 2.270624784259579, + "grad_norm": 0.21190094947814941, + "learning_rate": 8.551531851507186e-07, + "log_odds_chosen": 1.431606411933899, + "log_odds_ratio": -0.22368153929710388, + "logits/chosen": -0.7815225720405579, + "logits/rejected": -1.8217527866363525, + "logps/chosen": -1.3981519937515259, + "logps/rejected": -2.6237053871154785, + "loss": 1.4763, + "nll_loss": 1.4538975954055786, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13981519639492035, + "rewards/margins": 0.12255533784627914, + "rewards/rejected": -0.2623705565929413, + "step": 824 + }, + { + "epoch": 2.273386261649983, + "grad_norm": 0.2221967726945877, + "learning_rate": 8.491263313821021e-07, + "log_odds_chosen": 1.438922643661499, + "log_odds_ratio": -0.22679126262664795, + "logits/chosen": -0.8472309112548828, + "logits/rejected": -1.8394192457199097, + "logps/chosen": -1.4349843263626099, + "logps/rejected": -2.6651053428649902, + "loss": 1.507, + "nll_loss": 1.4843312501907349, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14349845051765442, + "rewards/margins": 0.12301210314035416, + "rewards/rejected": -0.266510546207428, + "step": 825 + }, + { + "epoch": 2.2761477390403866, + "grad_norm": 0.27319595217704773, + "learning_rate": 8.431164428946928e-07, + "log_odds_chosen": 1.5053706169128418, + "log_odds_ratio": -0.21574383974075317, + "logits/chosen": -0.9391317367553711, + "logits/rejected": -1.8469014167785645, + "logps/chosen": -1.4213306903839111, + "logps/rejected": -2.715855121612549, + "loss": 1.4929, + "nll_loss": 1.4712984561920166, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14213307201862335, + "rewards/margins": 0.12945246696472168, + "rewards/rejected": -0.2715855538845062, + "step": 826 + }, + { + "epoch": 2.2789092164307903, + "grad_norm": 0.2336154580116272, + "learning_rate": 8.371235814493947e-07, + "log_odds_chosen": 1.6580530405044556, + "log_odds_ratio": -0.1864277720451355, + "logits/chosen": -0.8675610423088074, + "logits/rejected": -2.160818338394165, + "logps/chosen": -1.4152588844299316, + "logps/rejected": -2.8566691875457764, + "loss": 1.4831, + "nll_loss": 1.4644639492034912, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14152587950229645, + "rewards/margins": 0.14414101839065552, + "rewards/rejected": -0.28566691279411316, + "step": 827 + }, + { + "epoch": 2.2816706938211944, + "grad_norm": 0.21498602628707886, + "learning_rate": 8.311478086321301e-07, + "log_odds_chosen": 1.4063800573349, + "log_odds_ratio": -0.22114284336566925, + "logits/chosen": -0.7863494753837585, + "logits/rejected": -1.8558886051177979, + "logps/chosen": -1.3713574409484863, + "logps/rejected": -2.564828395843506, + "loss": 1.4365, + "nll_loss": 1.4143601655960083, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13713574409484863, + "rewards/margins": 0.11934711039066315, + "rewards/rejected": -0.2564828395843506, + "step": 828 + }, + { + "epoch": 2.284432171211598, + "grad_norm": 0.20723487436771393, + "learning_rate": 8.251891858532124e-07, + "log_odds_chosen": 1.6476480960845947, + "log_odds_ratio": -0.1784750372171402, + "logits/chosen": -0.8183894753456116, + "logits/rejected": -2.0899806022644043, + "logps/chosen": -1.3663532733917236, + "logps/rejected": -2.7823915481567383, + "loss": 1.4319, + "nll_loss": 1.4140135049819946, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13663533329963684, + "rewards/margins": 0.14160385727882385, + "rewards/rejected": -0.2782391905784607, + "step": 829 + }, + { + "epoch": 2.287193648602002, + "grad_norm": 0.2511179447174072, + "learning_rate": 8.192477743467078e-07, + "log_odds_chosen": 1.433205485343933, + "log_odds_ratio": -0.2308994084596634, + "logits/chosen": -0.9073160886764526, + "logits/rejected": -1.5418671369552612, + "logps/chosen": -1.4494136571884155, + "logps/rejected": -2.689509630203247, + "loss": 1.5321, + "nll_loss": 1.5090384483337402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14494135975837708, + "rewards/margins": 0.12400959432125092, + "rewards/rejected": -0.2689509689807892, + "step": 830 + }, + { + "epoch": 2.289955125992406, + "grad_norm": 0.22408699989318848, + "learning_rate": 8.133236351698143e-07, + "log_odds_chosen": 1.4096298217773438, + "log_odds_ratio": -0.22869807481765747, + "logits/chosen": -0.7961119413375854, + "logits/rejected": -1.8842319250106812, + "logps/chosen": -1.4202234745025635, + "logps/rejected": -2.6291675567626953, + "loss": 1.4995, + "nll_loss": 1.4766323566436768, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14202234148979187, + "rewards/margins": 0.1208944171667099, + "rewards/rejected": -0.26291677355766296, + "step": 831 + }, + { + "epoch": 2.29271660338281, + "grad_norm": 0.24003839492797852, + "learning_rate": 8.074168292022269e-07, + "log_odds_chosen": 1.4157015085220337, + "log_odds_ratio": -0.2238752245903015, + "logits/chosen": -0.7954575419425964, + "logits/rejected": -1.9194411039352417, + "logps/chosen": -1.4735336303710938, + "logps/rejected": -2.6962783336639404, + "loss": 1.541, + "nll_loss": 1.5186426639556885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1473533660173416, + "rewards/margins": 0.1222744733095169, + "rewards/rejected": -0.2696278393268585, + "step": 832 + }, + { + "epoch": 2.2954780807732136, + "grad_norm": 0.22047549486160278, + "learning_rate": 8.015274171455151e-07, + "log_odds_chosen": 1.5765596628189087, + "log_odds_ratio": -0.1994118094444275, + "logits/chosen": -0.7459832429885864, + "logits/rejected": -2.163300037384033, + "logps/chosen": -1.4016599655151367, + "logps/rejected": -2.763180732727051, + "loss": 1.4583, + "nll_loss": 1.4383318424224854, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1401660144329071, + "rewards/margins": 0.13615208864212036, + "rewards/rejected": -0.27631810307502747, + "step": 833 + }, + { + "epoch": 2.2982395581636177, + "grad_norm": 0.21983671188354492, + "learning_rate": 7.956554595225017e-07, + "log_odds_chosen": 1.4907373189926147, + "log_odds_ratio": -0.211158886551857, + "logits/chosen": -0.962412416934967, + "logits/rejected": -1.9069632291793823, + "logps/chosen": -1.3121854066848755, + "logps/rejected": -2.571423292160034, + "loss": 1.3831, + "nll_loss": 1.3619731664657593, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1312185525894165, + "rewards/margins": 0.1259237825870514, + "rewards/rejected": -0.2571423351764679, + "step": 834 + }, + { + "epoch": 2.3010010355540214, + "grad_norm": 0.2331991195678711, + "learning_rate": 7.898010166766348e-07, + "log_odds_chosen": 1.552021861076355, + "log_odds_ratio": -0.20787937939167023, + "logits/chosen": -0.8280553221702576, + "logits/rejected": -1.8152735233306885, + "logps/chosen": -1.4129295349121094, + "logps/rejected": -2.754270315170288, + "loss": 1.483, + "nll_loss": 1.4621742963790894, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1412929743528366, + "rewards/margins": 0.13413408398628235, + "rewards/rejected": -0.27542704343795776, + "step": 835 + }, + { + "epoch": 2.3037625129444255, + "grad_norm": 0.219749316573143, + "learning_rate": 7.839641487713745e-07, + "log_odds_chosen": 1.4371466636657715, + "log_odds_ratio": -0.22468623518943787, + "logits/chosen": -0.820203423500061, + "logits/rejected": -1.8462026119232178, + "logps/chosen": -1.366627812385559, + "logps/rejected": -2.589350461959839, + "loss": 1.4294, + "nll_loss": 1.4069098234176636, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1366627812385559, + "rewards/margins": 0.12227226793766022, + "rewards/rejected": -0.25893503427505493, + "step": 836 + }, + { + "epoch": 2.306523990334829, + "grad_norm": 0.23122566938400269, + "learning_rate": 7.781449157895677e-07, + "log_odds_chosen": 1.547440767288208, + "log_odds_ratio": -0.1968943178653717, + "logits/chosen": -0.9170833826065063, + "logits/rejected": -2.0454964637756348, + "logps/chosen": -1.455304741859436, + "logps/rejected": -2.797694683074951, + "loss": 1.5188, + "nll_loss": 1.4990651607513428, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14553046226501465, + "rewards/margins": 0.13423900306224823, + "rewards/rejected": -0.2797694504261017, + "step": 837 + }, + { + "epoch": 2.309285467725233, + "grad_norm": 0.2177518755197525, + "learning_rate": 7.723433775328385e-07, + "log_odds_chosen": 2.1011574268341064, + "log_odds_ratio": -0.13231369853019714, + "logits/chosen": -0.8008699417114258, + "logits/rejected": -2.206787586212158, + "logps/chosen": -1.3365592956542969, + "logps/rejected": -3.181867837905884, + "loss": 1.4069, + "nll_loss": 1.3936563730239868, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13365595042705536, + "rewards/margins": 0.1845308095216751, + "rewards/rejected": -0.31818675994873047, + "step": 838 + }, + { + "epoch": 2.312046945115637, + "grad_norm": 0.24736399948596954, + "learning_rate": 7.665595936209674e-07, + "log_odds_chosen": 1.336047887802124, + "log_odds_ratio": -0.23815786838531494, + "logits/chosen": -0.7619213461875916, + "logits/rejected": -1.9846187829971313, + "logps/chosen": -1.5085265636444092, + "logps/rejected": -2.6583876609802246, + "loss": 1.5721, + "nll_loss": 1.5482698678970337, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15085266530513763, + "rewards/margins": 0.1149861216545105, + "rewards/rejected": -0.2658388018608093, + "step": 839 + }, + { + "epoch": 2.3148084225060406, + "grad_norm": 0.21641425788402557, + "learning_rate": 7.607936234912841e-07, + "log_odds_chosen": 1.5291671752929688, + "log_odds_ratio": -0.2096787989139557, + "logits/chosen": -0.8673451542854309, + "logits/rejected": -1.9320155382156372, + "logps/chosen": -1.3398340940475464, + "logps/rejected": -2.6366426944732666, + "loss": 1.4023, + "nll_loss": 1.3813666105270386, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13398341834545135, + "rewards/margins": 0.12968085706233978, + "rewards/rejected": -0.26366427540779114, + "step": 840 + }, + { + "epoch": 2.3175698998964447, + "grad_norm": 0.22011668980121613, + "learning_rate": 7.550455263980525e-07, + "log_odds_chosen": 1.4579017162322998, + "log_odds_ratio": -0.21185196936130524, + "logits/chosen": -0.8572349548339844, + "logits/rejected": -2.1599864959716797, + "logps/chosen": -1.4740046262741089, + "logps/rejected": -2.7385215759277344, + "loss": 1.5327, + "nll_loss": 1.5115079879760742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14740046858787537, + "rewards/margins": 0.12645168602466583, + "rewards/rejected": -0.2738521695137024, + "step": 841 + }, + { + "epoch": 2.3203313772868484, + "grad_norm": 0.22545059025287628, + "learning_rate": 7.493153614118634e-07, + "log_odds_chosen": 1.755352258682251, + "log_odds_ratio": -0.16482490301132202, + "logits/chosen": -0.8975868225097656, + "logits/rejected": -1.90993332862854, + "logps/chosen": -1.3448066711425781, + "logps/rejected": -2.853928565979004, + "loss": 1.427, + "nll_loss": 1.4105489253997803, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13448068499565125, + "rewards/margins": 0.15091221034526825, + "rewards/rejected": -0.2853928804397583, + "step": 842 + }, + { + "epoch": 2.3230928546772525, + "grad_norm": 0.22612684965133667, + "learning_rate": 7.436031874190272e-07, + "log_odds_chosen": 1.5865403413772583, + "log_odds_ratio": -0.19421318173408508, + "logits/chosen": -0.7267550826072693, + "logits/rejected": -2.0342113971710205, + "logps/chosen": -1.4796632528305054, + "logps/rejected": -2.8678059577941895, + "loss": 1.5443, + "nll_loss": 1.5248392820358276, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14796634018421173, + "rewards/margins": 0.13881425559520721, + "rewards/rejected": -0.28678056597709656, + "step": 843 + }, + { + "epoch": 2.325854332067656, + "grad_norm": 0.2235104888677597, + "learning_rate": 7.379090631209712e-07, + "log_odds_chosen": 1.61055326461792, + "log_odds_ratio": -0.19099710881710052, + "logits/chosen": -0.8531662821769714, + "logits/rejected": -2.0457820892333984, + "logps/chosen": -1.3573286533355713, + "logps/rejected": -2.737484931945801, + "loss": 1.4251, + "nll_loss": 1.4059858322143555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13573288917541504, + "rewards/margins": 0.13801559805870056, + "rewards/rejected": -0.2737484872341156, + "step": 844 + }, + { + "epoch": 2.3286158094580602, + "grad_norm": 0.21737630665302277, + "learning_rate": 7.322330470336314e-07, + "log_odds_chosen": 1.5560356378555298, + "log_odds_ratio": -0.19873520731925964, + "logits/chosen": -0.8432495594024658, + "logits/rejected": -1.9900035858154297, + "logps/chosen": -1.3147186040878296, + "logps/rejected": -2.630089282989502, + "loss": 1.391, + "nll_loss": 1.371086835861206, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13147185742855072, + "rewards/margins": 0.1315370798110962, + "rewards/rejected": -0.2630089521408081, + "step": 845 + }, + { + "epoch": 2.331377286848464, + "grad_norm": 0.22583864629268646, + "learning_rate": 7.265751974868554e-07, + "log_odds_chosen": 1.4394731521606445, + "log_odds_ratio": -0.22187888622283936, + "logits/chosen": -0.8381003141403198, + "logits/rejected": -1.7541844844818115, + "logps/chosen": -1.3545819520950317, + "logps/rejected": -2.5769851207733154, + "loss": 1.4338, + "nll_loss": 1.411636471748352, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13545820116996765, + "rewards/margins": 0.12224029004573822, + "rewards/rejected": -0.25769850611686707, + "step": 846 + }, + { + "epoch": 2.334138764238868, + "grad_norm": 0.23510843515396118, + "learning_rate": 7.209355726238024e-07, + "log_odds_chosen": 1.6167476177215576, + "log_odds_ratio": -0.20064175128936768, + "logits/chosen": -0.8486358523368835, + "logits/rejected": -2.045172929763794, + "logps/chosen": -1.4668694734573364, + "logps/rejected": -2.884349822998047, + "loss": 1.5193, + "nll_loss": 1.4991921186447144, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14668695628643036, + "rewards/margins": 0.14174804091453552, + "rewards/rejected": -0.2884350121021271, + "step": 847 + }, + { + "epoch": 2.3369002416292717, + "grad_norm": 0.2219594568014145, + "learning_rate": 7.153142304003419e-07, + "log_odds_chosen": 1.559194564819336, + "log_odds_ratio": -0.20855563879013062, + "logits/chosen": -0.820976734161377, + "logits/rejected": -1.947663426399231, + "logps/chosen": -1.4534995555877686, + "logps/rejected": -2.8155107498168945, + "loss": 1.5203, + "nll_loss": 1.4994096755981445, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14534996449947357, + "rewards/margins": 0.1362011432647705, + "rewards/rejected": -0.2815510928630829, + "step": 848 + }, + { + "epoch": 2.3396617190196753, + "grad_norm": 0.22509777545928955, + "learning_rate": 7.097112285844643e-07, + "log_odds_chosen": 1.5197633504867554, + "log_odds_ratio": -0.2113376408815384, + "logits/chosen": -0.8353469967842102, + "logits/rejected": -1.7747441530227661, + "logps/chosen": -1.3918851613998413, + "logps/rejected": -2.6959621906280518, + "loss": 1.4579, + "nll_loss": 1.4367426633834839, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1391885131597519, + "rewards/margins": 0.13040770590305328, + "rewards/rejected": -0.2695962190628052, + "step": 849 + }, + { + "epoch": 2.3424231964100795, + "grad_norm": 0.22396017611026764, + "learning_rate": 7.041266247556814e-07, + "log_odds_chosen": 1.5306124687194824, + "log_odds_ratio": -0.2112870216369629, + "logits/chosen": -0.731094479560852, + "logits/rejected": -1.919350266456604, + "logps/chosen": -1.4003535509109497, + "logps/rejected": -2.7186193466186523, + "loss": 1.4648, + "nll_loss": 1.4436579942703247, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14003536105155945, + "rewards/margins": 0.13182657957077026, + "rewards/rejected": -0.2718619406223297, + "step": 850 + }, + { + "epoch": 2.345184673800483, + "grad_norm": 0.22262442111968994, + "learning_rate": 6.985604763044382e-07, + "log_odds_chosen": 1.577681303024292, + "log_odds_ratio": -0.20103678107261658, + "logits/chosen": -0.8643767237663269, + "logits/rejected": -1.923162579536438, + "logps/chosen": -1.3931130170822144, + "logps/rejected": -2.7541937828063965, + "loss": 1.4549, + "nll_loss": 1.4347765445709229, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1393112987279892, + "rewards/margins": 0.13610810041427612, + "rewards/rejected": -0.2754194140434265, + "step": 851 + }, + { + "epoch": 2.3479461511908872, + "grad_norm": 0.22737272083759308, + "learning_rate": 6.930128404315214e-07, + "log_odds_chosen": 1.5616035461425781, + "log_odds_ratio": -0.20092810690402985, + "logits/chosen": -0.875281035900116, + "logits/rejected": -1.9816867113113403, + "logps/chosen": -1.4429211616516113, + "logps/rejected": -2.799321174621582, + "loss": 1.5083, + "nll_loss": 1.488168478012085, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14429211616516113, + "rewards/margins": 0.1356399655342102, + "rewards/rejected": -0.27993208169937134, + "step": 852 + }, + { + "epoch": 2.350707628581291, + "grad_norm": 0.21603858470916748, + "learning_rate": 6.874837741474744e-07, + "log_odds_chosen": 1.561700701713562, + "log_odds_ratio": -0.2068319022655487, + "logits/chosen": -0.8764750957489014, + "logits/rejected": -1.9299256801605225, + "logps/chosen": -1.362099289894104, + "logps/rejected": -2.70504093170166, + "loss": 1.4298, + "nll_loss": 1.4091013669967651, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13620993494987488, + "rewards/margins": 0.13429416716098785, + "rewards/rejected": -0.2705041170120239, + "step": 853 + }, + { + "epoch": 2.353469105971695, + "grad_norm": 0.2454364150762558, + "learning_rate": 6.819733342720067e-07, + "log_odds_chosen": 1.6186573505401611, + "log_odds_ratio": -0.19199815392494202, + "logits/chosen": -0.9424867630004883, + "logits/rejected": -1.8045995235443115, + "logps/chosen": -1.4089401960372925, + "logps/rejected": -2.8112776279449463, + "loss": 1.4579, + "nll_loss": 1.4386602640151978, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14089402556419373, + "rewards/margins": 0.14023372530937195, + "rewards/rejected": -0.2811277508735657, + "step": 854 + }, + { + "epoch": 2.3562305833620987, + "grad_norm": 0.24490110576152802, + "learning_rate": 6.764815774334149e-07, + "log_odds_chosen": 1.357130527496338, + "log_odds_ratio": -0.23701991140842438, + "logits/chosen": -0.857049822807312, + "logits/rejected": -2.015214443206787, + "logps/chosen": -1.4582642316818237, + "logps/rejected": -2.625171422958374, + "loss": 1.5179, + "nll_loss": 1.4941763877868652, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14582642912864685, + "rewards/margins": 0.11669071018695831, + "rewards/rejected": -0.26251715421676636, + "step": 855 + }, + { + "epoch": 2.3589920607525023, + "grad_norm": 0.22016265988349915, + "learning_rate": 6.710085600679967e-07, + "log_odds_chosen": 1.4929471015930176, + "log_odds_ratio": -0.21420983970165253, + "logits/chosen": -0.7236154675483704, + "logits/rejected": -1.8628549575805664, + "logps/chosen": -1.3636245727539062, + "logps/rejected": -2.6389825344085693, + "loss": 1.4373, + "nll_loss": 1.4158825874328613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1363624632358551, + "rewards/margins": 0.12753577530384064, + "rewards/rejected": -0.26389825344085693, + "step": 856 + }, + { + "epoch": 2.3617535381429065, + "grad_norm": 0.24124066531658173, + "learning_rate": 6.65554338419474e-07, + "log_odds_chosen": 1.5906827449798584, + "log_odds_ratio": -0.2055184245109558, + "logits/chosen": -0.8457351922988892, + "logits/rejected": -1.848963737487793, + "logps/chosen": -1.486085057258606, + "logps/rejected": -2.886665105819702, + "loss": 1.5418, + "nll_loss": 1.5212182998657227, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1486085057258606, + "rewards/margins": 0.1400580108165741, + "rewards/rejected": -0.2886665165424347, + "step": 857 + }, + { + "epoch": 2.3645150155333106, + "grad_norm": 0.22789664566516876, + "learning_rate": 6.601189685384127e-07, + "log_odds_chosen": 1.4584548473358154, + "log_odds_ratio": -0.21383509039878845, + "logits/chosen": -0.8199442625045776, + "logits/rejected": -1.9394491910934448, + "logps/chosen": -1.4257709980010986, + "logps/rejected": -2.6794774532318115, + "loss": 1.4848, + "nll_loss": 1.4634119272232056, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1425771266222, + "rewards/margins": 0.12537063658237457, + "rewards/rejected": -0.2679477632045746, + "step": 858 + }, + { + "epoch": 2.3672764929237142, + "grad_norm": 0.2334202527999878, + "learning_rate": 6.547025062816487e-07, + "log_odds_chosen": 1.7176035642623901, + "log_odds_ratio": -0.17412160336971283, + "logits/chosen": -0.8760895133018494, + "logits/rejected": -1.910693645477295, + "logps/chosen": -1.4110983610153198, + "logps/rejected": -2.9046947956085205, + "loss": 1.4698, + "nll_loss": 1.4523557424545288, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14110985398292542, + "rewards/margins": 0.14935962855815887, + "rewards/rejected": -0.2904694974422455, + "step": 859 + }, + { + "epoch": 2.370037970314118, + "grad_norm": 0.2590780556201935, + "learning_rate": 6.493050073117115e-07, + "log_odds_chosen": 1.4820945262908936, + "log_odds_ratio": -0.21282009780406952, + "logits/chosen": -0.8368885517120361, + "logits/rejected": -1.9160068035125732, + "logps/chosen": -1.379476547241211, + "logps/rejected": -2.642612934112549, + "loss": 1.4554, + "nll_loss": 1.4341613054275513, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13794764876365662, + "rewards/margins": 0.12631365656852722, + "rewards/rejected": -0.26426130533218384, + "step": 860 + }, + { + "epoch": 2.372799447704522, + "grad_norm": 0.2302803099155426, + "learning_rate": 6.439265270962538e-07, + "log_odds_chosen": 1.5066372156143188, + "log_odds_ratio": -0.21063902974128723, + "logits/chosen": -0.7722992300987244, + "logits/rejected": -1.9620102643966675, + "logps/chosen": -1.336499810218811, + "logps/rejected": -2.615638494491577, + "loss": 1.4171, + "nll_loss": 1.3959881067276, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13364997506141663, + "rewards/margins": 0.12791387736797333, + "rewards/rejected": -0.26156383752822876, + "step": 861 + }, + { + "epoch": 2.3755609250949257, + "grad_norm": 0.20692874491214752, + "learning_rate": 6.385671209074829e-07, + "log_odds_chosen": 1.7060338258743286, + "log_odds_ratio": -0.17033779621124268, + "logits/chosen": -0.7261908650398254, + "logits/rejected": -2.0713276863098145, + "logps/chosen": -1.3343982696533203, + "logps/rejected": -2.7912662029266357, + "loss": 1.3985, + "nll_loss": 1.3814215660095215, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1334398239850998, + "rewards/margins": 0.1456868201494217, + "rewards/rejected": -0.2791266441345215, + "step": 862 + }, + { + "epoch": 2.37832240248533, + "grad_norm": 0.2275240123271942, + "learning_rate": 6.332268438215878e-07, + "log_odds_chosen": 1.7242450714111328, + "log_odds_ratio": -0.1701982617378235, + "logits/chosen": -0.8238985538482666, + "logits/rejected": -2.1816961765289307, + "logps/chosen": -1.4347082376480103, + "logps/rejected": -2.9419825077056885, + "loss": 1.4911, + "nll_loss": 1.4741240739822388, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14347082376480103, + "rewards/margins": 0.15072746574878693, + "rewards/rejected": -0.29419827461242676, + "step": 863 + }, + { + "epoch": 2.3810838798757334, + "grad_norm": 0.25345364212989807, + "learning_rate": 6.279057507181796e-07, + "log_odds_chosen": 1.7354642152786255, + "log_odds_ratio": -0.16710469126701355, + "logits/chosen": -0.8893179893493652, + "logits/rejected": -1.9863471984863281, + "logps/chosen": -1.4171266555786133, + "logps/rejected": -2.9303369522094727, + "loss": 1.4731, + "nll_loss": 1.4564212560653687, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14171266555786133, + "rewards/margins": 0.15132102370262146, + "rewards/rejected": -0.2930336892604828, + "step": 864 + }, + { + "epoch": 2.3838453572661376, + "grad_norm": 0.2283063381910324, + "learning_rate": 6.226038962797218e-07, + "log_odds_chosen": 1.4934532642364502, + "log_odds_ratio": -0.2183188498020172, + "logits/chosen": -0.8219193816184998, + "logits/rejected": -1.9478013515472412, + "logps/chosen": -1.3244479894638062, + "logps/rejected": -2.5922112464904785, + "loss": 1.4077, + "nll_loss": 1.3858716487884521, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13244479894638062, + "rewards/margins": 0.1267762929201126, + "rewards/rejected": -0.2592211067676544, + "step": 865 + }, + { + "epoch": 2.386606834656541, + "grad_norm": 0.2298060655593872, + "learning_rate": 6.17321334990973e-07, + "log_odds_chosen": 1.481666088104248, + "log_odds_ratio": -0.20736965537071228, + "logits/chosen": -0.748926043510437, + "logits/rejected": -1.9716696739196777, + "logps/chosen": -1.4211902618408203, + "logps/rejected": -2.6946358680725098, + "loss": 1.481, + "nll_loss": 1.4602220058441162, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14211902022361755, + "rewards/margins": 0.12734454870224, + "rewards/rejected": -0.26946359872817993, + "step": 866 + }, + { + "epoch": 2.389368312046945, + "grad_norm": 0.21714286506175995, + "learning_rate": 6.120581211384222e-07, + "log_odds_chosen": 1.6732999086380005, + "log_odds_ratio": -0.2173757553100586, + "logits/chosen": -0.7669368386268616, + "logits/rejected": -1.9981590509414673, + "logps/chosen": -1.3830281496047974, + "logps/rejected": -2.8426358699798584, + "loss": 1.4629, + "nll_loss": 1.4411296844482422, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13830281794071198, + "rewards/margins": 0.14596077799797058, + "rewards/rejected": -0.28426361083984375, + "step": 867 + }, + { + "epoch": 2.392129789437349, + "grad_norm": 0.23840640485286713, + "learning_rate": 6.068143088097372e-07, + "log_odds_chosen": 1.6300663948059082, + "log_odds_ratio": -0.1832505315542221, + "logits/chosen": -0.846603512763977, + "logits/rejected": -1.958760142326355, + "logps/chosen": -1.360883116722107, + "logps/rejected": -2.7600831985473633, + "loss": 1.4466, + "nll_loss": 1.4282522201538086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1360883116722107, + "rewards/margins": 0.13992001116275787, + "rewards/rejected": -0.27600833773612976, + "step": 868 + }, + { + "epoch": 2.3948912668277527, + "grad_norm": 0.21919068694114685, + "learning_rate": 6.015899518932005e-07, + "log_odds_chosen": 1.7166774272918701, + "log_odds_ratio": -0.17428842186927795, + "logits/chosen": -0.8554909825325012, + "logits/rejected": -1.9679923057556152, + "logps/chosen": -1.366170048713684, + "logps/rejected": -2.845726490020752, + "loss": 1.4217, + "nll_loss": 1.404255986213684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1366170048713684, + "rewards/margins": 0.14795567095279694, + "rewards/rejected": -0.28457266092300415, + "step": 869 + }, + { + "epoch": 2.3976527442181568, + "grad_norm": 0.21695028245449066, + "learning_rate": 5.963851040771639e-07, + "log_odds_chosen": 1.4922925233840942, + "log_odds_ratio": -0.22315824031829834, + "logits/chosen": -0.7975160479545593, + "logits/rejected": -1.9359625577926636, + "logps/chosen": -1.3698251247406006, + "logps/rejected": -2.6484439373016357, + "loss": 1.4442, + "nll_loss": 1.4218785762786865, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1369825154542923, + "rewards/margins": 0.1278618574142456, + "rewards/rejected": -0.2648443579673767, + "step": 870 + }, + { + "epoch": 2.4004142216085604, + "grad_norm": 0.2348206788301468, + "learning_rate": 5.911998188494924e-07, + "log_odds_chosen": 1.6946475505828857, + "log_odds_ratio": -0.17974430322647095, + "logits/chosen": -0.8474396467208862, + "logits/rejected": -2.162605047225952, + "logps/chosen": -1.439106822013855, + "logps/rejected": -2.918508291244507, + "loss": 1.4884, + "nll_loss": 1.4704686403274536, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14391069114208221, + "rewards/margins": 0.14794014394283295, + "rewards/rejected": -0.29185086488723755, + "step": 871 + }, + { + "epoch": 2.4031756989989645, + "grad_norm": 0.21678882837295532, + "learning_rate": 5.860341494970132e-07, + "log_odds_chosen": 1.686343789100647, + "log_odds_ratio": -0.176815465092659, + "logits/chosen": -0.8357268571853638, + "logits/rejected": -2.049056053161621, + "logps/chosen": -1.3776237964630127, + "logps/rejected": -2.8327560424804688, + "loss": 1.4463, + "nll_loss": 1.4285705089569092, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1377623826265335, + "rewards/margins": 0.14551322162151337, + "rewards/rejected": -0.28327563405036926, + "step": 872 + }, + { + "epoch": 2.405937176389368, + "grad_norm": 0.23483207821846008, + "learning_rate": 5.808881491049723e-07, + "log_odds_chosen": 1.5488799810409546, + "log_odds_ratio": -0.21071894466876984, + "logits/chosen": -0.8064358234405518, + "logits/rejected": -1.6850775480270386, + "logps/chosen": -1.4290885925292969, + "logps/rejected": -2.7724051475524902, + "loss": 1.4928, + "nll_loss": 1.4716988801956177, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14290885627269745, + "rewards/margins": 0.13433167338371277, + "rewards/rejected": -0.277240514755249, + "step": 873 + }, + { + "epoch": 2.4086986537797723, + "grad_norm": 0.21404524147510529, + "learning_rate": 5.757618705564849e-07, + "log_odds_chosen": 1.6298061609268188, + "log_odds_ratio": -0.18933196365833282, + "logits/chosen": -0.7765361070632935, + "logits/rejected": -2.0819590091705322, + "logps/chosen": -1.3593862056732178, + "logps/rejected": -2.7558720111846924, + "loss": 1.4306, + "nll_loss": 1.4117118120193481, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1359386146068573, + "rewards/margins": 0.13964857161045074, + "rewards/rejected": -0.27558720111846924, + "step": 874 + }, + { + "epoch": 2.411460131170176, + "grad_norm": 0.2233470380306244, + "learning_rate": 5.706553665319955e-07, + "log_odds_chosen": 1.6726114749908447, + "log_odds_ratio": -0.18509700894355774, + "logits/chosen": -0.7863438129425049, + "logits/rejected": -2.077401876449585, + "logps/chosen": -1.4051029682159424, + "logps/rejected": -2.8573782444000244, + "loss": 1.4697, + "nll_loss": 1.4511629343032837, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14051030576229095, + "rewards/margins": 0.14522749185562134, + "rewards/rejected": -0.2857378125190735, + "step": 875 + }, + { + "epoch": 2.41422160856058, + "grad_norm": 0.24152301251888275, + "learning_rate": 5.655686895087328e-07, + "log_odds_chosen": 1.6769300699234009, + "log_odds_ratio": -0.18689285218715668, + "logits/chosen": -0.8174842596054077, + "logits/rejected": -2.004807949066162, + "logps/chosen": -1.435114860534668, + "logps/rejected": -2.8996763229370117, + "loss": 1.502, + "nll_loss": 1.4832693338394165, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14351148903369904, + "rewards/margins": 0.14645616710186005, + "rewards/rejected": -0.2899676561355591, + "step": 876 + }, + { + "epoch": 2.4169830859509838, + "grad_norm": 0.2544821798801422, + "learning_rate": 5.605018917601754e-07, + "log_odds_chosen": 1.5323283672332764, + "log_odds_ratio": -0.20912876725196838, + "logits/chosen": -0.7400486469268799, + "logits/rejected": -1.9783880710601807, + "logps/chosen": -1.423980712890625, + "logps/rejected": -2.744169235229492, + "loss": 1.4978, + "nll_loss": 1.476898431777954, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14239807426929474, + "rewards/margins": 0.13201886415481567, + "rewards/rejected": -0.2744169235229492, + "step": 877 + }, + { + "epoch": 2.4197445633413874, + "grad_norm": 0.25507211685180664, + "learning_rate": 5.554550253555066e-07, + "log_odds_chosen": 1.4117803573608398, + "log_odds_ratio": -0.22325754165649414, + "logits/chosen": -0.8820750713348389, + "logits/rejected": -1.6174063682556152, + "logps/chosen": -1.352549433708191, + "logps/rejected": -2.5473551750183105, + "loss": 1.4452, + "nll_loss": 1.4228417873382568, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13525494933128357, + "rewards/margins": 0.11948058754205704, + "rewards/rejected": -0.2547355592250824, + "step": 878 + }, + { + "epoch": 2.4225060407317915, + "grad_norm": 0.22250917553901672, + "learning_rate": 5.504281421590898e-07, + "log_odds_chosen": 1.6617779731750488, + "log_odds_ratio": -0.1937955915927887, + "logits/chosen": -0.8170138001441956, + "logits/rejected": -2.026573657989502, + "logps/chosen": -1.3464716672897339, + "logps/rejected": -2.777165651321411, + "loss": 1.4149, + "nll_loss": 1.395482063293457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1346471756696701, + "rewards/margins": 0.14306940138339996, + "rewards/rejected": -0.27771657705307007, + "step": 879 + }, + { + "epoch": 2.425267518122195, + "grad_norm": 0.23034892976284027, + "learning_rate": 5.454212938299256e-07, + "log_odds_chosen": 1.646827220916748, + "log_odds_ratio": -0.1863548904657364, + "logits/chosen": -0.9122263789176941, + "logits/rejected": -1.9986140727996826, + "logps/chosen": -1.412491798400879, + "logps/rejected": -2.8388447761535645, + "loss": 1.4726, + "nll_loss": 1.4539694786071777, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1412491649389267, + "rewards/margins": 0.14263533055782318, + "rewards/rejected": -0.2838844954967499, + "step": 880 + }, + { + "epoch": 2.4280289955125993, + "grad_norm": 0.20888766646385193, + "learning_rate": 5.404345318211294e-07, + "log_odds_chosen": 1.5793465375900269, + "log_odds_ratio": -0.20006848871707916, + "logits/chosen": -0.8890371918678284, + "logits/rejected": -1.9650756120681763, + "logps/chosen": -1.4828230142593384, + "logps/rejected": -2.86384654045105, + "loss": 1.5303, + "nll_loss": 1.5103161334991455, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14828228950500488, + "rewards/margins": 0.13810236752033234, + "rewards/rejected": -0.2863846719264984, + "step": 881 + }, + { + "epoch": 2.430790472903003, + "grad_norm": 0.24945060908794403, + "learning_rate": 5.354679073793942e-07, + "log_odds_chosen": 1.5567692518234253, + "log_odds_ratio": -0.2121957540512085, + "logits/chosen": -0.8492003679275513, + "logits/rejected": -1.7714898586273193, + "logps/chosen": -1.370238184928894, + "logps/rejected": -2.7132062911987305, + "loss": 1.4423, + "nll_loss": 1.4210622310638428, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13702382147312164, + "rewards/margins": 0.13429680466651917, + "rewards/rejected": -0.271320641040802, + "step": 882 + }, + { + "epoch": 2.433551950293407, + "grad_norm": 0.22785811126232147, + "learning_rate": 5.305214715444726e-07, + "log_odds_chosen": 1.3213268518447876, + "log_odds_ratio": -0.2632153034210205, + "logits/chosen": -0.7535655498504639, + "logits/rejected": -1.8558615446090698, + "logps/chosen": -1.468719482421875, + "logps/rejected": -2.605940580368042, + "loss": 1.5309, + "nll_loss": 1.5045620203018188, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14687195420265198, + "rewards/margins": 0.11372211575508118, + "rewards/rejected": -0.26059406995773315, + "step": 883 + }, + { + "epoch": 2.4363134276838108, + "grad_norm": 0.2245754897594452, + "learning_rate": 5.255952751486442e-07, + "log_odds_chosen": 1.3915148973464966, + "log_odds_ratio": -0.2355232983827591, + "logits/chosen": -0.7501202821731567, + "logits/rejected": -1.6952217817306519, + "logps/chosen": -1.428135871887207, + "logps/rejected": -2.6236112117767334, + "loss": 1.4991, + "nll_loss": 1.475578784942627, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14281359314918518, + "rewards/margins": 0.1195475235581398, + "rewards/rejected": -0.2623611390590668, + "step": 884 + }, + { + "epoch": 2.439074905074215, + "grad_norm": 0.2622843384742737, + "learning_rate": 5.20689368816201e-07, + "log_odds_chosen": 1.627671241760254, + "log_odds_ratio": -0.18791747093200684, + "logits/chosen": -0.7995626926422119, + "logits/rejected": -2.078826665878296, + "logps/chosen": -1.5019242763519287, + "logps/rejected": -2.931610584259033, + "loss": 1.5565, + "nll_loss": 1.537746548652649, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15019242465496063, + "rewards/margins": 0.14296863973140717, + "rewards/rejected": -0.2931610643863678, + "step": 885 + }, + { + "epoch": 2.4418363824646185, + "grad_norm": 0.22714628279209137, + "learning_rate": 5.158038029629195e-07, + "log_odds_chosen": 1.4414920806884766, + "log_odds_ratio": -0.21628612279891968, + "logits/chosen": -0.8436942100524902, + "logits/rejected": -1.9419432878494263, + "logps/chosen": -1.461559772491455, + "logps/rejected": -2.708940267562866, + "loss": 1.5133, + "nll_loss": 1.4916455745697021, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14615598320960999, + "rewards/margins": 0.12473806738853455, + "rewards/rejected": -0.27089405059814453, + "step": 886 + }, + { + "epoch": 2.4445978598550226, + "grad_norm": 0.23047834634780884, + "learning_rate": 5.109386277955477e-07, + "log_odds_chosen": 1.4839013814926147, + "log_odds_ratio": -0.21651501953601837, + "logits/chosen": -0.7708752155303955, + "logits/rejected": -1.970124363899231, + "logps/chosen": -1.391357421875, + "logps/rejected": -2.6658291816711426, + "loss": 1.4515, + "nll_loss": 1.4298431873321533, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13913573324680328, + "rewards/margins": 0.12744717299938202, + "rewards/rejected": -0.2665829360485077, + "step": 887 + }, + { + "epoch": 2.4473593372454263, + "grad_norm": 0.22831033170223236, + "learning_rate": 5.060938933112891e-07, + "log_odds_chosen": 1.4900858402252197, + "log_odds_ratio": -0.22767001390457153, + "logits/chosen": -0.9148192405700684, + "logits/rejected": -1.889332890510559, + "logps/chosen": -1.411495566368103, + "logps/rejected": -2.7004637718200684, + "loss": 1.4818, + "nll_loss": 1.4590768814086914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14114956557750702, + "rewards/margins": 0.1288968026638031, + "rewards/rejected": -0.2700463831424713, + "step": 888 + }, + { + "epoch": 2.45012081463583, + "grad_norm": 0.22466985881328583, + "learning_rate": 5.012696492972852e-07, + "log_odds_chosen": 1.5186207294464111, + "log_odds_ratio": -0.21536147594451904, + "logits/chosen": -0.8782755136489868, + "logits/rejected": -1.8780262470245361, + "logps/chosen": -1.3794608116149902, + "logps/rejected": -2.681459903717041, + "loss": 1.4452, + "nll_loss": 1.4237024784088135, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13794609904289246, + "rewards/margins": 0.13019990921020508, + "rewards/rejected": -0.26814597845077515, + "step": 889 + }, + { + "epoch": 2.452882292026234, + "grad_norm": 0.19952109456062317, + "learning_rate": 4.964659453301088e-07, + "log_odds_chosen": 1.964066505432129, + "log_odds_ratio": -0.1502349078655243, + "logits/chosen": -0.7120921611785889, + "logits/rejected": -2.4251503944396973, + "logps/chosen": -1.3529983758926392, + "logps/rejected": -3.066349506378174, + "loss": 1.4145, + "nll_loss": 1.3994845151901245, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13529984652996063, + "rewards/margins": 0.17133508622646332, + "rewards/rejected": -0.30663493275642395, + "step": 890 + }, + { + "epoch": 2.4556437694166378, + "grad_norm": 0.21941575407981873, + "learning_rate": 4.916828307752489e-07, + "log_odds_chosen": 1.5352859497070312, + "log_odds_ratio": -0.20665743947029114, + "logits/chosen": -0.854226291179657, + "logits/rejected": -1.9289751052856445, + "logps/chosen": -1.417055606842041, + "logps/rejected": -2.7416605949401855, + "loss": 1.474, + "nll_loss": 1.453291893005371, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14170557260513306, + "rewards/margins": 0.13246050477027893, + "rewards/rejected": -0.274166077375412, + "step": 891 + }, + { + "epoch": 2.458405246807042, + "grad_norm": 0.22385253012180328, + "learning_rate": 4.869203547866097e-07, + "log_odds_chosen": 1.5946441888809204, + "log_odds_ratio": -0.20085304975509644, + "logits/chosen": -0.7629926800727844, + "logits/rejected": -2.0393998622894287, + "logps/chosen": -1.4476027488708496, + "logps/rejected": -2.840999126434326, + "loss": 1.4987, + "nll_loss": 1.4786458015441895, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14476028084754944, + "rewards/margins": 0.13933967053890228, + "rewards/rejected": -0.2840999364852905, + "step": 892 + }, + { + "epoch": 2.4611667241974455, + "grad_norm": 0.23724229633808136, + "learning_rate": 4.821785663059991e-07, + "log_odds_chosen": 1.5859582424163818, + "log_odds_ratio": -0.19816404581069946, + "logits/chosen": -0.7932180762290955, + "logits/rejected": -1.9734810590744019, + "logps/chosen": -1.4295369386672974, + "logps/rejected": -2.806306838989258, + "loss": 1.4843, + "nll_loss": 1.4644794464111328, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14295370876789093, + "rewards/margins": 0.13767698407173157, + "rewards/rejected": -0.2806306779384613, + "step": 893 + }, + { + "epoch": 2.4639282015878496, + "grad_norm": 0.24634841084480286, + "learning_rate": 4.774575140626317e-07, + "log_odds_chosen": 1.6122064590454102, + "log_odds_ratio": -0.2016284167766571, + "logits/chosen": -0.79060959815979, + "logits/rejected": -1.965233564376831, + "logps/chosen": -1.497755527496338, + "logps/rejected": -2.916637420654297, + "loss": 1.5529, + "nll_loss": 1.532710313796997, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14977557957172394, + "rewards/margins": 0.14188817143440247, + "rewards/rejected": -0.2916637361049652, + "step": 894 + }, + { + "epoch": 2.4666896789782533, + "grad_norm": 0.22494585812091827, + "learning_rate": 4.7275724657262293e-07, + "log_odds_chosen": 1.7693073749542236, + "log_odds_ratio": -0.17247578501701355, + "logits/chosen": -0.7484267354011536, + "logits/rejected": -2.083472728729248, + "logps/chosen": -1.3836661577224731, + "logps/rejected": -2.9199507236480713, + "loss": 1.4367, + "nll_loss": 1.4195003509521484, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13836662471294403, + "rewards/margins": 0.15362845361232758, + "rewards/rejected": -0.2919950485229492, + "step": 895 + }, + { + "epoch": 2.4694511563686574, + "grad_norm": 0.22843420505523682, + "learning_rate": 4.6807781213849354e-07, + "log_odds_chosen": 1.6209605932235718, + "log_odds_ratio": -0.20323897898197174, + "logits/chosen": -0.8005288243293762, + "logits/rejected": -1.9480664730072021, + "logps/chosen": -1.4012664556503296, + "logps/rejected": -2.8044662475585938, + "loss": 1.4799, + "nll_loss": 1.459563136100769, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14012664556503296, + "rewards/margins": 0.14031997323036194, + "rewards/rejected": -0.2804466485977173, + "step": 896 + }, + { + "epoch": 2.472212633759061, + "grad_norm": 0.2613917291164398, + "learning_rate": 4.63419258848673e-07, + "log_odds_chosen": 1.5579595565795898, + "log_odds_ratio": -0.21004337072372437, + "logits/chosen": -0.8195448517799377, + "logits/rejected": -1.721724033355713, + "logps/chosen": -1.4049590826034546, + "logps/rejected": -2.7522690296173096, + "loss": 1.474, + "nll_loss": 1.4529823064804077, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1404959112405777, + "rewards/margins": 0.1347309947013855, + "rewards/rejected": -0.275226891040802, + "step": 897 + }, + { + "epoch": 2.474974111149465, + "grad_norm": 0.23283885419368744, + "learning_rate": 4.5878163457700327e-07, + "log_odds_chosen": 1.6135423183441162, + "log_odds_ratio": -0.20367959141731262, + "logits/chosen": -0.7768855094909668, + "logits/rejected": -2.0446574687957764, + "logps/chosen": -1.45505952835083, + "logps/rejected": -2.865198850631714, + "loss": 1.4948, + "nll_loss": 1.4743952751159668, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14550594985485077, + "rewards/margins": 0.14101392030715942, + "rewards/rejected": -0.2865198850631714, + "step": 898 + }, + { + "epoch": 2.477735588539869, + "grad_norm": 0.2130129635334015, + "learning_rate": 4.5416498698225034e-07, + "log_odds_chosen": 1.580468773841858, + "log_odds_ratio": -0.20243686437606812, + "logits/chosen": -0.7336734533309937, + "logits/rejected": -2.138326644897461, + "logps/chosen": -1.4078233242034912, + "logps/rejected": -2.7767281532287598, + "loss": 1.4735, + "nll_loss": 1.4533002376556396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14078234136104584, + "rewards/margins": 0.1368904709815979, + "rewards/rejected": -0.27767282724380493, + "step": 899 + }, + { + "epoch": 2.4804970659302725, + "grad_norm": 0.2216830551624298, + "learning_rate": 4.495693635076101e-07, + "log_odds_chosen": 1.535346508026123, + "log_odds_ratio": -0.20014424622058868, + "logits/chosen": -0.8925894498825073, + "logits/rejected": -2.127046585083008, + "logps/chosen": -1.376361608505249, + "logps/rejected": -2.692883014678955, + "loss": 1.4484, + "nll_loss": 1.4284191131591797, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13763616979122162, + "rewards/margins": 0.13165216147899628, + "rewards/rejected": -0.2692883312702179, + "step": 900 + }, + { + "epoch": 2.4832585433206766, + "grad_norm": 0.21299172937870026, + "learning_rate": 4.4499481138022546e-07, + "log_odds_chosen": 1.933703064918518, + "log_odds_ratio": -0.14841099083423615, + "logits/chosen": -0.8260481953620911, + "logits/rejected": -2.3376615047454834, + "logps/chosen": -1.372591495513916, + "logps/rejected": -3.0575618743896484, + "loss": 1.4185, + "nll_loss": 1.4036223888397217, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13725915551185608, + "rewards/margins": 0.16849704086780548, + "rewards/rejected": -0.30575618147850037, + "step": 901 + }, + { + "epoch": 2.4860200207110803, + "grad_norm": 0.23825101554393768, + "learning_rate": 4.4044137761069584e-07, + "log_odds_chosen": 1.4366477727890015, + "log_odds_ratio": -0.22512398660182953, + "logits/chosen": -0.7757810950279236, + "logits/rejected": -1.893244981765747, + "logps/chosen": -1.4974112510681152, + "logps/rejected": -2.7518978118896484, + "loss": 1.564, + "nll_loss": 1.5414974689483643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14974112808704376, + "rewards/margins": 0.12544865906238556, + "rewards/rejected": -0.2751897871494293, + "step": 902 + }, + { + "epoch": 2.4887814981014844, + "grad_norm": 0.23197093605995178, + "learning_rate": 4.359091089925999e-07, + "log_odds_chosen": 1.7243913412094116, + "log_odds_ratio": -0.1668962985277176, + "logits/chosen": -0.7759650349617004, + "logits/rejected": -2.1746482849121094, + "logps/chosen": -1.4509389400482178, + "logps/rejected": -2.961793899536133, + "loss": 1.4962, + "nll_loss": 1.479463815689087, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1450938880443573, + "rewards/margins": 0.1510854959487915, + "rewards/rejected": -0.2961794137954712, + "step": 903 + }, + { + "epoch": 2.491542975491888, + "grad_norm": 0.23694312572479248, + "learning_rate": 4.3139805210200917e-07, + "log_odds_chosen": 1.805801510810852, + "log_odds_ratio": -0.1572706699371338, + "logits/chosen": -0.8221292495727539, + "logits/rejected": -2.0871081352233887, + "logps/chosen": -1.4446747303009033, + "logps/rejected": -3.031667470932007, + "loss": 1.5126, + "nll_loss": 1.496914029121399, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14446748793125153, + "rewards/margins": 0.15869927406311035, + "rewards/rejected": -0.3031667470932007, + "step": 904 + }, + { + "epoch": 2.494304452882292, + "grad_norm": 0.22664044797420502, + "learning_rate": 4.2690825329701313e-07, + "log_odds_chosen": 1.5435681343078613, + "log_odds_ratio": -0.1995273232460022, + "logits/chosen": -0.7720770835876465, + "logits/rejected": -2.0005624294281006, + "logps/chosen": -1.3651032447814941, + "logps/rejected": -2.6808128356933594, + "loss": 1.4317, + "nll_loss": 1.4117234945297241, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13651034235954285, + "rewards/margins": 0.1315709352493286, + "rewards/rejected": -0.26808127760887146, + "step": 905 + }, + { + "epoch": 2.497065930272696, + "grad_norm": 0.22075149416923523, + "learning_rate": 4.2243975871724017e-07, + "log_odds_chosen": 1.3728384971618652, + "log_odds_ratio": -0.2398672103881836, + "logits/chosen": -0.8564939498901367, + "logits/rejected": -1.6772489547729492, + "logps/chosen": -1.420592188835144, + "logps/rejected": -2.5997753143310547, + "loss": 1.499, + "nll_loss": 1.474989652633667, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14205922186374664, + "rewards/margins": 0.11791832000017166, + "rewards/rejected": -0.2599775493144989, + "step": 906 + }, + { + "epoch": 2.4998274076630995, + "grad_norm": 0.2160337120294571, + "learning_rate": 4.179926142833876e-07, + "log_odds_chosen": 1.5068174600601196, + "log_odds_ratio": -0.20641684532165527, + "logits/chosen": -0.7754029631614685, + "logits/rejected": -2.1742076873779297, + "logps/chosen": -1.46108078956604, + "logps/rejected": -2.767190456390381, + "loss": 1.5144, + "nll_loss": 1.4937331676483154, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14610810577869415, + "rewards/margins": 0.13061094284057617, + "rewards/rejected": -0.27671903371810913, + "step": 907 + }, + { + "epoch": 2.5025888850535036, + "grad_norm": 0.23876003921031952, + "learning_rate": 4.1356686569674344e-07, + "log_odds_chosen": 1.5174545049667358, + "log_odds_ratio": -0.21640154719352722, + "logits/chosen": -0.8613941669464111, + "logits/rejected": -1.8654732704162598, + "logps/chosen": -1.4302722215652466, + "logps/rejected": -2.7472269535064697, + "loss": 1.4941, + "nll_loss": 1.4724962711334229, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14302721619606018, + "rewards/margins": 0.1316954791545868, + "rewards/rejected": -0.27472272515296936, + "step": 908 + }, + { + "epoch": 2.5053503624439077, + "grad_norm": 0.2481316179037094, + "learning_rate": 4.091625584387238e-07, + "log_odds_chosen": 1.6250629425048828, + "log_odds_ratio": -0.18948356807231903, + "logits/chosen": -0.830609917640686, + "logits/rejected": -2.119431495666504, + "logps/chosen": -1.4281871318817139, + "logps/rejected": -2.837716817855835, + "loss": 1.4933, + "nll_loss": 1.4743633270263672, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14281873404979706, + "rewards/margins": 0.1409529745578766, + "rewards/rejected": -0.28377169370651245, + "step": 909 + }, + { + "epoch": 2.5081118398343114, + "grad_norm": 0.22198787331581116, + "learning_rate": 4.0477973777039854e-07, + "log_odds_chosen": 1.4991892576217651, + "log_odds_ratio": -0.21167874336242676, + "logits/chosen": -0.884207010269165, + "logits/rejected": -1.8678700923919678, + "logps/chosen": -1.407118797302246, + "logps/rejected": -2.6911559104919434, + "loss": 1.4627, + "nll_loss": 1.4415607452392578, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14071188867092133, + "rewards/margins": 0.1284037083387375, + "rewards/rejected": -0.2691155970096588, + "step": 910 + }, + { + "epoch": 2.510873317224715, + "grad_norm": 0.2337605506181717, + "learning_rate": 4.004184487320317e-07, + "log_odds_chosen": 1.7035045623779297, + "log_odds_ratio": -0.1746985763311386, + "logits/chosen": -0.7929513454437256, + "logits/rejected": -2.064314842224121, + "logps/chosen": -1.472114086151123, + "logps/rejected": -2.966843843460083, + "loss": 1.5242, + "nll_loss": 1.5067758560180664, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14721143245697021, + "rewards/margins": 0.14947296679019928, + "rewards/rejected": -0.2966843843460083, + "step": 911 + }, + { + "epoch": 2.513634794615119, + "grad_norm": 0.22415797412395477, + "learning_rate": 3.9607873614261717e-07, + "log_odds_chosen": 1.6890547275543213, + "log_odds_ratio": -0.17474831640720367, + "logits/chosen": -0.7648621201515198, + "logits/rejected": -2.0227315425872803, + "logps/chosen": -1.430749773979187, + "logps/rejected": -2.9043784141540527, + "loss": 1.4881, + "nll_loss": 1.4706588983535767, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14307497441768646, + "rewards/margins": 0.1473628729581833, + "rewards/rejected": -0.29043784737586975, + "step": 912 + }, + { + "epoch": 2.516396272005523, + "grad_norm": 0.24374602735042572, + "learning_rate": 3.917606445994146e-07, + "log_odds_chosen": 1.6098326444625854, + "log_odds_ratio": -0.193284809589386, + "logits/chosen": -0.8505316972732544, + "logits/rejected": -1.93906569480896, + "logps/chosen": -1.4172815084457397, + "logps/rejected": -2.814074754714966, + "loss": 1.4815, + "nll_loss": 1.4621838331222534, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14172816276550293, + "rewards/margins": 0.13967935740947723, + "rewards/rejected": -0.28140750527381897, + "step": 913 + }, + { + "epoch": 2.519157749395927, + "grad_norm": 0.21846385300159454, + "learning_rate": 3.874642184774977e-07, + "log_odds_chosen": 1.5628008842468262, + "log_odds_ratio": -0.1962602436542511, + "logits/chosen": -0.8740373253822327, + "logits/rejected": -2.151853084564209, + "logps/chosen": -1.3966370820999146, + "logps/rejected": -2.7418158054351807, + "loss": 1.458, + "nll_loss": 1.4383662939071655, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1396637111902237, + "rewards/margins": 0.13451789319515228, + "rewards/rejected": -0.274181604385376, + "step": 914 + }, + { + "epoch": 2.5219192267863306, + "grad_norm": 0.24229982495307922, + "learning_rate": 3.831895019292897e-07, + "log_odds_chosen": 1.8062140941619873, + "log_odds_ratio": -0.18693974614143372, + "logits/chosen": -0.8387290239334106, + "logits/rejected": -1.937464714050293, + "logps/chosen": -1.4534918069839478, + "logps/rejected": -3.0489726066589355, + "loss": 1.5246, + "nll_loss": 1.5059365034103394, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1453491747379303, + "rewards/margins": 0.1595480740070343, + "rewards/rejected": -0.304897278547287, + "step": 915 + }, + { + "epoch": 2.5246807041767347, + "grad_norm": 0.23088249564170837, + "learning_rate": 3.789365388841193e-07, + "log_odds_chosen": 1.73601233959198, + "log_odds_ratio": -0.1753402054309845, + "logits/chosen": -0.8377318382263184, + "logits/rejected": -2.0821731090545654, + "logps/chosen": -1.432181477546692, + "logps/rejected": -2.9494948387145996, + "loss": 1.5013, + "nll_loss": 1.4837443828582764, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14321815967559814, + "rewards/margins": 0.15173134207725525, + "rewards/rejected": -0.2949495315551758, + "step": 916 + }, + { + "epoch": 2.5274421815671384, + "grad_norm": 0.24658960103988647, + "learning_rate": 3.7470537304776037e-07, + "log_odds_chosen": 1.4152191877365112, + "log_odds_ratio": -0.22389471530914307, + "logits/chosen": -0.7509374618530273, + "logits/rejected": -1.9573094844818115, + "logps/chosen": -1.4993343353271484, + "logps/rejected": -2.7293834686279297, + "loss": 1.5611, + "nll_loss": 1.5387518405914307, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14993345737457275, + "rewards/margins": 0.12300491333007812, + "rewards/rejected": -0.2729383409023285, + "step": 917 + }, + { + "epoch": 2.530203658957542, + "grad_norm": 0.22455866634845734, + "learning_rate": 3.7049604790198976e-07, + "log_odds_chosen": 1.5770710706710815, + "log_odds_ratio": -0.19922304153442383, + "logits/chosen": -0.8366534113883972, + "logits/rejected": -2.1329345703125, + "logps/chosen": -1.4017503261566162, + "logps/rejected": -2.7608864307403564, + "loss": 1.4762, + "nll_loss": 1.4563055038452148, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14017504453659058, + "rewards/margins": 0.13591359555721283, + "rewards/rejected": -0.2760886251926422, + "step": 918 + }, + { + "epoch": 2.532965136347946, + "grad_norm": 0.22560246288776398, + "learning_rate": 3.663086067041352e-07, + "log_odds_chosen": 1.6462829113006592, + "log_odds_ratio": -0.18665999174118042, + "logits/chosen": -0.7782268524169922, + "logits/rejected": -1.9328405857086182, + "logps/chosen": -1.4376909732818604, + "logps/rejected": -2.872429609298706, + "loss": 1.4934, + "nll_loss": 1.4747681617736816, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14376911520957947, + "rewards/margins": 0.14347384870052338, + "rewards/rejected": -0.28724294900894165, + "step": 919 + }, + { + "epoch": 2.5357266137383503, + "grad_norm": 0.23410910367965698, + "learning_rate": 3.621430924866348e-07, + "log_odds_chosen": 1.5727438926696777, + "log_odds_ratio": -0.19569893181324005, + "logits/chosen": -0.8823702931404114, + "logits/rejected": -1.9492605924606323, + "logps/chosen": -1.3187779188156128, + "logps/rejected": -2.647488594055176, + "loss": 1.4058, + "nll_loss": 1.3862695693969727, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13187779486179352, + "rewards/margins": 0.13287106156349182, + "rewards/rejected": -0.26474887132644653, + "step": 920 + }, + { + "epoch": 2.538488091128754, + "grad_norm": 0.22586840391159058, + "learning_rate": 3.579995480565909e-07, + "log_odds_chosen": 1.5409903526306152, + "log_odds_ratio": -0.2025071531534195, + "logits/chosen": -0.6942576766014099, + "logits/rejected": -1.9230279922485352, + "logps/chosen": -1.4208831787109375, + "logps/rejected": -2.7438626289367676, + "loss": 1.4877, + "nll_loss": 1.4674333333969116, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14208830893039703, + "rewards/margins": 0.13229797780513763, + "rewards/rejected": -0.27438628673553467, + "step": 921 + }, + { + "epoch": 2.5412495685191576, + "grad_norm": 0.21825292706489563, + "learning_rate": 3.538780159953348e-07, + "log_odds_chosen": 1.6649723052978516, + "log_odds_ratio": -0.198430597782135, + "logits/chosen": -0.7998772263526917, + "logits/rejected": -1.8460190296173096, + "logps/chosen": -1.3014402389526367, + "logps/rejected": -2.702242851257324, + "loss": 1.3639, + "nll_loss": 1.344069480895996, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13014401495456696, + "rewards/margins": 0.1400802731513977, + "rewards/rejected": -0.27022430300712585, + "step": 922 + }, + { + "epoch": 2.5440110459095617, + "grad_norm": 0.2391572743654251, + "learning_rate": 3.4977853865798466e-07, + "log_odds_chosen": 1.5206778049468994, + "log_odds_ratio": -0.21618333458900452, + "logits/chosen": -0.8164076209068298, + "logits/rejected": -1.6934431791305542, + "logps/chosen": -1.3476591110229492, + "logps/rejected": -2.639338970184326, + "loss": 1.4295, + "nll_loss": 1.4078460931777954, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13476590812206268, + "rewards/margins": 0.12916800379753113, + "rewards/rejected": -0.2639338970184326, + "step": 923 + }, + { + "epoch": 2.5467725232999654, + "grad_norm": 0.258484423160553, + "learning_rate": 3.457011581730124e-07, + "log_odds_chosen": 1.6479763984680176, + "log_odds_ratio": -0.20399057865142822, + "logits/chosen": -0.8047228455543518, + "logits/rejected": -1.9174230098724365, + "logps/chosen": -1.397106647491455, + "logps/rejected": -2.8355350494384766, + "loss": 1.4614, + "nll_loss": 1.4410136938095093, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1397106796503067, + "rewards/margins": 0.14384284615516663, + "rewards/rejected": -0.28355351090431213, + "step": 924 + }, + { + "epoch": 2.5495340006903695, + "grad_norm": 0.23048213124275208, + "learning_rate": 3.4164591644181233e-07, + "log_odds_chosen": 1.558042287826538, + "log_odds_ratio": -0.19656692445278168, + "logits/chosen": -0.803486704826355, + "logits/rejected": -1.7794066667556763, + "logps/chosen": -1.3663177490234375, + "logps/rejected": -2.700619697570801, + "loss": 1.4391, + "nll_loss": 1.4194140434265137, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1366317868232727, + "rewards/margins": 0.13343019783496857, + "rewards/rejected": -0.27006199955940247, + "step": 925 + }, + { + "epoch": 2.552295478080773, + "grad_norm": 0.23470544815063477, + "learning_rate": 3.3761285513826627e-07, + "log_odds_chosen": 1.6755598783493042, + "log_odds_ratio": -0.1804642528295517, + "logits/chosen": -0.7886132001876831, + "logits/rejected": -1.880044937133789, + "logps/chosen": -1.3564350605010986, + "logps/rejected": -2.797255516052246, + "loss": 1.4131, + "nll_loss": 1.3950680494308472, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13564349710941315, + "rewards/margins": 0.14408203959465027, + "rewards/rejected": -0.2797255516052246, + "step": 926 + }, + { + "epoch": 2.5550569554711773, + "grad_norm": 0.2104436457157135, + "learning_rate": 3.3360201570832e-07, + "log_odds_chosen": 1.78098464012146, + "log_odds_ratio": -0.18341396749019623, + "logits/chosen": -0.8001008629798889, + "logits/rejected": -1.9677042961120605, + "logps/chosen": -1.3394931554794312, + "logps/rejected": -2.8801090717315674, + "loss": 1.402, + "nll_loss": 1.383684754371643, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13394932448863983, + "rewards/margins": 0.15406160056591034, + "rewards/rejected": -0.28801092505455017, + "step": 927 + }, + { + "epoch": 2.557818432861581, + "grad_norm": 0.2330377697944641, + "learning_rate": 3.2961343936955384e-07, + "log_odds_chosen": 1.5845892429351807, + "log_odds_ratio": -0.19865839183330536, + "logits/chosen": -0.7797976732254028, + "logits/rejected": -1.9081692695617676, + "logps/chosen": -1.4326475858688354, + "logps/rejected": -2.804527759552002, + "loss": 1.5074, + "nll_loss": 1.487499713897705, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1432647556066513, + "rewards/margins": 0.13718801736831665, + "rewards/rejected": -0.28045278787612915, + "step": 928 + }, + { + "epoch": 2.5605799102519846, + "grad_norm": 0.23377783596515656, + "learning_rate": 3.256471671107617e-07, + "log_odds_chosen": 1.3970494270324707, + "log_odds_ratio": -0.23868386447429657, + "logits/chosen": -0.775575578212738, + "logits/rejected": -1.8493518829345703, + "logps/chosen": -1.3924803733825684, + "logps/rejected": -2.5844945907592773, + "loss": 1.4607, + "nll_loss": 1.436859369277954, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1392480432987213, + "rewards/margins": 0.1192014068365097, + "rewards/rejected": -0.2584494650363922, + "step": 929 + }, + { + "epoch": 2.5633413876423887, + "grad_norm": 0.23156331479549408, + "learning_rate": 3.217032396915265e-07, + "log_odds_chosen": 1.3594629764556885, + "log_odds_ratio": -0.2378542125225067, + "logits/chosen": -0.7673474550247192, + "logits/rejected": -1.6643857955932617, + "logps/chosen": -1.3724521398544312, + "logps/rejected": -2.525003433227539, + "loss": 1.4354, + "nll_loss": 1.4115841388702393, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13724520802497864, + "rewards/margins": 0.11525516957044601, + "rewards/rejected": -0.25250038504600525, + "step": 930 + }, + { + "epoch": 2.5661028650327924, + "grad_norm": 0.20743201673030853, + "learning_rate": 3.1778169764180575e-07, + "log_odds_chosen": 1.3011500835418701, + "log_odds_ratio": -0.2518286108970642, + "logits/chosen": -0.762809157371521, + "logits/rejected": -1.9848495721817017, + "logps/chosen": -1.408487319946289, + "logps/rejected": -2.5149049758911133, + "loss": 1.469, + "nll_loss": 1.4438223838806152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14084872603416443, + "rewards/margins": 0.11064177751541138, + "rewards/rejected": -0.2514905035495758, + "step": 931 + }, + { + "epoch": 2.5688643424231965, + "grad_norm": 0.23613394796848297, + "learning_rate": 3.1388258126151093e-07, + "log_odds_chosen": 1.639764666557312, + "log_odds_ratio": -0.18439045548439026, + "logits/chosen": -0.8464503288269043, + "logits/rejected": -2.162656784057617, + "logps/chosen": -1.3933584690093994, + "logps/rejected": -2.8041787147521973, + "loss": 1.4705, + "nll_loss": 1.4521013498306274, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13933585584163666, + "rewards/margins": 0.14108198881149292, + "rewards/rejected": -0.28041785955429077, + "step": 932 + }, + { + "epoch": 2.5716258198136, + "grad_norm": 0.22031117975711823, + "learning_rate": 3.100059306200959e-07, + "log_odds_chosen": 1.5673953294754028, + "log_odds_ratio": -0.19293257594108582, + "logits/chosen": -0.8196423649787903, + "logits/rejected": -2.1118574142456055, + "logps/chosen": -1.4162676334381104, + "logps/rejected": -2.7700836658477783, + "loss": 1.4824, + "nll_loss": 1.4630780220031738, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1416267603635788, + "rewards/margins": 0.1353815793991089, + "rewards/rejected": -0.2770083546638489, + "step": 933 + }, + { + "epoch": 2.5743872972040043, + "grad_norm": 0.22910936176776886, + "learning_rate": 3.06151785556143e-07, + "log_odds_chosen": 1.5494434833526611, + "log_odds_ratio": -0.20234021544456482, + "logits/chosen": -0.8608691692352295, + "logits/rejected": -2.1640162467956543, + "logps/chosen": -1.4172899723052979, + "logps/rejected": -2.7569541931152344, + "loss": 1.468, + "nll_loss": 1.447774887084961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1417289823293686, + "rewards/margins": 0.13396643102169037, + "rewards/rejected": -0.27569544315338135, + "step": 934 + }, + { + "epoch": 2.577148774594408, + "grad_norm": 0.22023598849773407, + "learning_rate": 3.02320185676957e-07, + "log_odds_chosen": 1.4850515127182007, + "log_odds_ratio": -0.21363158524036407, + "logits/chosen": -0.745186984539032, + "logits/rejected": -1.937159538269043, + "logps/chosen": -1.4178829193115234, + "logps/rejected": -2.6953628063201904, + "loss": 1.4788, + "nll_loss": 1.457419514656067, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1417883038520813, + "rewards/margins": 0.12774798274040222, + "rewards/rejected": -0.2695362865924835, + "step": 935 + }, + { + "epoch": 2.5799102519848116, + "grad_norm": 0.24953798949718475, + "learning_rate": 2.98511170358155e-07, + "log_odds_chosen": 1.6144005060195923, + "log_odds_ratio": -0.19470098614692688, + "logits/chosen": -0.884423553943634, + "logits/rejected": -2.062915325164795, + "logps/chosen": -1.519521713256836, + "logps/rejected": -2.9385733604431152, + "loss": 1.5727, + "nll_loss": 1.5532505512237549, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15195219218730927, + "rewards/margins": 0.14190514385700226, + "rewards/rejected": -0.29385730624198914, + "step": 936 + }, + { + "epoch": 2.5826717293752157, + "grad_norm": 0.223532572388649, + "learning_rate": 2.947247787432625e-07, + "log_odds_chosen": 1.4764959812164307, + "log_odds_ratio": -0.21015074849128723, + "logits/chosen": -0.6920619606971741, + "logits/rejected": -1.8913313150405884, + "logps/chosen": -1.4597268104553223, + "logps/rejected": -2.7401700019836426, + "loss": 1.5324, + "nll_loss": 1.5114260911941528, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14597268402576447, + "rewards/margins": 0.12804432213306427, + "rewards/rejected": -0.27401700615882874, + "step": 937 + }, + { + "epoch": 2.58543320676562, + "grad_norm": 0.24072711169719696, + "learning_rate": 2.9096104974331186e-07, + "log_odds_chosen": 1.680213212966919, + "log_odds_ratio": -0.1735863834619522, + "logits/chosen": -0.8284453749656677, + "logits/rejected": -1.9820377826690674, + "logps/chosen": -1.5137585401535034, + "logps/rejected": -2.988992214202881, + "loss": 1.5605, + "nll_loss": 1.5431687831878662, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.151375874876976, + "rewards/margins": 0.14752335846424103, + "rewards/rejected": -0.29889923334121704, + "step": 938 + }, + { + "epoch": 2.5881946841560235, + "grad_norm": 0.2170211672782898, + "learning_rate": 2.872200220364413e-07, + "log_odds_chosen": 1.6567010879516602, + "log_odds_ratio": -0.17847812175750732, + "logits/chosen": -0.7924590110778809, + "logits/rejected": -2.16463041305542, + "logps/chosen": -1.3362153768539429, + "logps/rejected": -2.7536497116088867, + "loss": 1.3989, + "nll_loss": 1.3810375928878784, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13362154364585876, + "rewards/margins": 0.14174343645572662, + "rewards/rejected": -0.2753649652004242, + "step": 939 + }, + { + "epoch": 2.590956161546427, + "grad_norm": 0.23444724082946777, + "learning_rate": 2.8350173406749975e-07, + "log_odds_chosen": 1.5718295574188232, + "log_odds_ratio": -0.19533279538154602, + "logits/chosen": -0.8168334364891052, + "logits/rejected": -1.8591034412384033, + "logps/chosen": -1.378641963005066, + "logps/rejected": -2.7296016216278076, + "loss": 1.4542, + "nll_loss": 1.4346563816070557, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13786420226097107, + "rewards/margins": 0.13509595394134521, + "rewards/rejected": -0.2729601562023163, + "step": 940 + }, + { + "epoch": 2.5937176389368313, + "grad_norm": 0.2225639373064041, + "learning_rate": 2.7980622404764876e-07, + "log_odds_chosen": 1.6045359373092651, + "log_odds_ratio": -0.18750955164432526, + "logits/chosen": -0.8011846542358398, + "logits/rejected": -2.075422763824463, + "logps/chosen": -1.4409395456314087, + "logps/rejected": -2.8338732719421387, + "loss": 1.4968, + "nll_loss": 1.478006362915039, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14409396052360535, + "rewards/margins": 0.1392933875322342, + "rewards/rejected": -0.28338736295700073, + "step": 941 + }, + { + "epoch": 2.596479116327235, + "grad_norm": 0.22642040252685547, + "learning_rate": 2.761335299539708e-07, + "log_odds_chosen": 1.523880958557129, + "log_odds_ratio": -0.2097545862197876, + "logits/chosen": -0.7880173921585083, + "logits/rejected": -2.0264089107513428, + "logps/chosen": -1.3784044981002808, + "logps/rejected": -2.6851532459259033, + "loss": 1.4377, + "nll_loss": 1.4167677164077759, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13784044981002808, + "rewards/margins": 0.13067489862442017, + "rewards/rejected": -0.26851534843444824, + "step": 942 + }, + { + "epoch": 2.599240593717639, + "grad_norm": 0.3195907473564148, + "learning_rate": 2.7248368952908055e-07, + "log_odds_chosen": 1.461771845817566, + "log_odds_ratio": -0.21980953216552734, + "logits/chosen": -0.848595380783081, + "logits/rejected": -1.7577111721038818, + "logps/chosen": -1.4408600330352783, + "logps/rejected": -2.7052931785583496, + "loss": 1.5089, + "nll_loss": 1.4868979454040527, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14408600330352783, + "rewards/margins": 0.12644335627555847, + "rewards/rejected": -0.2705293893814087, + "step": 943 + }, + { + "epoch": 2.6020020711080427, + "grad_norm": 0.26655301451683044, + "learning_rate": 2.688567402807357e-07, + "log_odds_chosen": 1.8628268241882324, + "log_odds_ratio": -0.15494072437286377, + "logits/chosen": -0.8694819211959839, + "logits/rejected": -1.999489188194275, + "logps/chosen": -1.3600715398788452, + "logps/rejected": -2.98134708404541, + "loss": 1.4275, + "nll_loss": 1.4120118618011475, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1360071450471878, + "rewards/margins": 0.16212758421897888, + "rewards/rejected": -0.2981347441673279, + "step": 944 + }, + { + "epoch": 2.604763548498447, + "grad_norm": 0.23980240523815155, + "learning_rate": 2.652527194814511e-07, + "log_odds_chosen": 1.4213838577270508, + "log_odds_ratio": -0.22679410874843597, + "logits/chosen": -0.8118551969528198, + "logits/rejected": -1.8827842473983765, + "logps/chosen": -1.3833218812942505, + "logps/rejected": -2.594524383544922, + "loss": 1.4616, + "nll_loss": 1.4388917684555054, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13833221793174744, + "rewards/margins": 0.12112022936344147, + "rewards/rejected": -0.2594524323940277, + "step": 945 + }, + { + "epoch": 2.6075250258888505, + "grad_norm": 0.2134982943534851, + "learning_rate": 2.6167166416811745e-07, + "log_odds_chosen": 1.740950345993042, + "log_odds_ratio": -0.16992521286010742, + "logits/chosen": -0.8220755457878113, + "logits/rejected": -1.9810476303100586, + "logps/chosen": -1.3848917484283447, + "logps/rejected": -2.8960180282592773, + "loss": 1.46, + "nll_loss": 1.4429820775985718, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13848918676376343, + "rewards/margins": 0.1511126607656479, + "rewards/rejected": -0.2896018326282501, + "step": 946 + }, + { + "epoch": 2.610286503279254, + "grad_norm": 0.26111698150634766, + "learning_rate": 2.5811361114161745e-07, + "log_odds_chosen": 1.6446951627731323, + "log_odds_ratio": -0.18224820494651794, + "logits/chosen": -0.8129628896713257, + "logits/rejected": -1.9805124998092651, + "logps/chosen": -1.4249690771102905, + "logps/rejected": -2.854400396347046, + "loss": 1.4867, + "nll_loss": 1.4684371948242188, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14249692857265472, + "rewards/margins": 0.1429431289434433, + "rewards/rejected": -0.285440057516098, + "step": 947 + }, + { + "epoch": 2.6130479806696583, + "grad_norm": 0.2299281805753708, + "learning_rate": 2.5457859696645245e-07, + "log_odds_chosen": 1.480647325515747, + "log_odds_ratio": -0.21617284417152405, + "logits/chosen": -0.8652956485748291, + "logits/rejected": -1.9248936176300049, + "logps/chosen": -1.398078203201294, + "logps/rejected": -2.6709463596343994, + "loss": 1.461, + "nll_loss": 1.4394274950027466, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1398078352212906, + "rewards/margins": 0.12728680670261383, + "rewards/rejected": -0.2670946419239044, + "step": 948 + }, + { + "epoch": 2.6158094580600624, + "grad_norm": 0.2682130038738251, + "learning_rate": 2.510666579703616e-07, + "log_odds_chosen": 1.8603851795196533, + "log_odds_ratio": -0.15828196704387665, + "logits/chosen": -0.9050633907318115, + "logits/rejected": -1.748826026916504, + "logps/chosen": -1.3446389436721802, + "logps/rejected": -2.956476926803589, + "loss": 1.4083, + "nll_loss": 1.3925139904022217, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13446390628814697, + "rewards/margins": 0.16118377447128296, + "rewards/rejected": -0.29564768075942993, + "step": 949 + }, + { + "epoch": 2.618570935450466, + "grad_norm": 0.22530537843704224, + "learning_rate": 2.4757783024395244e-07, + "log_odds_chosen": 1.7264577150344849, + "log_odds_ratio": -0.1791447103023529, + "logits/chosen": -0.7733549475669861, + "logits/rejected": -2.1992223262786865, + "logps/chosen": -1.4929616451263428, + "logps/rejected": -3.01509428024292, + "loss": 1.5547, + "nll_loss": 1.5368317365646362, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14929616451263428, + "rewards/margins": 0.15221327543258667, + "rewards/rejected": -0.30150943994522095, + "step": 950 + }, + { + "epoch": 2.6213324128408697, + "grad_norm": 0.23810777068138123, + "learning_rate": 2.441121496403273e-07, + "log_odds_chosen": 1.6358726024627686, + "log_odds_ratio": -0.18412204086780548, + "logits/chosen": -0.8609445095062256, + "logits/rejected": -2.067732810974121, + "logps/chosen": -1.4377186298370361, + "logps/rejected": -2.861238479614258, + "loss": 1.4963, + "nll_loss": 1.4778951406478882, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14377185702323914, + "rewards/margins": 0.14235195517539978, + "rewards/rejected": -0.2861238121986389, + "step": 951 + }, + { + "epoch": 2.624093890231274, + "grad_norm": 0.22364532947540283, + "learning_rate": 2.4066965177471645e-07, + "log_odds_chosen": 1.5541980266571045, + "log_odds_ratio": -0.2067326307296753, + "logits/chosen": -0.7771640419960022, + "logits/rejected": -1.7838490009307861, + "logps/chosen": -1.354691505432129, + "logps/rejected": -2.687770366668701, + "loss": 1.4278, + "nll_loss": 1.407082200050354, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13546916842460632, + "rewards/margins": 0.13330788910388947, + "rewards/rejected": -0.268777072429657, + "step": 952 + }, + { + "epoch": 2.6268553676216775, + "grad_norm": 0.2233099341392517, + "learning_rate": 2.372503720241126e-07, + "log_odds_chosen": 1.640329122543335, + "log_odds_ratio": -0.1796395182609558, + "logits/chosen": -0.7143995761871338, + "logits/rejected": -1.8810615539550781, + "logps/chosen": -1.4569883346557617, + "logps/rejected": -2.888185501098633, + "loss": 1.5234, + "nll_loss": 1.5054751634597778, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14569886028766632, + "rewards/margins": 0.1431196928024292, + "rewards/rejected": -0.2888185381889343, + "step": 953 + }, + { + "epoch": 2.6296168450120816, + "grad_norm": 0.2334192395210266, + "learning_rate": 2.338543455269046e-07, + "log_odds_chosen": 1.6295439004898071, + "log_odds_ratio": -0.18369105458259583, + "logits/chosen": -0.7941421866416931, + "logits/rejected": -1.828993797302246, + "logps/chosen": -1.334347128868103, + "logps/rejected": -2.7231056690216064, + "loss": 1.4216, + "nll_loss": 1.4031999111175537, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1334347277879715, + "rewards/margins": 0.13887585699558258, + "rewards/rejected": -0.2723105847835541, + "step": 954 + }, + { + "epoch": 2.6323783224024853, + "grad_norm": 0.21528248488903046, + "learning_rate": 2.3048160718252e-07, + "log_odds_chosen": 1.8298442363739014, + "log_odds_ratio": -0.15041130781173706, + "logits/chosen": -0.9097654223442078, + "logits/rejected": -2.181267499923706, + "logps/chosen": -1.3622467517852783, + "logps/rejected": -2.9486942291259766, + "loss": 1.4346, + "nll_loss": 1.419554591178894, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1362246870994568, + "rewards/margins": 0.15864476561546326, + "rewards/rejected": -0.29486945271492004, + "step": 955 + }, + { + "epoch": 2.6351397997928894, + "grad_norm": 0.24860750138759613, + "learning_rate": 2.271321916510627e-07, + "log_odds_chosen": 2.00567889213562, + "log_odds_ratio": -0.13462091982364655, + "logits/chosen": -0.8964054584503174, + "logits/rejected": -2.0301802158355713, + "logps/chosen": -1.3431572914123535, + "logps/rejected": -3.094275712966919, + "loss": 1.4133, + "nll_loss": 1.3998205661773682, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13431572914123535, + "rewards/margins": 0.17511184513568878, + "rewards/rejected": -0.3094276189804077, + "step": 956 + }, + { + "epoch": 2.637901277183293, + "grad_norm": 0.2248452603816986, + "learning_rate": 2.2380613335296037e-07, + "log_odds_chosen": 1.632684588432312, + "log_odds_ratio": -0.19020292162895203, + "logits/chosen": -0.7716628909111023, + "logits/rejected": -2.124098300933838, + "logps/chosen": -1.4696786403656006, + "logps/rejected": -2.9018936157226562, + "loss": 1.5189, + "nll_loss": 1.499929428100586, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14696787297725677, + "rewards/margins": 0.14322149753570557, + "rewards/rejected": -0.29018938541412354, + "step": 957 + }, + { + "epoch": 2.6406627545736967, + "grad_norm": 0.2505524158477783, + "learning_rate": 2.205034664686076e-07, + "log_odds_chosen": 1.6690312623977661, + "log_odds_ratio": -0.19188562035560608, + "logits/chosen": -0.8181016445159912, + "logits/rejected": -1.8956444263458252, + "logps/chosen": -1.399005651473999, + "logps/rejected": -2.8518404960632324, + "loss": 1.4847, + "nll_loss": 1.4655529260635376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1399005800485611, + "rewards/margins": 0.14528347551822662, + "rewards/rejected": -0.2851840555667877, + "step": 958 + }, + { + "epoch": 2.643424231964101, + "grad_norm": 0.21002128720283508, + "learning_rate": 2.1722422493801716e-07, + "log_odds_chosen": 1.861242413520813, + "log_odds_ratio": -0.16027867794036865, + "logits/chosen": -0.788373589515686, + "logits/rejected": -2.0933547019958496, + "logps/chosen": -1.401833415031433, + "logps/rejected": -3.0320067405700684, + "loss": 1.4803, + "nll_loss": 1.4643118381500244, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14018335938453674, + "rewards/margins": 0.16301734745502472, + "rewards/rejected": -0.3032006621360779, + "step": 959 + }, + { + "epoch": 2.646185709354505, + "grad_norm": 0.24437056481838226, + "learning_rate": 2.1396844246046904e-07, + "log_odds_chosen": 1.6724685430526733, + "log_odds_ratio": -0.1835877001285553, + "logits/chosen": -0.8264733552932739, + "logits/rejected": -1.9276909828186035, + "logps/chosen": -1.4727625846862793, + "logps/rejected": -2.939807415008545, + "loss": 1.5417, + "nll_loss": 1.52329683303833, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14727625250816345, + "rewards/margins": 0.14670446515083313, + "rewards/rejected": -0.29398074746131897, + "step": 960 + }, + { + "epoch": 2.6489471867449086, + "grad_norm": 0.2100415825843811, + "learning_rate": 2.1073615249416536e-07, + "log_odds_chosen": 1.6273092031478882, + "log_odds_ratio": -0.18217778205871582, + "logits/chosen": -0.7561967372894287, + "logits/rejected": -2.013071298599243, + "logps/chosen": -1.3234224319458008, + "logps/rejected": -2.7003798484802246, + "loss": 1.3868, + "nll_loss": 1.3686209917068481, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13234223425388336, + "rewards/margins": 0.13769574463367462, + "rewards/rejected": -0.270037978887558, + "step": 961 + }, + { + "epoch": 2.6517086641353123, + "grad_norm": 0.22587327659130096, + "learning_rate": 2.075273882558873e-07, + "log_odds_chosen": 1.6430224180221558, + "log_odds_ratio": -0.1941165179014206, + "logits/chosen": -0.8549885153770447, + "logits/rejected": -1.9616295099258423, + "logps/chosen": -1.4437212944030762, + "logps/rejected": -2.8785641193389893, + "loss": 1.5107, + "nll_loss": 1.491284728050232, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1443721204996109, + "rewards/margins": 0.14348429441452026, + "rewards/rejected": -0.28785640001296997, + "step": 962 + }, + { + "epoch": 2.6544701415257164, + "grad_norm": 0.23256927728652954, + "learning_rate": 2.0434218272065154e-07, + "log_odds_chosen": 1.7010345458984375, + "log_odds_ratio": -0.17828573286533356, + "logits/chosen": -0.7128124833106995, + "logits/rejected": -1.9806416034698486, + "logps/chosen": -1.4034103155136108, + "logps/rejected": -2.88523530960083, + "loss": 1.4768, + "nll_loss": 1.4589712619781494, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14034104347229004, + "rewards/margins": 0.1481824815273285, + "rewards/rejected": -0.28852352499961853, + "step": 963 + }, + { + "epoch": 2.65723161891612, + "grad_norm": 0.2344788759946823, + "learning_rate": 2.0118056862137358e-07, + "log_odds_chosen": 1.6828176975250244, + "log_odds_ratio": -0.1736169308423996, + "logits/chosen": -0.8599337339401245, + "logits/rejected": -2.0625905990600586, + "logps/chosen": -1.3553197383880615, + "logps/rejected": -2.8010125160217285, + "loss": 1.4039, + "nll_loss": 1.3865183591842651, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13553199172019958, + "rewards/margins": 0.1445692479610443, + "rewards/rejected": -0.2801012396812439, + "step": 964 + }, + { + "epoch": 2.659993096306524, + "grad_norm": 0.236652672290802, + "learning_rate": 1.980425784485293e-07, + "log_odds_chosen": 1.5190538167953491, + "log_odds_ratio": -0.21639059484004974, + "logits/chosen": -0.7456527948379517, + "logits/rejected": -1.8640908002853394, + "logps/chosen": -1.3491963148117065, + "logps/rejected": -2.6379213333129883, + "loss": 1.4086, + "nll_loss": 1.3869160413742065, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1349196434020996, + "rewards/margins": 0.12887249886989594, + "rewards/rejected": -0.26379215717315674, + "step": 965 + }, + { + "epoch": 2.662754573696928, + "grad_norm": 0.21286852657794952, + "learning_rate": 1.9492824444982378e-07, + "log_odds_chosen": 1.6951912641525269, + "log_odds_ratio": -0.17544129490852356, + "logits/chosen": -0.792149007320404, + "logits/rejected": -2.137972593307495, + "logps/chosen": -1.3928117752075195, + "logps/rejected": -2.8630385398864746, + "loss": 1.4502, + "nll_loss": 1.4327032566070557, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13928118348121643, + "rewards/margins": 0.14702269434928894, + "rewards/rejected": -0.286303848028183, + "step": 966 + }, + { + "epoch": 2.665516051087332, + "grad_norm": 0.23913699388504028, + "learning_rate": 1.918375986298565e-07, + "log_odds_chosen": 1.722267508506775, + "log_odds_ratio": -0.17880691587924957, + "logits/chosen": -0.7047528028488159, + "logits/rejected": -1.9316701889038086, + "logps/chosen": -1.433093786239624, + "logps/rejected": -2.9400246143341064, + "loss": 1.488, + "nll_loss": 1.4701223373413086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14330938458442688, + "rewards/margins": 0.15069308876991272, + "rewards/rejected": -0.2940024733543396, + "step": 967 + }, + { + "epoch": 2.6682775284777356, + "grad_norm": 0.24249720573425293, + "learning_rate": 1.887706727497965e-07, + "log_odds_chosen": 1.5532490015029907, + "log_odds_ratio": -0.21471494436264038, + "logits/chosen": -0.787434995174408, + "logits/rejected": -1.668123722076416, + "logps/chosen": -1.3064297437667847, + "logps/rejected": -2.625889539718628, + "loss": 1.3785, + "nll_loss": 1.3570327758789062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13064298033714294, + "rewards/margins": 0.13194599747657776, + "rewards/rejected": -0.2625889778137207, + "step": 968 + }, + { + "epoch": 2.6710390058681392, + "grad_norm": 0.2098485380411148, + "learning_rate": 1.8572749832705074e-07, + "log_odds_chosen": 1.7862035036087036, + "log_odds_ratio": -0.16700312495231628, + "logits/chosen": -0.8437852263450623, + "logits/rejected": -2.3016438484191895, + "logps/chosen": -1.377842903137207, + "logps/rejected": -2.9322235584259033, + "loss": 1.4347, + "nll_loss": 1.4179821014404297, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13778428733348846, + "rewards/margins": 0.15543806552886963, + "rewards/rejected": -0.2932223677635193, + "step": 969 + }, + { + "epoch": 2.6738004832585434, + "grad_norm": 0.22286038100719452, + "learning_rate": 1.8270810663494591e-07, + "log_odds_chosen": 1.5529109239578247, + "log_odds_ratio": -0.1945139765739441, + "logits/chosen": -0.765722393989563, + "logits/rejected": -2.180504322052002, + "logps/chosen": -1.436352252960205, + "logps/rejected": -2.7805469036102295, + "loss": 1.5017, + "nll_loss": 1.482216715812683, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14363522827625275, + "rewards/margins": 0.1344194859266281, + "rewards/rejected": -0.27805471420288086, + "step": 970 + }, + { + "epoch": 2.676561960648947, + "grad_norm": 0.21117275953292847, + "learning_rate": 1.7971252870240292e-07, + "log_odds_chosen": 1.7019497156143188, + "log_odds_ratio": -0.17576996982097626, + "logits/chosen": -0.7316712737083435, + "logits/rejected": -2.2303671836853027, + "logps/chosen": -1.419854998588562, + "logps/rejected": -2.902592182159424, + "loss": 1.4805, + "nll_loss": 1.4629535675048828, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14198550581932068, + "rewards/margins": 0.14827370643615723, + "rewards/rejected": -0.2902592122554779, + "step": 971 + }, + { + "epoch": 2.679323438039351, + "grad_norm": 0.23764653503894806, + "learning_rate": 1.767407953136202e-07, + "log_odds_chosen": 1.4426761865615845, + "log_odds_ratio": -0.21827517449855804, + "logits/chosen": -0.8238155245780945, + "logits/rejected": -1.960955262184143, + "logps/chosen": -1.4752743244171143, + "logps/rejected": -2.727480888366699, + "loss": 1.5432, + "nll_loss": 1.5213298797607422, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14752742648124695, + "rewards/margins": 0.1252206712961197, + "rewards/rejected": -0.27274811267852783, + "step": 972 + }, + { + "epoch": 2.682084915429755, + "grad_norm": 0.23228906095027924, + "learning_rate": 1.737929370077554e-07, + "log_odds_chosen": 1.5190385580062866, + "log_odds_ratio": -0.21730369329452515, + "logits/chosen": -0.7989239692687988, + "logits/rejected": -1.6477134227752686, + "logps/chosen": -1.3697959184646606, + "logps/rejected": -2.6735963821411133, + "loss": 1.442, + "nll_loss": 1.4202287197113037, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1369796097278595, + "rewards/margins": 0.1303800344467163, + "rewards/rejected": -0.2673596441745758, + "step": 973 + }, + { + "epoch": 2.684846392820159, + "grad_norm": 0.22859090566635132, + "learning_rate": 1.7086898407861486e-07, + "log_odds_chosen": 1.8996504545211792, + "log_odds_ratio": -0.1469726860523224, + "logits/chosen": -0.8028102517127991, + "logits/rejected": -2.2114791870117188, + "logps/chosen": -1.437674880027771, + "logps/rejected": -3.1114306449890137, + "loss": 1.5027, + "nll_loss": 1.488043189048767, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14376749098300934, + "rewards/margins": 0.1673755794763565, + "rewards/rejected": -0.31114307045936584, + "step": 974 + }, + { + "epoch": 2.6876078702105626, + "grad_norm": 0.21925011277198792, + "learning_rate": 1.679689665743381e-07, + "log_odds_chosen": 1.5367701053619385, + "log_odds_ratio": -0.20496302843093872, + "logits/chosen": -0.7729565501213074, + "logits/rejected": -2.092475652694702, + "logps/chosen": -1.4283016920089722, + "logps/rejected": -2.751614809036255, + "loss": 1.4789, + "nll_loss": 1.458438515663147, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14283017814159393, + "rewards/margins": 0.13233129680156708, + "rewards/rejected": -0.275161474943161, + "step": 975 + }, + { + "epoch": 2.6903693476009662, + "grad_norm": 0.2413259893655777, + "learning_rate": 1.6509291429709224e-07, + "log_odds_chosen": 1.5076382160186768, + "log_odds_ratio": -0.20157039165496826, + "logits/chosen": -0.8660017848014832, + "logits/rejected": -1.917562484741211, + "logps/chosen": -1.4516851902008057, + "logps/rejected": -2.7548704147338867, + "loss": 1.5102, + "nll_loss": 1.49000084400177, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1451684981584549, + "rewards/margins": 0.1303185373544693, + "rewards/rejected": -0.2754870653152466, + "step": 976 + }, + { + "epoch": 2.6931308249913704, + "grad_norm": 0.21110667288303375, + "learning_rate": 1.6224085680276418e-07, + "log_odds_chosen": 1.7144944667816162, + "log_odds_ratio": -0.17031097412109375, + "logits/chosen": -0.911084771156311, + "logits/rejected": -2.202202558517456, + "logps/chosen": -1.3859715461730957, + "logps/rejected": -2.872420310974121, + "loss": 1.4479, + "nll_loss": 1.4308537244796753, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13859716057777405, + "rewards/margins": 0.14864489436149597, + "rewards/rejected": -0.28724205493927, + "step": 977 + }, + { + "epoch": 2.6958923023817745, + "grad_norm": 0.2264646589756012, + "learning_rate": 1.59412823400657e-07, + "log_odds_chosen": 1.5225772857666016, + "log_odds_ratio": -0.20823891460895538, + "logits/chosen": -0.8968706727027893, + "logits/rejected": -1.7435719966888428, + "logps/chosen": -1.3663400411605835, + "logps/rejected": -2.6695518493652344, + "loss": 1.4288, + "nll_loss": 1.4079402685165405, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1366340070962906, + "rewards/margins": 0.1303211748600006, + "rewards/rejected": -0.2669551968574524, + "step": 978 + }, + { + "epoch": 2.698653779772178, + "grad_norm": 0.2247675657272339, + "learning_rate": 1.5660884315319036e-07, + "log_odds_chosen": 1.6573270559310913, + "log_odds_ratio": -0.18946120142936707, + "logits/chosen": -0.743579626083374, + "logits/rejected": -2.06406569480896, + "logps/chosen": -1.497208833694458, + "logps/rejected": -2.953242301940918, + "loss": 1.5269, + "nll_loss": 1.5079742670059204, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14972089231014252, + "rewards/margins": 0.14560338854789734, + "rewards/rejected": -0.29532429575920105, + "step": 979 + }, + { + "epoch": 2.701415257162582, + "grad_norm": 0.24291600286960602, + "learning_rate": 1.538289448755989e-07, + "log_odds_chosen": 1.976696491241455, + "log_odds_ratio": -0.13205505907535553, + "logits/chosen": -0.8543053865432739, + "logits/rejected": -2.206322193145752, + "logps/chosen": -1.392202615737915, + "logps/rejected": -3.125058174133301, + "loss": 1.4566, + "nll_loss": 1.4434106349945068, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13922026753425598, + "rewards/margins": 0.17328554391860962, + "rewards/rejected": -0.3125058114528656, + "step": 980 + }, + { + "epoch": 2.704176734552986, + "grad_norm": 0.22567534446716309, + "learning_rate": 1.5107315713563942e-07, + "log_odds_chosen": 1.5911579132080078, + "log_odds_ratio": -0.2004013955593109, + "logits/chosen": -0.8179699182510376, + "logits/rejected": -1.968597650527954, + "logps/chosen": -1.3965879678726196, + "logps/rejected": -2.766774892807007, + "loss": 1.4621, + "nll_loss": 1.4420568943023682, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13965877890586853, + "rewards/margins": 0.13701872527599335, + "rewards/rejected": -0.27667751908302307, + "step": 981 + }, + { + "epoch": 2.7069382119433896, + "grad_norm": 0.22604532539844513, + "learning_rate": 1.483415082532938e-07, + "log_odds_chosen": 1.6166930198669434, + "log_odds_ratio": -0.18223077058792114, + "logits/chosen": -0.8054912090301514, + "logits/rejected": -1.9989535808563232, + "logps/chosen": -1.4191519021987915, + "logps/rejected": -2.8198201656341553, + "loss": 1.4748, + "nll_loss": 1.456567645072937, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1419151872396469, + "rewards/margins": 0.14006684720516205, + "rewards/rejected": -0.28198203444480896, + "step": 982 + }, + { + "epoch": 2.7096996893337937, + "grad_norm": 0.26856815814971924, + "learning_rate": 1.456340263004813e-07, + "log_odds_chosen": 1.6243596076965332, + "log_odds_ratio": -0.18607592582702637, + "logits/chosen": -0.7150140404701233, + "logits/rejected": -1.617521047592163, + "logps/chosen": -1.441032886505127, + "logps/rejected": -2.855644702911377, + "loss": 1.5154, + "nll_loss": 1.496822476387024, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1441032886505127, + "rewards/margins": 0.14146117866039276, + "rewards/rejected": -0.28556448221206665, + "step": 983 + }, + { + "epoch": 2.7124611667241973, + "grad_norm": 0.23320171236991882, + "learning_rate": 1.4295073910076757e-07, + "log_odds_chosen": 1.7140666246414185, + "log_odds_ratio": -0.17526394128799438, + "logits/chosen": -0.7767703533172607, + "logits/rejected": -1.9408453702926636, + "logps/chosen": -1.2976926565170288, + "logps/rejected": -2.762031078338623, + "loss": 1.3844, + "nll_loss": 1.366840124130249, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12976926565170288, + "rewards/margins": 0.14643386006355286, + "rewards/rejected": -0.2762031555175781, + "step": 984 + }, + { + "epoch": 2.7152226441146015, + "grad_norm": 0.2552090883255005, + "learning_rate": 1.4029167422908107e-07, + "log_odds_chosen": 1.5235956907272339, + "log_odds_ratio": -0.20630496740341187, + "logits/chosen": -0.8022536635398865, + "logits/rejected": -1.7930173873901367, + "logps/chosen": -1.3867225646972656, + "logps/rejected": -2.6964144706726074, + "loss": 1.4484, + "nll_loss": 1.4278028011322021, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13867226243019104, + "rewards/margins": 0.13096915185451508, + "rewards/rejected": -0.2696414589881897, + "step": 985 + }, + { + "epoch": 2.717984121505005, + "grad_norm": 0.22836096584796906, + "learning_rate": 1.3765685901142718e-07, + "log_odds_chosen": 1.4520777463912964, + "log_odds_ratio": -0.22023352980613708, + "logits/chosen": -0.8523675203323364, + "logits/rejected": -1.841475009918213, + "logps/chosen": -1.402612328529358, + "logps/rejected": -2.647597074508667, + "loss": 1.4789, + "nll_loss": 1.456899881362915, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1402612328529358, + "rewards/margins": 0.12449847161769867, + "rewards/rejected": -0.26475971937179565, + "step": 986 + }, + { + "epoch": 2.720745598895409, + "grad_norm": 0.2556321620941162, + "learning_rate": 1.3504632052460803e-07, + "log_odds_chosen": 1.4889233112335205, + "log_odds_ratio": -0.21026012301445007, + "logits/chosen": -0.790485143661499, + "logits/rejected": -1.8951472043991089, + "logps/chosen": -1.3497055768966675, + "logps/rejected": -2.614793539047241, + "loss": 1.4307, + "nll_loss": 1.4096403121948242, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.134970560669899, + "rewards/margins": 0.12650878727436066, + "rewards/rejected": -0.26147934794425964, + "step": 987 + }, + { + "epoch": 2.723507076285813, + "grad_norm": 0.22848525643348694, + "learning_rate": 1.3246008559594708e-07, + "log_odds_chosen": 1.435408353805542, + "log_odds_ratio": -0.2240704894065857, + "logits/chosen": -0.8080549240112305, + "logits/rejected": -1.9176137447357178, + "logps/chosen": -1.3366992473602295, + "logps/rejected": -2.5517029762268066, + "loss": 1.4005, + "nll_loss": 1.378090262413025, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1336699277162552, + "rewards/margins": 0.12150037288665771, + "rewards/rejected": -0.2551702857017517, + "step": 988 + }, + { + "epoch": 2.726268553676217, + "grad_norm": 0.22359248995780945, + "learning_rate": 1.2989818080300836e-07, + "log_odds_chosen": 1.6755691766738892, + "log_odds_ratio": -0.17601075768470764, + "logits/chosen": -0.7978137135505676, + "logits/rejected": -2.044973134994507, + "logps/chosen": -1.3630049228668213, + "logps/rejected": -2.8028440475463867, + "loss": 1.4252, + "nll_loss": 1.4075582027435303, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13630050420761108, + "rewards/margins": 0.1439839005470276, + "rewards/rejected": -0.28028440475463867, + "step": 989 + }, + { + "epoch": 2.7290300310666207, + "grad_norm": 0.2378959059715271, + "learning_rate": 1.273606324733284e-07, + "log_odds_chosen": 1.7684226036071777, + "log_odds_ratio": -0.1636146754026413, + "logits/chosen": -0.8993475437164307, + "logits/rejected": -1.91559636592865, + "logps/chosen": -1.445130705833435, + "logps/rejected": -2.9962029457092285, + "loss": 1.5039, + "nll_loss": 1.4875314235687256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1445130705833435, + "rewards/margins": 0.15510722994804382, + "rewards/rejected": -0.2996203303337097, + "step": 990 + }, + { + "epoch": 2.7317915084570243, + "grad_norm": 0.2171250879764557, + "learning_rate": 1.2484746668414176e-07, + "log_odds_chosen": 1.6286214590072632, + "log_odds_ratio": -0.1897205263376236, + "logits/chosen": -0.7582758069038391, + "logits/rejected": -2.000474691390991, + "logps/chosen": -1.4032773971557617, + "logps/rejected": -2.8157689571380615, + "loss": 1.4781, + "nll_loss": 1.459139347076416, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14032772183418274, + "rewards/margins": 0.1412491649389267, + "rewards/rejected": -0.28157690167427063, + "step": 991 + }, + { + "epoch": 2.7345529858474285, + "grad_norm": 0.21213258802890778, + "learning_rate": 1.223587092621162e-07, + "log_odds_chosen": 1.5115113258361816, + "log_odds_ratio": -0.20901797711849213, + "logits/chosen": -0.8601601123809814, + "logits/rejected": -1.9279264211654663, + "logps/chosen": -1.4223215579986572, + "logps/rejected": -2.725409746170044, + "loss": 1.4812, + "nll_loss": 1.460340142250061, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14223216474056244, + "rewards/margins": 0.1303088366985321, + "rewards/rejected": -0.27254101634025574, + "step": 992 + }, + { + "epoch": 2.737314463237832, + "grad_norm": 0.2581441402435303, + "learning_rate": 1.1989438578308394e-07, + "log_odds_chosen": 1.6569344997406006, + "log_odds_ratio": -0.181952103972435, + "logits/chosen": -0.8941519856452942, + "logits/rejected": -1.814483880996704, + "logps/chosen": -1.4135879278182983, + "logps/rejected": -2.8543639183044434, + "loss": 1.4987, + "nll_loss": 1.4805060625076294, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14135879278182983, + "rewards/margins": 0.1440775990486145, + "rewards/rejected": -0.28543636202812195, + "step": 993 + }, + { + "epoch": 2.7400759406282362, + "grad_norm": 0.2210199534893036, + "learning_rate": 1.1745452157178206e-07, + "log_odds_chosen": 1.7576181888580322, + "log_odds_ratio": -0.18280641734600067, + "logits/chosen": -0.8181760311126709, + "logits/rejected": -2.02272891998291, + "logps/chosen": -1.3050153255462646, + "logps/rejected": -2.8192529678344727, + "loss": 1.3727, + "nll_loss": 1.3544033765792847, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13050153851509094, + "rewards/margins": 0.15142375230789185, + "rewards/rejected": -0.2819252610206604, + "step": 994 + }, + { + "epoch": 2.74283741801864, + "grad_norm": 0.21427030861377716, + "learning_rate": 1.1503914170159058e-07, + "log_odds_chosen": 1.7605931758880615, + "log_odds_ratio": -0.16457465291023254, + "logits/chosen": -0.7366156578063965, + "logits/rejected": -2.1901280879974365, + "logps/chosen": -1.4154034852981567, + "logps/rejected": -2.9496779441833496, + "loss": 1.4716, + "nll_loss": 1.455174446105957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14154034852981567, + "rewards/margins": 0.15342746675014496, + "rewards/rejected": -0.29496777057647705, + "step": 995 + }, + { + "epoch": 2.745598895409044, + "grad_norm": 0.22262705862522125, + "learning_rate": 1.1264827099427417e-07, + "log_odds_chosen": 1.4252431392669678, + "log_odds_ratio": -0.23039494454860687, + "logits/chosen": -0.813992977142334, + "logits/rejected": -1.825137734413147, + "logps/chosen": -1.3184692859649658, + "logps/rejected": -2.5151712894439697, + "loss": 1.3915, + "nll_loss": 1.3685065507888794, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13184693455696106, + "rewards/margins": 0.11967021971940994, + "rewards/rejected": -0.2515171468257904, + "step": 996 + }, + { + "epoch": 2.7483603727994477, + "grad_norm": 0.3166691064834595, + "learning_rate": 1.1028193401972865e-07, + "log_odds_chosen": 1.5943597555160522, + "log_odds_ratio": -0.19616588950157166, + "logits/chosen": -0.7697460055351257, + "logits/rejected": -1.8766896724700928, + "logps/chosen": -1.3608465194702148, + "logps/rejected": -2.720303773880005, + "loss": 1.4281, + "nll_loss": 1.4085121154785156, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.136084645986557, + "rewards/margins": 0.13594573736190796, + "rewards/rejected": -0.27203038334846497, + "step": 997 + }, + { + "epoch": 2.7511218501898513, + "grad_norm": 0.25932177901268005, + "learning_rate": 1.0794015509572819e-07, + "log_odds_chosen": 1.7572764158248901, + "log_odds_ratio": -0.17751987278461456, + "logits/chosen": -0.7274531126022339, + "logits/rejected": -2.109034776687622, + "logps/chosen": -1.4595894813537598, + "logps/rejected": -3.003801107406616, + "loss": 1.5366, + "nll_loss": 1.518887996673584, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14595896005630493, + "rewards/margins": 0.1544211506843567, + "rewards/rejected": -0.300380140542984, + "step": 998 + }, + { + "epoch": 2.7538833275802554, + "grad_norm": 0.21890608966350555, + "learning_rate": 1.0562295828767388e-07, + "log_odds_chosen": 1.4044764041900635, + "log_odds_ratio": -0.223425954580307, + "logits/chosen": -0.8820509910583496, + "logits/rejected": -1.9093981981277466, + "logps/chosen": -1.4362064599990845, + "logps/rejected": -2.643540859222412, + "loss": 1.5029, + "nll_loss": 1.4805753231048584, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14362065494060516, + "rewards/margins": 0.12073343247175217, + "rewards/rejected": -0.26435407996177673, + "step": 999 + }, + { + "epoch": 2.7566448049706596, + "grad_norm": 0.24640429019927979, + "learning_rate": 1.0333036740834857e-07, + "log_odds_chosen": 1.58961820602417, + "log_odds_ratio": -0.19259408116340637, + "logits/chosen": -0.7218985557556152, + "logits/rejected": -1.7569760084152222, + "logps/chosen": -1.4714789390563965, + "logps/rejected": -2.855508804321289, + "loss": 1.5109, + "nll_loss": 1.4916527271270752, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14714789390563965, + "rewards/margins": 0.13840298354625702, + "rewards/rejected": -0.28555089235305786, + "step": 1000 + }, + { + "epoch": 2.7594062823610632, + "grad_norm": 0.21864457428455353, + "learning_rate": 1.0106240601767042e-07, + "log_odds_chosen": 2.0963706970214844, + "log_odds_ratio": -0.11976057291030884, + "logits/chosen": -0.8929874300956726, + "logits/rejected": -2.175584077835083, + "logps/chosen": -1.3230259418487549, + "logps/rejected": -3.150878667831421, + "loss": 1.3855, + "nll_loss": 1.373477578163147, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13230259716510773, + "rewards/margins": 0.1827852725982666, + "rewards/rejected": -0.3150879144668579, + "step": 1001 + }, + { + "epoch": 2.762167759751467, + "grad_norm": 0.22030071914196014, + "learning_rate": 9.881909742245177e-08, + "log_odds_chosen": 1.7982732057571411, + "log_odds_ratio": -0.17107604444026947, + "logits/chosen": -0.8717531561851501, + "logits/rejected": -1.9789412021636963, + "logps/chosen": -1.355002522468567, + "logps/rejected": -2.9149961471557617, + "loss": 1.4187, + "nll_loss": 1.4016332626342773, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1355002522468567, + "rewards/margins": 0.1559993475675583, + "rewards/rejected": -0.2914995849132538, + "step": 1002 + }, + { + "epoch": 2.764929237141871, + "grad_norm": 0.22139884531497955, + "learning_rate": 9.66004646761598e-08, + "log_odds_chosen": 1.4478939771652222, + "log_odds_ratio": -0.23130184412002563, + "logits/chosen": -0.765604555606842, + "logits/rejected": -1.7363425493240356, + "logps/chosen": -1.3936283588409424, + "logps/rejected": -2.6382076740264893, + "loss": 1.4489, + "nll_loss": 1.4257304668426514, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13936282694339752, + "rewards/margins": 0.1244579553604126, + "rewards/rejected": -0.2638207674026489, + "step": 1003 + }, + { + "epoch": 2.7676907145322747, + "grad_norm": 0.2260187566280365, + "learning_rate": 9.440653057867816e-08, + "log_odds_chosen": 1.5612225532531738, + "log_odds_ratio": -0.19809266924858093, + "logits/chosen": -0.8372408151626587, + "logits/rejected": -1.9978140592575073, + "logps/chosen": -1.4296715259552002, + "logps/rejected": -2.781604290008545, + "loss": 1.4996, + "nll_loss": 1.4798098802566528, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14296714961528778, + "rewards/margins": 0.13519325852394104, + "rewards/rejected": -0.27816042304039, + "step": 1004 + }, + { + "epoch": 2.7704521919226788, + "grad_norm": 0.23682570457458496, + "learning_rate": 9.223731767607436e-08, + "log_odds_chosen": 1.5558514595031738, + "log_odds_ratio": -0.20280753076076508, + "logits/chosen": -0.8458856344223022, + "logits/rejected": -1.9428819417953491, + "logps/chosen": -1.3925399780273438, + "logps/rejected": -2.7330634593963623, + "loss": 1.4611, + "nll_loss": 1.440795660018921, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13925398886203766, + "rewards/margins": 0.1340523660182953, + "rewards/rejected": -0.27330636978149414, + "step": 1005 + }, + { + "epoch": 2.7732136693130824, + "grad_norm": 0.29044103622436523, + "learning_rate": 9.00928482603669e-08, + "log_odds_chosen": 1.513688325881958, + "log_odds_ratio": -0.20887477695941925, + "logits/chosen": -0.6599283814430237, + "logits/rejected": -1.8435096740722656, + "logps/chosen": -1.4128738641738892, + "logps/rejected": -2.7181389331817627, + "loss": 1.4785, + "nll_loss": 1.4576369524002075, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1412874013185501, + "rewards/margins": 0.13052652776241302, + "rewards/rejected": -0.27181389927864075, + "step": 1006 + }, + { + "epoch": 2.7759751467034866, + "grad_norm": 0.21997316181659698, + "learning_rate": 8.797314436929744e-08, + "log_odds_chosen": 1.2918083667755127, + "log_odds_ratio": -0.25957056879997253, + "logits/chosen": -0.7696832418441772, + "logits/rejected": -1.4652957916259766, + "logps/chosen": -1.4523776769638062, + "logps/rejected": -2.558363437652588, + "loss": 1.5169, + "nll_loss": 1.4908944368362427, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1452377736568451, + "rewards/margins": 0.11059856414794922, + "rewards/rejected": -0.2558363378047943, + "step": 1007 + }, + { + "epoch": 2.77873662409389, + "grad_norm": 0.219607412815094, + "learning_rate": 8.587822778610284e-08, + "log_odds_chosen": 1.7893471717834473, + "log_odds_ratio": -0.1684584766626358, + "logits/chosen": -0.7286239266395569, + "logits/rejected": -1.910126805305481, + "logps/chosen": -1.3401432037353516, + "logps/rejected": -2.8861756324768066, + "loss": 1.4018, + "nll_loss": 1.3849496841430664, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1340143233537674, + "rewards/margins": 0.15460322797298431, + "rewards/rejected": -0.2886175513267517, + "step": 1008 + }, + { + "epoch": 2.781498101484294, + "grad_norm": 0.22155028581619263, + "learning_rate": 8.380812003929262e-08, + "log_odds_chosen": 1.4924676418304443, + "log_odds_ratio": -0.214213564991951, + "logits/chosen": -0.7699148058891296, + "logits/rejected": -2.101424217224121, + "logps/chosen": -1.4633634090423584, + "logps/rejected": -2.7596728801727295, + "loss": 1.4937, + "nll_loss": 1.4722706079483032, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1463363617658615, + "rewards/margins": 0.1296309381723404, + "rewards/rejected": -0.2759672999382019, + "step": 1009 + }, + { + "epoch": 2.784259578874698, + "grad_norm": 0.21873348951339722, + "learning_rate": 8.176284240242638e-08, + "log_odds_chosen": 1.6635732650756836, + "log_odds_ratio": -0.19265908002853394, + "logits/chosen": -0.7586681842803955, + "logits/rejected": -2.0201668739318848, + "logps/chosen": -1.3821016550064087, + "logps/rejected": -2.8210911750793457, + "loss": 1.4441, + "nll_loss": 1.4248504638671875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13821017742156982, + "rewards/margins": 0.14389893412590027, + "rewards/rejected": -0.2821091115474701, + "step": 1010 + }, + { + "epoch": 2.787021056265102, + "grad_norm": 0.24069830775260925, + "learning_rate": 7.97424158938967e-08, + "log_odds_chosen": 1.7478724718093872, + "log_odds_ratio": -0.1709989607334137, + "logits/chosen": -0.7697115540504456, + "logits/rejected": -2.1029648780822754, + "logps/chosen": -1.4904512166976929, + "logps/rejected": -3.030327320098877, + "loss": 1.5413, + "nll_loss": 1.5241514444351196, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14904512465000153, + "rewards/margins": 0.15398761630058289, + "rewards/rejected": -0.3030327558517456, + "step": 1011 + }, + { + "epoch": 2.7897825336555058, + "grad_norm": 0.25440898537635803, + "learning_rate": 7.774686127671183e-08, + "log_odds_chosen": 1.7158838510513306, + "log_odds_ratio": -0.17278915643692017, + "logits/chosen": -0.9012861251831055, + "logits/rejected": -1.9430004358291626, + "logps/chosen": -1.3608380556106567, + "logps/rejected": -2.839327096939087, + "loss": 1.4156, + "nll_loss": 1.398327350616455, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13608382642269135, + "rewards/margins": 0.1478489190340042, + "rewards/rejected": -0.28393274545669556, + "step": 1012 + }, + { + "epoch": 2.7925440110459094, + "grad_norm": 0.21382860839366913, + "learning_rate": 7.577619905828281e-08, + "log_odds_chosen": 1.4606471061706543, + "log_odds_ratio": -0.21766388416290283, + "logits/chosen": -0.7786560654640198, + "logits/rejected": -2.091545820236206, + "logps/chosen": -1.405806064605713, + "logps/rejected": -2.663357973098755, + "loss": 1.4767, + "nll_loss": 1.4549262523651123, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14058060944080353, + "rewards/margins": 0.1257551908493042, + "rewards/rejected": -0.26633578538894653, + "step": 1013 + }, + { + "epoch": 2.7953054884363135, + "grad_norm": 0.22583609819412231, + "learning_rate": 7.383044949021339e-08, + "log_odds_chosen": 1.6658389568328857, + "log_odds_ratio": -0.1828538179397583, + "logits/chosen": -0.7913579344749451, + "logits/rejected": -2.100635528564453, + "logps/chosen": -1.4507535696029663, + "logps/rejected": -2.9083504676818848, + "loss": 1.5045, + "nll_loss": 1.486259937286377, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14507536590099335, + "rewards/margins": 0.1457597017288208, + "rewards/rejected": -0.29083505272865295, + "step": 1014 + }, + { + "epoch": 2.798066965826717, + "grad_norm": 0.2718450129032135, + "learning_rate": 7.190963256809069e-08, + "log_odds_chosen": 1.8116415739059448, + "log_odds_ratio": -0.1589178889989853, + "logits/chosen": -0.8496946692466736, + "logits/rejected": -2.0155324935913086, + "logps/chosen": -1.2840626239776611, + "logps/rejected": -2.827091693878174, + "loss": 1.3719, + "nll_loss": 1.3559659719467163, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12840627133846283, + "rewards/margins": 0.15430289506912231, + "rewards/rejected": -0.28270918130874634, + "step": 1015 + }, + { + "epoch": 2.8008284432171213, + "grad_norm": 0.21330896019935608, + "learning_rate": 7.001376803128041e-08, + "log_odds_chosen": 1.8585755825042725, + "log_odds_ratio": -0.15236105024814606, + "logits/chosen": -0.8107438683509827, + "logits/rejected": -2.0344109535217285, + "logps/chosen": -1.3621731996536255, + "logps/rejected": -2.976593494415283, + "loss": 1.448, + "nll_loss": 1.432778000831604, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13621731102466583, + "rewards/margins": 0.16144204139709473, + "rewards/rejected": -0.29765933752059937, + "step": 1016 + }, + { + "epoch": 2.803589920607525, + "grad_norm": 0.20405304431915283, + "learning_rate": 6.81428753627239e-08, + "log_odds_chosen": 1.5870081186294556, + "log_odds_ratio": -0.20238448679447174, + "logits/chosen": -0.7854856848716736, + "logits/rejected": -2.0655314922332764, + "logps/chosen": -1.4296401739120483, + "logps/rejected": -2.8039541244506836, + "loss": 1.4973, + "nll_loss": 1.4770238399505615, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14296402037143707, + "rewards/margins": 0.13743142783641815, + "rewards/rejected": -0.2803954482078552, + "step": 1017 + }, + { + "epoch": 2.806351397997929, + "grad_norm": 0.2566664516925812, + "learning_rate": 6.629697378873839e-08, + "log_odds_chosen": 1.4977682828903198, + "log_odds_ratio": -0.21054419875144958, + "logits/chosen": -0.9586695432662964, + "logits/rejected": -1.8258790969848633, + "logps/chosen": -1.3797677755355835, + "logps/rejected": -2.663789987564087, + "loss": 1.4449, + "nll_loss": 1.4238519668579102, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1379767656326294, + "rewards/margins": 0.1284022480249405, + "rewards/rejected": -0.2663789987564087, + "step": 1018 + }, + { + "epoch": 2.8091128753883328, + "grad_norm": 0.22852636873722076, + "learning_rate": 6.447608227881791e-08, + "log_odds_chosen": 1.5595167875289917, + "log_odds_ratio": -0.2022312581539154, + "logits/chosen": -0.8290364146232605, + "logits/rejected": -1.985574722290039, + "logps/chosen": -1.3868674039840698, + "logps/rejected": -2.7301459312438965, + "loss": 1.4508, + "nll_loss": 1.4305788278579712, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13868674635887146, + "rewards/margins": 0.13432787358760834, + "rewards/rejected": -0.2730146050453186, + "step": 1019 + }, + { + "epoch": 2.8118743527787364, + "grad_norm": 0.21699006855487823, + "learning_rate": 6.268021954544095e-08, + "log_odds_chosen": 1.9818371534347534, + "log_odds_ratio": -0.15955258905887604, + "logits/chosen": -0.8705717325210571, + "logits/rejected": -2.0158958435058594, + "logps/chosen": -1.3820674419403076, + "logps/rejected": -3.1282825469970703, + "loss": 1.4534, + "nll_loss": 1.4374067783355713, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13820675015449524, + "rewards/margins": 0.17462150752544403, + "rewards/rejected": -0.31282827258110046, + "step": 1020 + }, + { + "epoch": 2.8146358301691405, + "grad_norm": 0.2339145392179489, + "learning_rate": 6.090940404387513e-08, + "log_odds_chosen": 1.4671416282653809, + "log_odds_ratio": -0.2181987166404724, + "logits/chosen": -0.7629361748695374, + "logits/rejected": -1.8477253913879395, + "logps/chosen": -1.452636480331421, + "logps/rejected": -2.7193386554718018, + "loss": 1.51, + "nll_loss": 1.488135814666748, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1452636420726776, + "rewards/margins": 0.12667018175125122, + "rewards/rejected": -0.2719338536262512, + "step": 1021 + }, + { + "epoch": 2.817397307559544, + "grad_norm": 0.2369954138994217, + "learning_rate": 5.9163653971989754e-08, + "log_odds_chosen": 1.9682278633117676, + "log_odds_ratio": -0.1359677016735077, + "logits/chosen": -0.8030321598052979, + "logits/rejected": -2.2195727825164795, + "logps/chosen": -1.3785918951034546, + "logps/rejected": -3.1034982204437256, + "loss": 1.4406, + "nll_loss": 1.4270315170288086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13785919547080994, + "rewards/margins": 0.17249061167240143, + "rewards/rejected": -0.31034982204437256, + "step": 1022 + }, + { + "epoch": 2.8201587849499483, + "grad_norm": 0.2538975179195404, + "learning_rate": 5.744298727006769e-08, + "log_odds_chosen": 1.4653068780899048, + "log_odds_ratio": -0.22388669848442078, + "logits/chosen": -0.8812580108642578, + "logits/rejected": -1.8244831562042236, + "logps/chosen": -1.4292573928833008, + "logps/rejected": -2.692967414855957, + "loss": 1.4846, + "nll_loss": 1.4622304439544678, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14292573928833008, + "rewards/margins": 0.12637099623680115, + "rewards/rejected": -0.2692967355251312, + "step": 1023 + }, + { + "epoch": 2.822920262340352, + "grad_norm": 0.22296157479286194, + "learning_rate": 5.574742162062163e-08, + "log_odds_chosen": 1.5828129053115845, + "log_odds_ratio": -0.20364131033420563, + "logits/chosen": -0.822142481803894, + "logits/rejected": -2.045734167098999, + "logps/chosen": -1.4172347784042358, + "logps/rejected": -2.793931722640991, + "loss": 1.4825, + "nll_loss": 1.4621087312698364, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14172348380088806, + "rewards/margins": 0.13766971230506897, + "rewards/rejected": -0.27939319610595703, + "step": 1024 + }, + { + "epoch": 2.825681739730756, + "grad_norm": 0.27316057682037354, + "learning_rate": 5.407697444821169e-08, + "log_odds_chosen": 1.5825798511505127, + "log_odds_ratio": -0.1950148344039917, + "logits/chosen": -0.8047593832015991, + "logits/rejected": -1.9445643424987793, + "logps/chosen": -1.3558223247528076, + "logps/rejected": -2.7116684913635254, + "loss": 1.439, + "nll_loss": 1.419493556022644, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13558222353458405, + "rewards/margins": 0.13558462262153625, + "rewards/rejected": -0.2711668610572815, + "step": 1025 + }, + { + "epoch": 2.8284432171211598, + "grad_norm": 0.22010549902915955, + "learning_rate": 5.2431662919267825e-08, + "log_odds_chosen": 1.6912816762924194, + "log_odds_ratio": -0.17304657399654388, + "logits/chosen": -0.8214058876037598, + "logits/rejected": -2.1822879314422607, + "logps/chosen": -1.488768219947815, + "logps/rejected": -2.977398157119751, + "loss": 1.5355, + "nll_loss": 1.5182044506072998, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1488768458366394, + "rewards/margins": 0.14886298775672913, + "rewards/rejected": -0.29773983359336853, + "step": 1026 + }, + { + "epoch": 2.8312046945115634, + "grad_norm": 0.23960784077644348, + "learning_rate": 5.0811503941911314e-08, + "log_odds_chosen": 1.5059239864349365, + "log_odds_ratio": -0.21391917765140533, + "logits/chosen": -0.7945892810821533, + "logits/rejected": -1.8174819946289062, + "logps/chosen": -1.4129031896591187, + "logps/rejected": -2.7106990814208984, + "loss": 1.4695, + "nll_loss": 1.448089599609375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1412903219461441, + "rewards/margins": 0.12977956235408783, + "rewards/rejected": -0.27106988430023193, + "step": 1027 + }, + { + "epoch": 2.8339661719019675, + "grad_norm": 0.23000836372375488, + "learning_rate": 4.9216514165781885e-08, + "log_odds_chosen": 1.7422019243240356, + "log_odds_ratio": -0.16638413071632385, + "logits/chosen": -0.7677799463272095, + "logits/rejected": -1.9751092195510864, + "logps/chosen": -1.39187753200531, + "logps/rejected": -2.9060842990875244, + "loss": 1.465, + "nll_loss": 1.4483141899108887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.139187753200531, + "rewards/margins": 0.15142066776752472, + "rewards/rejected": -0.2906084358692169, + "step": 1028 + }, + { + "epoch": 2.8367276492923716, + "grad_norm": 0.23794633150100708, + "learning_rate": 4.7646709981868376e-08, + "log_odds_chosen": 1.6807467937469482, + "log_odds_ratio": -0.18301939964294434, + "logits/chosen": -0.8502731323242188, + "logits/rejected": -1.98602294921875, + "logps/chosen": -1.3824533224105835, + "logps/rejected": -2.8385558128356934, + "loss": 1.4542, + "nll_loss": 1.4359116554260254, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1382453441619873, + "rewards/margins": 0.14561022818088531, + "rewards/rejected": -0.2838555872440338, + "step": 1029 + }, + { + "epoch": 2.8394891266827753, + "grad_norm": 0.231614887714386, + "learning_rate": 4.61021075223364e-08, + "log_odds_chosen": 1.7197368144989014, + "log_odds_ratio": -0.16804471611976624, + "logits/chosen": -0.9392529129981995, + "logits/rejected": -1.9837356805801392, + "logps/chosen": -1.3155676126480103, + "logps/rejected": -2.783620595932007, + "loss": 1.4019, + "nll_loss": 1.3851432800292969, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13155676424503326, + "rewards/margins": 0.1468052715063095, + "rewards/rejected": -0.2783620357513428, + "step": 1030 + }, + { + "epoch": 2.842250604073179, + "grad_norm": 0.24176815152168274, + "learning_rate": 4.458272266036706e-08, + "log_odds_chosen": 1.55967378616333, + "log_odds_ratio": -0.19546456634998322, + "logits/chosen": -0.762442946434021, + "logits/rejected": -1.9133505821228027, + "logps/chosen": -1.335386872291565, + "logps/rejected": -2.65338134765625, + "loss": 1.4057, + "nll_loss": 1.3861135244369507, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13353869318962097, + "rewards/margins": 0.13179941475391388, + "rewards/rejected": -0.26533812284469604, + "step": 1031 + }, + { + "epoch": 2.845012081463583, + "grad_norm": 0.23981979489326477, + "learning_rate": 4.308857100999042e-08, + "log_odds_chosen": 1.8310432434082031, + "log_odds_ratio": -0.16425098478794098, + "logits/chosen": -0.7952133417129517, + "logits/rejected": -1.92050302028656, + "logps/chosen": -1.368022084236145, + "logps/rejected": -2.9524054527282715, + "loss": 1.4436, + "nll_loss": 1.427202820777893, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13680219650268555, + "rewards/margins": 0.15843833982944489, + "rewards/rejected": -0.2952405512332916, + "step": 1032 + }, + { + "epoch": 2.8477735588539868, + "grad_norm": 0.21197573840618134, + "learning_rate": 4.161966792592592e-08, + "log_odds_chosen": 1.3388291597366333, + "log_odds_ratio": -0.24759265780448914, + "logits/chosen": -0.7649065256118774, + "logits/rejected": -1.754675030708313, + "logps/chosen": -1.348063349723816, + "logps/rejected": -2.4776344299316406, + "loss": 1.4147, + "nll_loss": 1.3899341821670532, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13480634987354279, + "rewards/margins": 0.11295709013938904, + "rewards/rejected": -0.24776342511177063, + "step": 1033 + }, + { + "epoch": 2.850535036244391, + "grad_norm": 0.22503337264060974, + "learning_rate": 4.017602850342584e-08, + "log_odds_chosen": 1.665643334388733, + "log_odds_ratio": -0.17374378442764282, + "logits/chosen": -0.7818294167518616, + "logits/rejected": -2.1388704776763916, + "logps/chosen": -1.4129152297973633, + "logps/rejected": -2.8565893173217773, + "loss": 1.4763, + "nll_loss": 1.458892583847046, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1412915140390396, + "rewards/margins": 0.14436742663383484, + "rewards/rejected": -0.28565895557403564, + "step": 1034 + }, + { + "epoch": 2.8532965136347945, + "grad_norm": 0.23232252895832062, + "learning_rate": 3.8757667578119e-08, + "log_odds_chosen": 1.9293830394744873, + "log_odds_ratio": -0.1441594660282135, + "logits/chosen": -0.7760025262832642, + "logits/rejected": -2.0618064403533936, + "logps/chosen": -1.413285493850708, + "logps/rejected": -3.106600761413574, + "loss": 1.477, + "nll_loss": 1.4625605344772339, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14132854342460632, + "rewards/margins": 0.16933153569698334, + "rewards/rejected": -0.31066006422042847, + "step": 1035 + }, + { + "epoch": 2.8560579910251986, + "grad_norm": 0.22999964654445648, + "learning_rate": 3.736459972585815e-08, + "log_odds_chosen": 1.4999421834945679, + "log_odds_ratio": -0.20854999125003815, + "logits/chosen": -0.7216120958328247, + "logits/rejected": -2.1589982509613037, + "logps/chosen": -1.503130555152893, + "logps/rejected": -2.8132858276367188, + "loss": 1.5602, + "nll_loss": 1.5393545627593994, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15031304955482483, + "rewards/margins": 0.13101550936698914, + "rewards/rejected": -0.28132855892181396, + "step": 1036 + }, + { + "epoch": 2.8588194684156023, + "grad_norm": 0.2181134819984436, + "learning_rate": 3.5996839262571194e-08, + "log_odds_chosen": 1.4268596172332764, + "log_odds_ratio": -0.2180023491382599, + "logits/chosen": -0.8290450572967529, + "logits/rejected": -1.8734112977981567, + "logps/chosen": -1.4272325038909912, + "logps/rejected": -2.6529884338378906, + "loss": 1.4948, + "nll_loss": 1.4729880094528198, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14272327721118927, + "rewards/margins": 0.12257558107376099, + "rewards/rejected": -0.26529884338378906, + "step": 1037 + }, + { + "epoch": 2.861580945806006, + "grad_norm": 0.21630226075649261, + "learning_rate": 3.4654400244112654e-08, + "log_odds_chosen": 1.6311062574386597, + "log_odds_ratio": -0.18956345319747925, + "logits/chosen": -0.7698126435279846, + "logits/rejected": -2.0631442070007324, + "logps/chosen": -1.4231253862380981, + "logps/rejected": -2.838050127029419, + "loss": 1.4917, + "nll_loss": 1.4727219343185425, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14231254160404205, + "rewards/margins": 0.14149242639541626, + "rewards/rejected": -0.2838050127029419, + "step": 1038 + }, + { + "epoch": 2.86434242319641, + "grad_norm": 0.2282114028930664, + "learning_rate": 3.333729646612077e-08, + "log_odds_chosen": 1.1922599077224731, + "log_odds_ratio": -0.28115126490592957, + "logits/chosen": -0.8158324956893921, + "logits/rejected": -1.5724602937698364, + "logps/chosen": -1.439221978187561, + "logps/rejected": -2.454829454421997, + "loss": 1.5126, + "nll_loss": 1.4844740629196167, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14392220973968506, + "rewards/margins": 0.10156073421239853, + "rewards/rejected": -0.245482936501503, + "step": 1039 + }, + { + "epoch": 2.867103900586814, + "grad_norm": 0.21230220794677734, + "learning_rate": 3.204554146387456e-08, + "log_odds_chosen": 1.720137357711792, + "log_odds_ratio": -0.173086479306221, + "logits/chosen": -0.8133577108383179, + "logits/rejected": -2.1605277061462402, + "logps/chosen": -1.3774471282958984, + "logps/rejected": -2.8622329235076904, + "loss": 1.4418, + "nll_loss": 1.4244989156723022, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1377447098493576, + "rewards/margins": 0.14847859740257263, + "rewards/rejected": -0.2862233519554138, + "step": 1040 + }, + { + "epoch": 2.869865377977218, + "grad_norm": 0.2199348360300064, + "learning_rate": 3.077914851215585e-08, + "log_odds_chosen": 1.7481780052185059, + "log_odds_ratio": -0.16739507019519806, + "logits/chosen": -0.8020718693733215, + "logits/rejected": -2.197960138320923, + "logps/chosen": -1.3972680568695068, + "logps/rejected": -2.9158308506011963, + "loss": 1.4411, + "nll_loss": 1.4243154525756836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13972681760787964, + "rewards/margins": 0.15185627341270447, + "rewards/rejected": -0.2915830910205841, + "step": 1041 + }, + { + "epoch": 2.8726268553676215, + "grad_norm": 0.22360913455486298, + "learning_rate": 2.9538130625110796e-08, + "log_odds_chosen": 1.7841436862945557, + "log_odds_ratio": -0.16358324885368347, + "logits/chosen": -0.7960190773010254, + "logits/rejected": -2.2209391593933105, + "logps/chosen": -1.4135005474090576, + "logps/rejected": -2.9724953174591064, + "loss": 1.4611, + "nll_loss": 1.4447667598724365, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14135007560253143, + "rewards/margins": 0.15589945018291473, + "rewards/rejected": -0.29724955558776855, + "step": 1042 + }, + { + "epoch": 2.8753883327580256, + "grad_norm": 0.2290942221879959, + "learning_rate": 2.8322500556118847e-08, + "log_odds_chosen": 1.751172661781311, + "log_odds_ratio": -0.16167476773262024, + "logits/chosen": -0.7484520673751831, + "logits/rejected": -1.920337200164795, + "logps/chosen": -1.4236546754837036, + "logps/rejected": -2.9533438682556152, + "loss": 1.4913, + "nll_loss": 1.475111961364746, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1423654705286026, + "rewards/margins": 0.15296894311904907, + "rewards/rejected": -0.2953343987464905, + "step": 1043 + }, + { + "epoch": 2.8781498101484293, + "grad_norm": 0.22796989977359772, + "learning_rate": 2.7132270797659567e-08, + "log_odds_chosen": 1.6315937042236328, + "log_odds_ratio": -0.18258602917194366, + "logits/chosen": -0.6966241002082825, + "logits/rejected": -2.115699052810669, + "logps/chosen": -1.3849658966064453, + "logps/rejected": -2.7913544178009033, + "loss": 1.4382, + "nll_loss": 1.4199258089065552, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13849659264087677, + "rewards/margins": 0.14063885807991028, + "rewards/rejected": -0.27913543581962585, + "step": 1044 + }, + { + "epoch": 2.8809112875388334, + "grad_norm": 0.23840682208538055, + "learning_rate": 2.5967453581185187e-08, + "log_odds_chosen": 1.6786651611328125, + "log_odds_ratio": -0.1893036663532257, + "logits/chosen": -0.6516211032867432, + "logits/rejected": -1.883230209350586, + "logps/chosen": -1.4420583248138428, + "logps/rejected": -2.913923501968384, + "loss": 1.524, + "nll_loss": 1.5050742626190186, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14420585334300995, + "rewards/margins": 0.1471865177154541, + "rewards/rejected": -0.29139238595962524, + "step": 1045 + }, + { + "epoch": 2.883672764929237, + "grad_norm": 0.23551054298877716, + "learning_rate": 2.4828060876995462e-08, + "log_odds_chosen": 1.4917556047439575, + "log_odds_ratio": -0.2092304825782776, + "logits/chosen": -0.7992951273918152, + "logits/rejected": -1.956207036972046, + "logps/chosen": -1.499756932258606, + "logps/rejected": -2.800715923309326, + "loss": 1.5506, + "nll_loss": 1.5296664237976074, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14997568726539612, + "rewards/margins": 0.13009591400623322, + "rewards/rejected": -0.2800716161727905, + "step": 1046 + }, + { + "epoch": 2.886434242319641, + "grad_norm": 0.21869802474975586, + "learning_rate": 2.371410439411248e-08, + "log_odds_chosen": 1.826915979385376, + "log_odds_ratio": -0.1546439379453659, + "logits/chosen": -0.864876925945282, + "logits/rejected": -2.1301004886627197, + "logps/chosen": -1.3602914810180664, + "logps/rejected": -2.9441142082214355, + "loss": 1.4152, + "nll_loss": 1.3997533321380615, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13602915406227112, + "rewards/margins": 0.15838226675987244, + "rewards/rejected": -0.29441142082214355, + "step": 1047 + }, + { + "epoch": 2.889195719710045, + "grad_norm": 0.22462975978851318, + "learning_rate": 2.262559558016325e-08, + "log_odds_chosen": 1.7097482681274414, + "log_odds_ratio": -0.1781245321035385, + "logits/chosen": -0.8304237723350525, + "logits/rejected": -2.108201265335083, + "logps/chosen": -1.3457623720169067, + "logps/rejected": -2.819329261779785, + "loss": 1.4148, + "nll_loss": 1.397024154663086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13457626104354858, + "rewards/margins": 0.14735668897628784, + "rewards/rejected": -0.2819329500198364, + "step": 1048 + }, + { + "epoch": 2.8919571971004485, + "grad_norm": 0.22334124147891998, + "learning_rate": 2.1562545621259534e-08, + "log_odds_chosen": 1.6744170188903809, + "log_odds_ratio": -0.1740560382604599, + "logits/chosen": -0.7684246301651001, + "logits/rejected": -1.8581528663635254, + "logps/chosen": -1.316230297088623, + "logps/rejected": -2.7437851428985596, + "loss": 1.4007, + "nll_loss": 1.3832471370697021, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1316230446100235, + "rewards/margins": 0.14275550842285156, + "rewards/rejected": -0.2743785083293915, + "step": 1049 + }, + { + "epoch": 2.8947186744908526, + "grad_norm": 0.22718796133995056, + "learning_rate": 2.052496544188487e-08, + "log_odds_chosen": 1.427244782447815, + "log_odds_ratio": -0.2224445343017578, + "logits/chosen": -0.7508878111839294, + "logits/rejected": -2.012288808822632, + "logps/chosen": -1.4001238346099854, + "logps/rejected": -2.6197235584259033, + "loss": 1.4666, + "nll_loss": 1.444330096244812, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14001239836215973, + "rewards/margins": 0.12195995450019836, + "rewards/rejected": -0.2619723379611969, + "step": 1050 + }, + { + "epoch": 2.8974801518812567, + "grad_norm": 0.24057117104530334, + "learning_rate": 1.9512865704780504e-08, + "log_odds_chosen": 1.6308375597000122, + "log_odds_ratio": -0.19242976605892181, + "logits/chosen": -0.7897810339927673, + "logits/rejected": -1.8339242935180664, + "logps/chosen": -1.4479725360870361, + "logps/rejected": -2.8747165203094482, + "loss": 1.5115, + "nll_loss": 1.4922785758972168, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14479725062847137, + "rewards/margins": 0.14267440140247345, + "rewards/rejected": -0.2874716520309448, + "step": 1051 + }, + { + "epoch": 2.9002416292716604, + "grad_norm": 0.24796703457832336, + "learning_rate": 1.852625681083742e-08, + "log_odds_chosen": 1.5950390100479126, + "log_odds_ratio": -0.20395228266716003, + "logits/chosen": -0.9735223054885864, + "logits/rejected": -2.0237746238708496, + "logps/chosen": -1.4253889322280884, + "logps/rejected": -2.802295207977295, + "loss": 1.501, + "nll_loss": 1.4805935621261597, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1425389051437378, + "rewards/margins": 0.13769064843654633, + "rewards/rejected": -0.2802295386791229, + "step": 1052 + }, + { + "epoch": 2.903003106662064, + "grad_norm": 0.25104832649230957, + "learning_rate": 1.7565148898988916e-08, + "log_odds_chosen": 1.5715587139129639, + "log_odds_ratio": -0.19690975546836853, + "logits/chosen": -0.8224033713340759, + "logits/rejected": -1.7730584144592285, + "logps/chosen": -1.4370899200439453, + "logps/rejected": -2.799690008163452, + "loss": 1.4958, + "nll_loss": 1.4761466979980469, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1437089890241623, + "rewards/margins": 0.1362600475549698, + "rewards/rejected": -0.2799690365791321, + "step": 1053 + }, + { + "epoch": 2.905764584052468, + "grad_norm": 0.22561626136302948, + "learning_rate": 1.6629551846104874e-08, + "log_odds_chosen": 1.4102277755737305, + "log_odds_ratio": -0.24327263236045837, + "logits/chosen": -0.7862374782562256, + "logits/rejected": -1.792546272277832, + "logps/chosen": -1.4360134601593018, + "logps/rejected": -2.6559813022613525, + "loss": 1.4941, + "nll_loss": 1.469818353652954, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14360132813453674, + "rewards/margins": 0.12199681997299194, + "rewards/rejected": -0.2655981481075287, + "step": 1054 + }, + { + "epoch": 2.908526061442872, + "grad_norm": 0.2336493730545044, + "learning_rate": 1.571947526689349e-08, + "log_odds_chosen": 1.5892236232757568, + "log_odds_ratio": -0.1958022266626358, + "logits/chosen": -0.8617162108421326, + "logits/rejected": -1.8406635522842407, + "logps/chosen": -1.4430006742477417, + "logps/rejected": -2.8268589973449707, + "loss": 1.5114, + "nll_loss": 1.4918040037155151, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14430007338523865, + "rewards/margins": 0.1383858323097229, + "rewards/rejected": -0.28268590569496155, + "step": 1055 + }, + { + "epoch": 2.911287538833276, + "grad_norm": 0.23663151264190674, + "learning_rate": 1.483492851379914e-08, + "log_odds_chosen": 1.7302970886230469, + "log_odds_ratio": -0.16981743276119232, + "logits/chosen": -0.8490622639656067, + "logits/rejected": -2.0886123180389404, + "logps/chosen": -1.3956736326217651, + "logps/rejected": -2.897519588470459, + "loss": 1.4492, + "nll_loss": 1.4322402477264404, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13956737518310547, + "rewards/margins": 0.15018458664417267, + "rewards/rejected": -0.28975194692611694, + "step": 1056 + }, + { + "epoch": 2.9140490162236796, + "grad_norm": 0.24135632812976837, + "learning_rate": 1.3975920676908838e-08, + "log_odds_chosen": 1.4057644605636597, + "log_odds_ratio": -0.22804616391658783, + "logits/chosen": -0.6639127135276794, + "logits/rejected": -1.8646632432937622, + "logps/chosen": -1.440869688987732, + "logps/rejected": -2.6512222290039062, + "loss": 1.5137, + "nll_loss": 1.4908647537231445, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14408698678016663, + "rewards/margins": 0.12103521823883057, + "rewards/rejected": -0.2651222050189972, + "step": 1057 + }, + { + "epoch": 2.9168104936140837, + "grad_norm": 0.2815915644168854, + "learning_rate": 1.3142460583856487e-08, + "log_odds_chosen": 1.5973938703536987, + "log_odds_ratio": -0.18859460949897766, + "logits/chosen": -0.8560981750488281, + "logits/rejected": -1.7154216766357422, + "logps/chosen": -1.403993010520935, + "logps/rejected": -2.7832603454589844, + "loss": 1.4698, + "nll_loss": 1.4509804248809814, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1403992921113968, + "rewards/margins": 0.13792674243450165, + "rewards/rejected": -0.27832603454589844, + "step": 1058 + }, + { + "epoch": 2.9195719710044874, + "grad_norm": 0.26606282591819763, + "learning_rate": 1.2334556799734887e-08, + "log_odds_chosen": 1.3971562385559082, + "log_odds_ratio": -0.2267698049545288, + "logits/chosen": -0.8830655217170715, + "logits/rejected": -1.6509592533111572, + "logps/chosen": -1.3806607723236084, + "logps/rejected": -2.5692031383514404, + "loss": 1.4562, + "nll_loss": 1.4335050582885742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13806608319282532, + "rewards/margins": 0.11885424703359604, + "rewards/rejected": -0.25692033767700195, + "step": 1059 + }, + { + "epoch": 2.922333448394891, + "grad_norm": 0.22054573893547058, + "learning_rate": 1.1552217627004426e-08, + "log_odds_chosen": 1.5220046043395996, + "log_odds_ratio": -0.2108788937330246, + "logits/chosen": -0.7800368666648865, + "logits/rejected": -1.8905466794967651, + "logps/chosen": -1.3639496564865112, + "logps/rejected": -2.6639370918273926, + "loss": 1.4512, + "nll_loss": 1.4301084280014038, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13639497756958008, + "rewards/margins": 0.12999877333641052, + "rewards/rejected": -0.2663937211036682, + "step": 1060 + }, + { + "epoch": 2.925094925785295, + "grad_norm": 0.2264561504125595, + "learning_rate": 1.079545110541147e-08, + "log_odds_chosen": 1.7285921573638916, + "log_odds_ratio": -0.173441544175148, + "logits/chosen": -0.7664073705673218, + "logits/rejected": -2.037234306335449, + "logps/chosen": -1.389496088027954, + "logps/rejected": -2.8904924392700195, + "loss": 1.4607, + "nll_loss": 1.4433540105819702, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13894961774349213, + "rewards/margins": 0.15009962022304535, + "rewards/rejected": -0.2890492379665375, + "step": 1061 + }, + { + "epoch": 2.927856403175699, + "grad_norm": 0.214057058095932, + "learning_rate": 1.006426501190233e-08, + "log_odds_chosen": 1.740164875984192, + "log_odds_ratio": -0.1689753532409668, + "logits/chosen": -0.7992459535598755, + "logits/rejected": -2.1515536308288574, + "logps/chosen": -1.3622679710388184, + "logps/rejected": -2.8680598735809326, + "loss": 1.4201, + "nll_loss": 1.4032260179519653, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13622678816318512, + "rewards/margins": 0.15057921409606934, + "rewards/rejected": -0.28680598735809326, + "step": 1062 + }, + { + "epoch": 2.930617880566103, + "grad_norm": 0.23196722567081451, + "learning_rate": 9.358666860545817e-09, + "log_odds_chosen": 1.8300212621688843, + "log_odds_ratio": -0.1529330015182495, + "logits/chosen": -0.7448672652244568, + "logits/rejected": -2.0953400135040283, + "logps/chosen": -1.3911778926849365, + "logps/rejected": -2.9877777099609375, + "loss": 1.449, + "nll_loss": 1.4337555170059204, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1391177773475647, + "rewards/margins": 0.1596599817276001, + "rewards/rejected": -0.2987777292728424, + "step": 1063 + }, + { + "epoch": 2.9333793579565066, + "grad_norm": 0.23241263628005981, + "learning_rate": 8.67866390245442e-09, + "log_odds_chosen": 1.4628114700317383, + "log_odds_ratio": -0.2144569754600525, + "logits/chosen": -0.8802769184112549, + "logits/rejected": -1.8510754108428955, + "logps/chosen": -1.3962527513504028, + "logps/rejected": -2.648005723953247, + "loss": 1.4754, + "nll_loss": 1.4539076089859009, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13962529599666595, + "rewards/margins": 0.12517526745796204, + "rewards/rejected": -0.2648005783557892, + "step": 1064 + }, + { + "epoch": 2.9361408353469107, + "grad_norm": 0.2281099408864975, + "learning_rate": 8.024263125710751e-09, + "log_odds_chosen": 1.6584781408309937, + "log_odds_ratio": -0.19228924810886383, + "logits/chosen": -0.8245370388031006, + "logits/rejected": -2.0083014965057373, + "logps/chosen": -1.4000036716461182, + "logps/rejected": -2.8395965099334717, + "loss": 1.4715, + "nll_loss": 1.452258586883545, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1400003731250763, + "rewards/margins": 0.14395931363105774, + "rewards/rejected": -0.28395968675613403, + "step": 1065 + }, + { + "epoch": 2.9389023127373144, + "grad_norm": 0.22550606727600098, + "learning_rate": 7.3954712552953835e-09, + "log_odds_chosen": 1.671879768371582, + "log_odds_ratio": -0.18240472674369812, + "logits/chosen": -0.7968007326126099, + "logits/rejected": -2.037839889526367, + "logps/chosen": -1.4791769981384277, + "logps/rejected": -2.9491400718688965, + "loss": 1.5383, + "nll_loss": 1.5200833082199097, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14791768789291382, + "rewards/margins": 0.14699633419513702, + "rewards/rejected": -0.29491403698921204, + "step": 1066 + }, + { + "epoch": 2.941663790127718, + "grad_norm": 0.2156606912612915, + "learning_rate": 6.792294753017181e-09, + "log_odds_chosen": 1.5692713260650635, + "log_odds_ratio": -0.21755948662757874, + "logits/chosen": -0.8175379633903503, + "logits/rejected": -1.7547521591186523, + "logps/chosen": -1.3667752742767334, + "logps/rejected": -2.7211294174194336, + "loss": 1.4277, + "nll_loss": 1.4059330224990845, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13667753338813782, + "rewards/margins": 0.13543540239334106, + "rewards/rejected": -0.2721129357814789, + "step": 1067 + }, + { + "epoch": 2.944425267518122, + "grad_norm": 0.2185620218515396, + "learning_rate": 6.214739817448634e-09, + "log_odds_chosen": 1.5329852104187012, + "log_odds_ratio": -0.22458839416503906, + "logits/chosen": -0.7700731754302979, + "logits/rejected": -1.848755955696106, + "logps/chosen": -1.4170128107070923, + "logps/rejected": -2.7483105659484863, + "loss": 1.4819, + "nll_loss": 1.459450125694275, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14170128107070923, + "rewards/margins": 0.1331297755241394, + "rewards/rejected": -0.27483102679252625, + "step": 1068 + }, + { + "epoch": 2.9471867449085263, + "grad_norm": 0.22919808328151703, + "learning_rate": 5.662812383859795e-09, + "log_odds_chosen": 1.7428557872772217, + "log_odds_ratio": -0.16942371428012848, + "logits/chosen": -0.7553185224533081, + "logits/rejected": -1.9580934047698975, + "logps/chosen": -1.3186566829681396, + "logps/rejected": -2.807377576828003, + "loss": 1.3891, + "nll_loss": 1.3721272945404053, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1318656951189041, + "rewards/margins": 0.14887209236621857, + "rewards/rejected": -0.2807377576828003, + "step": 1069 + }, + { + "epoch": 2.94994822229893, + "grad_norm": 0.20699773728847504, + "learning_rate": 5.136518124159162e-09, + "log_odds_chosen": 1.5876624584197998, + "log_odds_ratio": -0.2104184627532959, + "logits/chosen": -0.7132779359817505, + "logits/rejected": -1.9351344108581543, + "logps/chosen": -1.360245704650879, + "logps/rejected": -2.7215259075164795, + "loss": 1.4345, + "nll_loss": 1.413459062576294, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1360245794057846, + "rewards/margins": 0.1361280381679535, + "rewards/rejected": -0.2721526324748993, + "step": 1070 + }, + { + "epoch": 2.9527096996893336, + "grad_norm": 0.2266066074371338, + "learning_rate": 4.63586244683456e-09, + "log_odds_chosen": 1.3620774745941162, + "log_odds_ratio": -0.23232533037662506, + "logits/chosen": -0.811676025390625, + "logits/rejected": -1.820404052734375, + "logps/chosen": -1.4501413106918335, + "logps/rejected": -2.6185903549194336, + "loss": 1.5106, + "nll_loss": 1.487402319908142, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14501415193080902, + "rewards/margins": 0.11684489995241165, + "rewards/rejected": -0.2618590295314789, + "step": 1071 + }, + { + "epoch": 2.9554711770797377, + "grad_norm": 0.2652856111526489, + "learning_rate": 4.160850496897906e-09, + "log_odds_chosen": 1.6856169700622559, + "log_odds_ratio": -0.1874770075082779, + "logits/chosen": -0.8645150661468506, + "logits/rejected": -1.93236243724823, + "logps/chosen": -1.4419993162155151, + "logps/rejected": -2.9198200702667236, + "loss": 1.5035, + "nll_loss": 1.4847571849822998, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.144199937582016, + "rewards/margins": 0.1477820724248886, + "rewards/rejected": -0.2919819951057434, + "step": 1072 + }, + { + "epoch": 2.9582326544701414, + "grad_norm": 0.21664103865623474, + "learning_rate": 3.7114871558313614e-09, + "log_odds_chosen": 1.7259833812713623, + "log_odds_ratio": -0.1650668829679489, + "logits/chosen": -0.8545508980751038, + "logits/rejected": -2.1027960777282715, + "logps/chosen": -1.424027681350708, + "logps/rejected": -2.9264843463897705, + "loss": 1.4887, + "nll_loss": 1.4721843004226685, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1424027681350708, + "rewards/margins": 0.15024566650390625, + "rewards/rejected": -0.29264843463897705, + "step": 1073 + }, + { + "epoch": 2.9609941318605455, + "grad_norm": 0.22490212321281433, + "learning_rate": 3.287777041539042e-09, + "log_odds_chosen": 1.437325119972229, + "log_odds_ratio": -0.22529327869415283, + "logits/chosen": -0.8498947620391846, + "logits/rejected": -1.7718011140823364, + "logps/chosen": -1.3232507705688477, + "logps/rejected": -2.540151596069336, + "loss": 1.4037, + "nll_loss": 1.3812105655670166, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13232508301734924, + "rewards/margins": 0.12169007956981659, + "rewards/rejected": -0.25401514768600464, + "step": 1074 + }, + { + "epoch": 2.963755609250949, + "grad_norm": 0.2170701026916504, + "learning_rate": 2.8897245082978863e-09, + "log_odds_chosen": 1.7161061763763428, + "log_odds_ratio": -0.17984658479690552, + "logits/chosen": -0.8351077437400818, + "logits/rejected": -2.1513471603393555, + "logps/chosen": -1.3885916471481323, + "logps/rejected": -2.882119655609131, + "loss": 1.4441, + "nll_loss": 1.426088571548462, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13885916769504547, + "rewards/margins": 0.1493528038263321, + "rewards/rejected": -0.2882119417190552, + "step": 1075 + }, + { + "epoch": 2.9665170866413533, + "grad_norm": 0.22006747126579285, + "learning_rate": 2.5173336467135266e-09, + "log_odds_chosen": 1.820219874382019, + "log_odds_ratio": -0.15693305432796478, + "logits/chosen": -0.8700850009918213, + "logits/rejected": -2.169057607650757, + "logps/chosen": -1.352325201034546, + "logps/rejected": -2.9299466609954834, + "loss": 1.4083, + "nll_loss": 1.392561435699463, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13523253798484802, + "rewards/margins": 0.15776214003562927, + "rewards/rejected": -0.2929946780204773, + "step": 1076 + }, + { + "epoch": 2.969278564031757, + "grad_norm": 0.2348693460226059, + "learning_rate": 2.170608283677822e-09, + "log_odds_chosen": 1.7549494504928589, + "log_odds_ratio": -0.1712295114994049, + "logits/chosen": -0.9456162452697754, + "logits/rejected": -1.9611583948135376, + "logps/chosen": -1.3380199670791626, + "logps/rejected": -2.8467626571655273, + "loss": 1.4134, + "nll_loss": 1.3962781429290771, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13380199670791626, + "rewards/margins": 0.1508743017911911, + "rewards/rejected": -0.28467628359794617, + "step": 1077 + }, + { + "epoch": 2.9720400414221606, + "grad_norm": 0.23374752700328827, + "learning_rate": 1.8495519823308329e-09, + "log_odds_chosen": 1.8946998119354248, + "log_odds_ratio": -0.1446288675069809, + "logits/chosen": -0.8362556099891663, + "logits/rejected": -2.1337828636169434, + "logps/chosen": -1.4020692110061646, + "logps/rejected": -3.0605435371398926, + "loss": 1.4662, + "nll_loss": 1.4517593383789062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14020691812038422, + "rewards/margins": 0.16584742069244385, + "rewards/rejected": -0.30605435371398926, + "step": 1078 + }, + { + "epoch": 2.9748015188125647, + "grad_norm": 0.2259787917137146, + "learning_rate": 1.5541680420227968e-09, + "log_odds_chosen": 1.4082696437835693, + "log_odds_ratio": -0.22261746227741241, + "logits/chosen": -0.9028100371360779, + "logits/rejected": -1.8519865274429321, + "logps/chosen": -1.3738371133804321, + "logps/rejected": -2.56370210647583, + "loss": 1.4394, + "nll_loss": 1.4171141386032104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13738371431827545, + "rewards/margins": 0.11898650228977203, + "rewards/rejected": -0.2563702166080475, + "step": 1079 + }, + { + "epoch": 2.977562996202969, + "grad_norm": 0.22851480543613434, + "learning_rate": 1.284459498280266e-09, + "log_odds_chosen": 1.6992191076278687, + "log_odds_ratio": -0.1761513650417328, + "logits/chosen": -0.7371349334716797, + "logits/rejected": -1.8762682676315308, + "logps/chosen": -1.3590335845947266, + "logps/rejected": -2.823000431060791, + "loss": 1.4165, + "nll_loss": 1.3989176750183105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13590337336063385, + "rewards/margins": 0.1463966816663742, + "rewards/rejected": -0.28230005502700806, + "step": 1080 + }, + { + "epoch": 2.9803244735933725, + "grad_norm": 0.22858591377735138, + "learning_rate": 1.0404291227764097e-09, + "log_odds_chosen": 1.6860997676849365, + "log_odds_ratio": -0.1761903464794159, + "logits/chosen": -0.8258135914802551, + "logits/rejected": -2.1534647941589355, + "logps/chosen": -1.435535192489624, + "logps/rejected": -2.90619158744812, + "loss": 1.4886, + "nll_loss": 1.4709571599960327, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14355352520942688, + "rewards/margins": 0.1470656543970108, + "rewards/rejected": -0.2906191945075989, + "step": 1081 + }, + { + "epoch": 2.983085950983776, + "grad_norm": 0.2309102714061737, + "learning_rate": 8.220794233004814e-10, + "log_odds_chosen": 1.8477295637130737, + "log_odds_ratio": -0.17313013970851898, + "logits/chosen": -0.8398474454879761, + "logits/rejected": -1.861135721206665, + "logps/chosen": -1.325697898864746, + "logps/rejected": -2.925485134124756, + "loss": 1.41, + "nll_loss": 1.3926753997802734, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13256977498531342, + "rewards/margins": 0.1599787175655365, + "rewards/rejected": -0.2925485074520111, + "step": 1082 + }, + { + "epoch": 2.9858474283741803, + "grad_norm": 0.2285986989736557, + "learning_rate": 6.294126437336734e-10, + "log_odds_chosen": 1.8615604639053345, + "log_odds_ratio": -0.14680801331996918, + "logits/chosen": -0.87724369764328, + "logits/rejected": -2.150709390640259, + "logps/chosen": -1.4179985523223877, + "logps/rejected": -3.0496137142181396, + "loss": 1.476, + "nll_loss": 1.4613220691680908, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14179985225200653, + "rewards/margins": 0.1631615310907364, + "rewards/rejected": -0.3049613833427429, + "step": 1083 + }, + { + "epoch": 2.988608905764584, + "grad_norm": 0.21875019371509552, + "learning_rate": 4.624307640249681e-10, + "log_odds_chosen": 1.664400339126587, + "log_odds_ratio": -0.18604637682437897, + "logits/chosen": -0.8599823117256165, + "logits/rejected": -2.165078639984131, + "logps/chosen": -1.4403799772262573, + "logps/rejected": -2.8875458240509033, + "loss": 1.4962, + "nll_loss": 1.4775702953338623, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14403800666332245, + "rewards/margins": 0.14471659064292908, + "rewards/rejected": -0.2887546122074127, + "step": 1084 + }, + { + "epoch": 2.991370383154988, + "grad_norm": 0.21976594626903534, + "learning_rate": 3.2113550017198734e-10, + "log_odds_chosen": 1.7429512739181519, + "log_odds_ratio": -0.17201046645641327, + "logits/chosen": -0.8370999097824097, + "logits/rejected": -2.065781831741333, + "logps/chosen": -1.3308002948760986, + "logps/rejected": -2.826251268386841, + "loss": 1.4156, + "nll_loss": 1.3984227180480957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13308003544807434, + "rewards/margins": 0.1495451182126999, + "rewards/rejected": -0.28262513875961304, + "step": 1085 + }, + { + "epoch": 2.9941318605453917, + "grad_norm": 0.22135642170906067, + "learning_rate": 2.0552830420184077e-10, + "log_odds_chosen": 1.7063344717025757, + "log_odds_ratio": -0.2038329541683197, + "logits/chosen": -0.7838236093521118, + "logits/rejected": -1.8901429176330566, + "logps/chosen": -1.2961207628250122, + "logps/rejected": -2.761850357055664, + "loss": 1.3611, + "nll_loss": 1.34067964553833, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12961207330226898, + "rewards/margins": 0.14657296240329742, + "rewards/rejected": -0.2761850357055664, + "step": 1086 + }, + { + "epoch": 2.996893337935796, + "grad_norm": 0.21547023952007294, + "learning_rate": 1.1561036415752568e-10, + "log_odds_chosen": 1.8393921852111816, + "log_odds_ratio": -0.16761940717697144, + "logits/chosen": -0.7874873280525208, + "logits/rejected": -1.8532605171203613, + "logps/chosen": -1.335709571838379, + "logps/rejected": -2.9297051429748535, + "loss": 1.4178, + "nll_loss": 1.4010545015335083, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13357096910476685, + "rewards/margins": 0.15939952433109283, + "rewards/rejected": -0.2929704785346985, + "step": 1087 + }, + { + "epoch": 2.9996548153261995, + "grad_norm": 0.21358518302440643, + "learning_rate": 5.138260408488194e-11, + "log_odds_chosen": 1.7693160772323608, + "log_odds_ratio": -0.1748848706483841, + "logits/chosen": -0.7919652462005615, + "logits/rejected": -2.1071107387542725, + "logps/chosen": -1.4612958431243896, + "logps/rejected": -3.023159980773926, + "loss": 1.5195, + "nll_loss": 1.501983642578125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1461295783519745, + "rewards/margins": 0.15618643164634705, + "rewards/rejected": -0.30231598019599915, + "step": 1088 + }, + { + "epoch": 3.0, + "grad_norm": 0.5550550222396851, + "learning_rate": 1.2845684023432559e-11, + "log_odds_chosen": 2.2474207878112793, + "log_odds_ratio": -0.10045277327299118, + "logits/chosen": -0.9216854572296143, + "logits/rejected": -1.9990668296813965, + "logps/chosen": -1.146590232849121, + "logps/rejected": -3.0597341060638428, + "loss": 1.224, + "nll_loss": 1.2139885425567627, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11465902626514435, + "rewards/margins": 0.19131438434123993, + "rewards/rejected": -0.3059734106063843, + "step": 1089 + } + ], + "logging_steps": 1, + "max_steps": 1089, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}