{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 2.3792731903222646, "eval_steps": 35, "global_step": 1520, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.023508668821627974, "grad_norm": 66.55298614501953, "learning_rate": 9.333333333333334e-08, "logits/chosen": -0.8181892037391663, "logits/rejected": -0.6109388470649719, "logps/chosen": -360.2866516113281, "logps/rejected": -290.2643127441406, "loss": 0.6916, "rewards/accuracies": 0.4333333373069763, "rewards/chosen": 0.0018199360929429531, "rewards/margins": 0.004990327637642622, "rewards/rejected": -0.0031703924760222435, "step": 15 }, { "epoch": 0.04701733764325595, "grad_norm": 80.19271850585938, "learning_rate": 1.9333333333333332e-07, "logits/chosen": -0.9239326119422913, "logits/rejected": -0.6111201643943787, "logps/chosen": -361.14398193359375, "logps/rejected": -279.4661865234375, "loss": 0.6957, "rewards/accuracies": 0.4791666567325592, "rewards/chosen": -0.0014068834716454148, "rewards/margins": -0.002790238242596388, "rewards/rejected": 0.0013833552366122603, "step": 30 }, { "epoch": 0.05485356058379861, "eval_logits/chosen": -0.9112520217895508, "eval_logits/rejected": -0.6521649360656738, "eval_logps/chosen": -352.1986999511719, "eval_logps/rejected": -285.4880065917969, "eval_loss": 0.6858627796173096, "eval_rewards/accuracies": 0.5549542307853699, "eval_rewards/chosen": 0.011523518711328506, "eval_rewards/margins": 0.017005058005452156, "eval_rewards/rejected": -0.005481537897139788, "eval_runtime": 181.0795, "eval_samples_per_second": 13.265, "eval_steps_per_second": 6.632, "step": 35 }, { "epoch": 0.07052600646488393, "grad_norm": 61.10358428955078, "learning_rate": 2.933333333333333e-07, "logits/chosen": -0.983363926410675, "logits/rejected": -0.6932616829872131, "logps/chosen": -343.0465393066406, "logps/rejected": -297.50042724609375, "loss": 0.6847, "rewards/accuracies": 0.5562499761581421, "rewards/chosen": 0.012528100982308388, "rewards/margins": 0.019641580060124397, "rewards/rejected": -0.0071134804747998714, "step": 45 }, { "epoch": 0.0940346752865119, "grad_norm": 62.27146911621094, "learning_rate": 3.933333333333333e-07, "logits/chosen": -0.923626720905304, "logits/rejected": -0.6753237247467041, "logps/chosen": -379.0172119140625, "logps/rejected": -298.9611511230469, "loss": 0.6765, "rewards/accuracies": 0.6041666865348816, "rewards/chosen": 0.024433977901935577, "rewards/margins": 0.038544632494449615, "rewards/rejected": -0.014110651798546314, "step": 60 }, { "epoch": 0.10970712116759722, "eval_logits/chosen": -0.8566208481788635, "eval_logits/rejected": -0.5961222648620605, "eval_logps/chosen": -351.8851318359375, "eval_logps/rejected": -285.7581481933594, "eval_loss": 0.6613612771034241, "eval_rewards/accuracies": 0.6353039145469666, "eval_rewards/chosen": 0.04288659617304802, "eval_rewards/margins": 0.07538340985774994, "eval_rewards/rejected": -0.03249681368470192, "eval_runtime": 181.1266, "eval_samples_per_second": 13.261, "eval_steps_per_second": 6.631, "step": 70 }, { "epoch": 0.11754334410813988, "grad_norm": 60.890411376953125, "learning_rate": 4.933333333333333e-07, "logits/chosen": -0.8443899154663086, "logits/rejected": -0.5239131450653076, "logps/chosen": -340.6385803222656, "logps/rejected": -286.01763916015625, "loss": 0.6611, "rewards/accuracies": 0.6520833373069763, "rewards/chosen": 0.03822815418243408, "rewards/margins": 0.07544167339801788, "rewards/rejected": -0.0372135154902935, "step": 75 }, { "epoch": 0.14105201292976785, "grad_norm": 61.06071853637695, "learning_rate": 5.933333333333334e-07, "logits/chosen": -0.8115103840827942, "logits/rejected": -0.5216272473335266, "logps/chosen": -354.4888916015625, "logps/rejected": -267.555419921875, "loss": 0.637, "rewards/accuracies": 0.6895833611488342, "rewards/chosen": 0.05075030028820038, "rewards/margins": 0.13897046446800232, "rewards/rejected": -0.08822017163038254, "step": 90 }, { "epoch": 0.16456068175139582, "grad_norm": 56.0710563659668, "learning_rate": 6.933333333333333e-07, "logits/chosen": -0.8203279972076416, "logits/rejected": -0.4818764925003052, "logps/chosen": -363.0785827636719, "logps/rejected": -297.43756103515625, "loss": 0.6295, "rewards/accuracies": 0.6729166507720947, "rewards/chosen": 0.053048841655254364, "rewards/margins": 0.17447903752326965, "rewards/rejected": -0.12143018841743469, "step": 105 }, { "epoch": 0.16456068175139582, "eval_logits/chosen": -0.8201783895492554, "eval_logits/rejected": -0.5542652606964111, "eval_logps/chosen": -351.9151916503906, "eval_logps/rejected": -286.86541748046875, "eval_loss": 0.6280434131622314, "eval_rewards/accuracies": 0.6615320444107056, "eval_rewards/chosen": 0.0398729108273983, "eval_rewards/margins": 0.18310010433197021, "eval_rewards/rejected": -0.1432272046804428, "eval_runtime": 181.1366, "eval_samples_per_second": 13.261, "eval_steps_per_second": 6.63, "step": 105 }, { "epoch": 0.1880693505730238, "grad_norm": 54.890419006347656, "learning_rate": 7.933333333333333e-07, "logits/chosen": -0.8400412201881409, "logits/rejected": -0.5213409662246704, "logps/chosen": -365.0088195800781, "logps/rejected": -308.81427001953125, "loss": 0.6043, "rewards/accuracies": 0.7104166746139526, "rewards/chosen": 0.053323231637477875, "rewards/margins": 0.24515444040298462, "rewards/rejected": -0.19183121621608734, "step": 120 }, { "epoch": 0.2115780193946518, "grad_norm": 60.136470794677734, "learning_rate": 8.933333333333333e-07, "logits/chosen": -0.8502958416938782, "logits/rejected": -0.5787886381149292, "logps/chosen": -350.45751953125, "logps/rejected": -284.59967041015625, "loss": 0.6126, "rewards/accuracies": 0.6854166388511658, "rewards/chosen": 0.02512473240494728, "rewards/margins": 0.2510988712310791, "rewards/rejected": -0.22597412765026093, "step": 135 }, { "epoch": 0.21941424233519444, "eval_logits/chosen": -0.8695166707038879, "eval_logits/rejected": -0.6029744148254395, "eval_logps/chosen": -351.90997314453125, "eval_logps/rejected": -287.88140869140625, "eval_loss": 0.5991479158401489, "eval_rewards/accuracies": 0.6965029239654541, "eval_rewards/chosen": 0.0403980053961277, "eval_rewards/margins": 0.28522104024887085, "eval_rewards/rejected": -0.24482303857803345, "eval_runtime": 181.2258, "eval_samples_per_second": 13.254, "eval_steps_per_second": 6.627, "step": 140 }, { "epoch": 0.23508668821627976, "grad_norm": 64.90687561035156, "learning_rate": 9.933333333333333e-07, "logits/chosen": -0.8242729306221008, "logits/rejected": -0.5118594765663147, "logps/chosen": -360.62066650390625, "logps/rejected": -290.8011779785156, "loss": 0.6232, "rewards/accuracies": 0.6604166626930237, "rewards/chosen": 0.005629782099276781, "rewards/margins": 0.2206929475069046, "rewards/rejected": -0.2150631695985794, "step": 150 }, { "epoch": 0.2585953570379077, "grad_norm": 55.48398208618164, "learning_rate": 9.998451182298019e-07, "logits/chosen": -0.903767466545105, "logits/rejected": -0.5809265375137329, "logps/chosen": -338.2618713378906, "logps/rejected": -285.73797607421875, "loss": 0.6014, "rewards/accuracies": 0.6770833134651184, "rewards/chosen": 0.0597819946706295, "rewards/margins": 0.28081849217414856, "rewards/rejected": -0.22103647887706757, "step": 165 }, { "epoch": 0.27426780291899305, "eval_logits/chosen": -0.9308104515075684, "eval_logits/rejected": -0.6681538224220276, "eval_logps/chosen": -351.89520263671875, "eval_logps/rejected": -288.6672058105469, "eval_loss": 0.5756543874740601, "eval_rewards/accuracies": 0.7173188924789429, "eval_rewards/chosen": 0.04187745228409767, "eval_rewards/margins": 0.36528098583221436, "eval_rewards/rejected": -0.32340356707572937, "eval_runtime": 181.4158, "eval_samples_per_second": 13.24, "eval_steps_per_second": 6.62, "step": 175 }, { "epoch": 0.2821040258595357, "grad_norm": 53.91843032836914, "learning_rate": 9.993355436786068e-07, "logits/chosen": -0.9650304317474365, "logits/rejected": -0.5752300024032593, "logps/chosen": -352.0422668457031, "logps/rejected": -305.9342041015625, "loss": 0.5756, "rewards/accuracies": 0.7145833373069763, "rewards/chosen": 0.0450352244079113, "rewards/margins": 0.34943825006484985, "rewards/rejected": -0.3044029772281647, "step": 180 }, { "epoch": 0.3056126946811637, "grad_norm": 58.17976379394531, "learning_rate": 9.98470848495185e-07, "logits/chosen": -0.9314310550689697, "logits/rejected": -0.6725074648857117, "logps/chosen": -367.3880310058594, "logps/rejected": -280.9293518066406, "loss": 0.5662, "rewards/accuracies": 0.71875, "rewards/chosen": 0.03602181747555733, "rewards/margins": 0.40341871976852417, "rewards/rejected": -0.3673969507217407, "step": 195 }, { "epoch": 0.32912136350279164, "grad_norm": 52.23281478881836, "learning_rate": 9.972516476389642e-07, "logits/chosen": -0.9290924668312073, "logits/rejected": -0.6681386232376099, "logps/chosen": -382.5225830078125, "logps/rejected": -288.5783386230469, "loss": 0.5534, "rewards/accuracies": 0.7354166507720947, "rewards/chosen": 0.009489117190241814, "rewards/margins": 0.4669964611530304, "rewards/rejected": -0.45750725269317627, "step": 210 }, { "epoch": 0.32912136350279164, "eval_logits/chosen": -0.9316383004188538, "eval_logits/rejected": -0.6768124103546143, "eval_logps/chosen": -352.13800048828125, "eval_logps/rejected": -289.91552734375, "eval_loss": 0.5547987222671509, "eval_rewards/accuracies": 0.7285595536231995, "eval_rewards/chosen": 0.01759806089103222, "eval_rewards/margins": 0.4658346474170685, "eval_rewards/rejected": -0.4482365548610687, "eval_runtime": 180.7226, "eval_samples_per_second": 13.291, "eval_steps_per_second": 6.646, "step": 210 }, { "epoch": 0.35263003232441964, "grad_norm": 51.5745849609375, "learning_rate": 9.956788081889744e-07, "logits/chosen": -1.0272151231765747, "logits/rejected": -0.7074090242385864, "logps/chosen": -327.7598876953125, "logps/rejected": -291.8259582519531, "loss": 0.5717, "rewards/accuracies": 0.7083333134651184, "rewards/chosen": -0.03899319842457771, "rewards/margins": 0.4320748746395111, "rewards/rejected": -0.47106799483299255, "step": 225 }, { "epoch": 0.3761387011460476, "grad_norm": 52.133583068847656, "learning_rate": 9.93753448727193e-07, "logits/chosen": -0.9769286513328552, "logits/rejected": -0.6823646426200867, "logps/chosen": -356.5062255859375, "logps/rejected": -284.9337463378906, "loss": 0.5371, "rewards/accuracies": 0.7583333253860474, "rewards/chosen": 0.034630779176950455, "rewards/margins": 0.5447677373886108, "rewards/rejected": -0.5101370215415955, "step": 240 }, { "epoch": 0.3839749240865903, "eval_logits/chosen": -0.9463596940040588, "eval_logits/rejected": -0.6955367922782898, "eval_logps/chosen": -352.4129943847656, "eval_logps/rejected": -291.015380859375, "eval_loss": 0.5392794013023376, "eval_rewards/accuracies": 0.729808509349823, "eval_rewards/chosen": -0.009902803227305412, "eval_rewards/margins": 0.5483154058456421, "eval_rewards/rejected": -0.5582181811332703, "eval_runtime": 181.8777, "eval_samples_per_second": 13.207, "eval_steps_per_second": 6.603, "step": 245 }, { "epoch": 0.3996473699676756, "grad_norm": 43.88309097290039, "learning_rate": 9.914769385430247e-07, "logits/chosen": -0.9463251233100891, "logits/rejected": -0.6262016296386719, "logps/chosen": -357.5391845703125, "logps/rejected": -307.8315124511719, "loss": 0.5629, "rewards/accuracies": 0.7124999761581421, "rewards/chosen": -0.028146782889962196, "rewards/margins": 0.5159488320350647, "rewards/rejected": -0.5440956354141235, "step": 255 }, { "epoch": 0.4231560387893036, "grad_norm": 64.59564971923828, "learning_rate": 9.888508966594814e-07, "logits/chosen": -0.9834640026092529, "logits/rejected": -0.6821132898330688, "logps/chosen": -361.8191833496094, "logps/rejected": -296.5501708984375, "loss": 0.5284, "rewards/accuracies": 0.7479166388511658, "rewards/chosen": -0.042962126433849335, "rewards/margins": 0.56870037317276, "rewards/rejected": -0.6116625070571899, "step": 270 }, { "epoch": 0.4388284846703889, "eval_logits/chosen": -0.9437891244888306, "eval_logits/rejected": -0.6993773579597473, "eval_logps/chosen": -353.16070556640625, "eval_logps/rejected": -292.8016662597656, "eval_loss": 0.5247384905815125, "eval_rewards/accuracies": 0.7348043322563171, "eval_rewards/chosen": -0.0846758484840393, "eval_rewards/margins": 0.6521764397621155, "eval_rewards/rejected": -0.7368523478507996, "eval_runtime": 181.0518, "eval_samples_per_second": 13.267, "eval_steps_per_second": 6.633, "step": 280 }, { "epoch": 0.4466647076109315, "grad_norm": 68.69232177734375, "learning_rate": 9.85877190681755e-07, "logits/chosen": -0.9795821309089661, "logits/rejected": -0.7652584314346313, "logps/chosen": -367.8410339355469, "logps/rejected": -297.42730712890625, "loss": 0.5191, "rewards/accuracies": 0.7708333134651184, "rewards/chosen": -0.10684531182050705, "rewards/margins": 0.6454100012779236, "rewards/rejected": -0.752255380153656, "step": 285 }, { "epoch": 0.4701733764325595, "grad_norm": 44.80474853515625, "learning_rate": 9.825579354690029e-07, "logits/chosen": -1.0019594430923462, "logits/rejected": -0.7404360175132751, "logps/chosen": -349.2023620605469, "logps/rejected": -286.4942932128906, "loss": 0.5028, "rewards/accuracies": 0.7604166865348816, "rewards/chosen": -0.07686860114336014, "rewards/margins": 0.6980764269828796, "rewards/rejected": -0.774945080280304, "step": 300 }, { "epoch": 0.49368204525418746, "grad_norm": 64.48184967041016, "learning_rate": 9.788954916302867e-07, "logits/chosen": -1.033690333366394, "logits/rejected": -0.7505324482917786, "logps/chosen": -353.74407958984375, "logps/rejected": -301.9696350097656, "loss": 0.509, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.19922642409801483, "rewards/margins": 0.7380853891372681, "rewards/rejected": -0.9373118877410889, "step": 315 }, { "epoch": 0.49368204525418746, "eval_logits/chosen": -1.0560635328292847, "eval_logits/rejected": -0.8106775879859924, "eval_logps/chosen": -354.1845397949219, "eval_logps/rejected": -294.6258239746094, "eval_loss": 0.5152602195739746, "eval_rewards/accuracies": 0.7435470223426819, "eval_rewards/chosen": -0.18705680966377258, "eval_rewards/margins": 0.7322102189064026, "eval_rewards/rejected": -0.9192669987678528, "eval_runtime": 181.0466, "eval_samples_per_second": 13.267, "eval_steps_per_second": 6.634, "step": 315 }, { "epoch": 0.5171907140758154, "grad_norm": 60.448970794677734, "learning_rate": 9.748924638457425e-07, "logits/chosen": -1.1234601736068726, "logits/rejected": -0.7872902750968933, "logps/chosen": -363.0025329589844, "logps/rejected": -309.54241943359375, "loss": 0.4751, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": -0.08163168281316757, "rewards/margins": 0.8535270094871521, "rewards/rejected": -0.9351587295532227, "step": 330 }, { "epoch": 0.5406993828974435, "grad_norm": 59.353973388671875, "learning_rate": 9.705516990141652e-07, "logits/chosen": -1.035413384437561, "logits/rejected": -0.7837415337562561, "logps/chosen": -374.60638427734375, "logps/rejected": -337.9807434082031, "loss": 0.5057, "rewards/accuracies": 0.7395833134651184, "rewards/chosen": -0.18711723387241364, "rewards/margins": 0.7444573640823364, "rewards/rejected": -0.9315746426582336, "step": 345 }, { "epoch": 0.5485356058379861, "eval_logits/chosen": -1.0895341634750366, "eval_logits/rejected": -0.8445096611976624, "eval_logps/chosen": -354.7453918457031, "eval_logps/rejected": -295.85137939453125, "eval_loss": 0.5081992745399475, "eval_rewards/accuracies": 0.7435470223426819, "eval_rewards/chosen": -0.2431420534849167, "eval_rewards/margins": 0.7986794114112854, "eval_rewards/rejected": -1.0418214797973633, "eval_runtime": 182.0988, "eval_samples_per_second": 13.191, "eval_steps_per_second": 6.595, "step": 350 }, { "epoch": 0.5642080517190714, "grad_norm": 65.68885040283203, "learning_rate": 9.658762842283341e-07, "logits/chosen": -1.07207453250885, "logits/rejected": -0.7635911107063293, "logps/chosen": -380.82977294921875, "logps/rejected": -317.8447570800781, "loss": 0.4982, "rewards/accuracies": 0.75, "rewards/chosen": -0.2277773916721344, "rewards/margins": 0.8403520584106445, "rewards/rejected": -1.0681294202804565, "step": 360 }, { "epoch": 0.5877167205406993, "grad_norm": 58.057342529296875, "learning_rate": 9.608695445795146e-07, "logits/chosen": -1.134562373161316, "logits/rejected": -0.8592749238014221, "logps/chosen": -372.07171630859375, "logps/rejected": -310.8824768066406, "loss": 0.4783, "rewards/accuracies": 0.7645833492279053, "rewards/chosen": -0.17556972801685333, "rewards/margins": 0.9482249021530151, "rewards/rejected": -1.1237945556640625, "step": 375 }, { "epoch": 0.6033891664217848, "eval_logits/chosen": -1.0469961166381836, "eval_logits/rejected": -0.8057866096496582, "eval_logps/chosen": -354.4659118652344, "eval_logps/rejected": -296.0675354003906, "eval_loss": 0.5002036690711975, "eval_rewards/accuracies": 0.7522897720336914, "eval_rewards/chosen": -0.21519356966018677, "eval_rewards/margins": 0.8482457399368286, "eval_rewards/rejected": -1.063439130783081, "eval_runtime": 181.9108, "eval_samples_per_second": 13.204, "eval_steps_per_second": 6.602, "step": 385 }, { "epoch": 0.6112253893623274, "grad_norm": 59.8174934387207, "learning_rate": 9.555350407926977e-07, "logits/chosen": -1.0428589582443237, "logits/rejected": -0.7815055847167969, "logps/chosen": -363.3075866699219, "logps/rejected": -304.4619140625, "loss": 0.5423, "rewards/accuracies": 0.7104166746139526, "rewards/chosen": -0.26339924335479736, "rewards/margins": 0.7616356015205383, "rewards/rejected": -1.0250346660614014, "step": 390 }, { "epoch": 0.6347340581839553, "grad_norm": 57.40476608276367, "learning_rate": 9.498765666942621e-07, "logits/chosen": -1.1355029344558716, "logits/rejected": -0.8117865324020386, "logps/chosen": -338.7142028808594, "logps/rejected": -303.7315673828125, "loss": 0.479, "rewards/accuracies": 0.768750011920929, "rewards/chosen": -0.238851860165596, "rewards/margins": 0.918969988822937, "rewards/rejected": -1.1578218936920166, "step": 405 }, { "epoch": 0.6582427270055833, "grad_norm": 58.45095443725586, "learning_rate": 9.438981465138564e-07, "logits/chosen": -1.0798269510269165, "logits/rejected": -0.860160231590271, "logps/chosen": -368.1922607421875, "logps/rejected": -300.15185546875, "loss": 0.4952, "rewards/accuracies": 0.75, "rewards/chosen": -0.2948712706565857, "rewards/margins": 0.9247942566871643, "rewards/rejected": -1.2196654081344604, "step": 420 }, { "epoch": 0.6582427270055833, "eval_logits/chosen": -1.1024389266967773, "eval_logits/rejected": -0.8648662567138672, "eval_logps/chosen": -355.2312316894531, "eval_logps/rejected": -297.4045104980469, "eval_loss": 0.4926217198371887, "eval_rewards/accuracies": 0.7572855949401855, "eval_rewards/chosen": -0.29172664880752563, "eval_rewards/margins": 0.9054065942764282, "eval_rewards/rejected": -1.197133183479309, "eval_runtime": 181.474, "eval_samples_per_second": 13.236, "eval_steps_per_second": 6.618, "step": 420 }, { "epoch": 0.6817513958272113, "grad_norm": 67.23454284667969, "learning_rate": 9.376040320224207e-07, "logits/chosen": -1.1256804466247559, "logits/rejected": -0.778734028339386, "logps/chosen": -353.2846374511719, "logps/rejected": -294.6732482910156, "loss": 0.5274, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.3149377405643463, "rewards/margins": 0.8212650418281555, "rewards/rejected": -1.1362026929855347, "step": 435 }, { "epoch": 0.7052600646488393, "grad_norm": 43.976097106933594, "learning_rate": 9.309986995083865e-07, "logits/chosen": -1.0163235664367676, "logits/rejected": -0.7996082305908203, "logps/chosen": -347.673828125, "logps/rejected": -304.9941101074219, "loss": 0.487, "rewards/accuracies": 0.7520833611488342, "rewards/chosen": -0.27800673246383667, "rewards/margins": 0.9257082343101501, "rewards/rejected": -1.2037149667739868, "step": 450 }, { "epoch": 0.7130962875893819, "eval_logits/chosen": -1.0526485443115234, "eval_logits/rejected": -0.819446325302124, "eval_logps/chosen": -355.0964050292969, "eval_logps/rejected": -297.8984069824219, "eval_loss": 0.4861847162246704, "eval_rewards/accuracies": 0.7643630504608154, "eval_rewards/chosen": -0.2782430350780487, "eval_rewards/margins": 0.9682838916778564, "eval_rewards/rejected": -1.246526837348938, "eval_runtime": 181.8521, "eval_samples_per_second": 13.209, "eval_steps_per_second": 6.604, "step": 455 }, { "epoch": 0.7287687334704672, "grad_norm": 44.1576042175293, "learning_rate": 9.240868465942004e-07, "logits/chosen": -1.0843127965927124, "logits/rejected": -0.8464950323104858, "logps/chosen": -368.0324401855469, "logps/rejected": -298.4980773925781, "loss": 0.443, "rewards/accuracies": 0.7895833253860474, "rewards/chosen": -0.20106664299964905, "rewards/margins": 1.0733890533447266, "rewards/rejected": -1.2744557857513428, "step": 465 }, { "epoch": 0.7522774022920952, "grad_norm": 75.02023315429688, "learning_rate": 9.168733888954398e-07, "logits/chosen": -0.9654378294944763, "logits/rejected": -0.766428530216217, "logps/chosen": -368.84527587890625, "logps/rejected": -310.3114013671875, "loss": 0.4913, "rewards/accuracies": 0.7729166746139526, "rewards/chosen": -0.31580713391304016, "rewards/margins": 0.9536004662513733, "rewards/rejected": -1.2694075107574463, "step": 480 }, { "epoch": 0.7679498481731806, "eval_logits/chosen": -1.018790364265442, "eval_logits/rejected": -0.7895683646202087, "eval_logps/chosen": -355.1246032714844, "eval_logps/rejected": -298.2669677734375, "eval_loss": 0.4843537211418152, "eval_rewards/accuracies": 0.7597835063934326, "eval_rewards/chosen": -0.2810628116130829, "eval_rewards/margins": 1.002318024635315, "eval_rewards/rejected": -1.2833808660507202, "eval_runtime": 181.6862, "eval_samples_per_second": 13.221, "eval_steps_per_second": 6.61, "step": 490 }, { "epoch": 0.7757860711137232, "grad_norm": 57.9295768737793, "learning_rate": 9.093634565248934e-07, "logits/chosen": -1.0683478116989136, "logits/rejected": -0.7814379930496216, "logps/chosen": -346.7208251953125, "logps/rejected": -305.7705383300781, "loss": 0.4662, "rewards/accuracies": 0.762499988079071, "rewards/chosen": -0.22331449389457703, "rewards/margins": 1.0663999319076538, "rewards/rejected": -1.2897144556045532, "step": 495 }, { "epoch": 0.7992947399353512, "grad_norm": 64.89127349853516, "learning_rate": 9.015623904440955e-07, "logits/chosen": -1.0165103673934937, "logits/rejected": -0.7501361966133118, "logps/chosen": -360.36444091796875, "logps/rejected": -311.2584533691406, "loss": 0.4763, "rewards/accuracies": 0.7729166746139526, "rewards/chosen": -0.3126125931739807, "rewards/margins": 1.0347987413406372, "rewards/rejected": -1.3474112749099731, "step": 510 }, { "epoch": 0.8228034087569791, "grad_norm": 57.808197021484375, "learning_rate": 8.934757386649061e-07, "logits/chosen": -1.0641908645629883, "logits/rejected": -0.7976428866386414, "logps/chosen": -331.990234375, "logps/rejected": -290.6216125488281, "loss": 0.4478, "rewards/accuracies": 0.7854166626930237, "rewards/chosen": -0.32272768020629883, "rewards/margins": 1.1051963567733765, "rewards/rejected": -1.4279239177703857, "step": 525 }, { "epoch": 0.8228034087569791, "eval_logits/chosen": -1.0868600606918335, "eval_logits/rejected": -0.8659496307373047, "eval_logps/chosen": -355.5617370605469, "eval_logps/rejected": -299.2372741699219, "eval_loss": 0.4808862507343292, "eval_rewards/accuracies": 0.7585345506668091, "eval_rewards/chosen": -0.32477837800979614, "eval_rewards/margins": 1.0556353330612183, "eval_rewards/rejected": -1.3804137706756592, "eval_runtime": 182.9841, "eval_samples_per_second": 13.127, "eval_steps_per_second": 6.563, "step": 525 }, { "epoch": 0.8463120775786072, "grad_norm": 53.98827362060547, "learning_rate": 8.851092523038417e-07, "logits/chosen": -1.118863821029663, "logits/rejected": -0.8719570636749268, "logps/chosen": -363.63299560546875, "logps/rejected": -302.99853515625, "loss": 0.4774, "rewards/accuracies": 0.7770833373069763, "rewards/chosen": -0.3193167746067047, "rewards/margins": 1.0706781148910522, "rewards/rejected": -1.3899948596954346, "step": 540 }, { "epoch": 0.8698207464002351, "grad_norm": 53.19316482543945, "learning_rate": 8.764688814919589e-07, "logits/chosen": -1.0061967372894287, "logits/rejected": -0.8136522173881531, "logps/chosen": -387.64447021484375, "logps/rejected": -306.2342529296875, "loss": 0.452, "rewards/accuracies": 0.7541666626930237, "rewards/chosen": -0.3774346113204956, "rewards/margins": 1.1483807563781738, "rewards/rejected": -1.525815486907959, "step": 555 }, { "epoch": 0.8776569693407777, "eval_logits/chosen": -1.0304757356643677, "eval_logits/rejected": -0.8115791082382202, "eval_logps/chosen": -355.95196533203125, "eval_logps/rejected": -300.0523681640625, "eval_loss": 0.47852277755737305, "eval_rewards/accuracies": 0.7635303735733032, "eval_rewards/chosen": -0.36379873752593994, "eval_rewards/margins": 1.09812331199646, "eval_rewards/rejected": -1.4619219303131104, "eval_runtime": 182.2848, "eval_samples_per_second": 13.177, "eval_steps_per_second": 6.589, "step": 560 }, { "epoch": 0.893329415221863, "grad_norm": 52.40211486816406, "learning_rate": 8.675607711432023e-07, "logits/chosen": -1.012203574180603, "logits/rejected": -0.8512925505638123, "logps/chosen": -353.43341064453125, "logps/rejected": -299.34576416015625, "loss": 0.4917, "rewards/accuracies": 0.7583333253860474, "rewards/chosen": -0.38326555490493774, "rewards/margins": 1.1050751209259033, "rewards/rejected": -1.4883407354354858, "step": 570 }, { "epoch": 0.916838084043491, "grad_norm": 77.06608581542969, "learning_rate": 8.583912565842256e-07, "logits/chosen": -1.0286543369293213, "logits/rejected": -0.8386385440826416, "logps/chosen": -363.56439208984375, "logps/rejected": -330.7981872558594, "loss": 0.4654, "rewards/accuracies": 0.7833333611488342, "rewards/chosen": -0.44688889384269714, "rewards/margins": 1.2291133403778076, "rewards/rejected": -1.6760022640228271, "step": 585 }, { "epoch": 0.9325105299245764, "eval_logits/chosen": -1.075386643409729, "eval_logits/rejected": -0.8566927909851074, "eval_logps/chosen": -355.85455322265625, "eval_logps/rejected": -299.8857727050781, "eval_loss": 0.4771224856376648, "eval_rewards/accuracies": 0.7572855949401855, "eval_rewards/chosen": -0.3540586531162262, "eval_rewards/margins": 1.091202735900879, "eval_rewards/rejected": -1.4452612400054932, "eval_runtime": 181.9619, "eval_samples_per_second": 13.201, "eval_steps_per_second": 6.6, "step": 595 }, { "epoch": 0.940346752865119, "grad_norm": 62.413719177246094, "learning_rate": 8.489668590487934e-07, "logits/chosen": -1.0846012830734253, "logits/rejected": -0.7979000210762024, "logps/chosen": -384.29083251953125, "logps/rejected": -316.1002197265625, "loss": 0.4709, "rewards/accuracies": 0.7708333134651184, "rewards/chosen": -0.3900051712989807, "rewards/margins": 1.184963345527649, "rewards/rejected": -1.574968695640564, "step": 600 }, { "epoch": 0.963855421686747, "grad_norm": 53.96653366088867, "learning_rate": 8.392942810399692e-07, "logits/chosen": -1.0858737230300903, "logits/rejected": -0.7765557765960693, "logps/chosen": -354.53985595703125, "logps/rejected": -297.6081848144531, "loss": 0.4754, "rewards/accuracies": 0.7708333134651184, "rewards/chosen": -0.3535262644290924, "rewards/margins": 1.0648608207702637, "rewards/rejected": -1.4183870553970337, "step": 615 }, { "epoch": 0.9873640905083749, "grad_norm": 52.692047119140625, "learning_rate": 8.293804015633861e-07, "logits/chosen": -0.9902106523513794, "logits/rejected": -0.8622989654541016, "logps/chosen": -364.4175109863281, "logps/rejected": -307.4686279296875, "loss": 0.4829, "rewards/accuracies": 0.7791666388511658, "rewards/chosen": -0.3149837851524353, "rewards/margins": 1.1350115537643433, "rewards/rejected": -1.4499951601028442, "step": 630 }, { "epoch": 0.9873640905083749, "eval_logits/chosen": -1.1149685382843018, "eval_logits/rejected": -0.8937057256698608, "eval_logps/chosen": -355.9053649902344, "eval_logps/rejected": -300.0727233886719, "eval_loss": 0.4741358458995819, "eval_rewards/accuracies": 0.7643630504608154, "eval_rewards/chosen": -0.35914117097854614, "eval_rewards/margins": 1.10481595993042, "eval_rewards/rejected": -1.4639569520950317, "eval_runtime": 181.7303, "eval_samples_per_second": 13.217, "eval_steps_per_second": 6.609, "step": 630 }, { "epoch": 1.0094034675286512, "grad_norm": 31.40093421936035, "learning_rate": 8.192322712349917e-07, "logits/chosen": -1.1683887243270874, "logits/rejected": -0.944835364818573, "logps/chosen": -368.8553466796875, "logps/rejected": -316.08819580078125, "loss": 0.381, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -0.2540862560272217, "rewards/margins": 1.389145016670227, "rewards/rejected": -1.6432311534881592, "step": 645 }, { "epoch": 1.0329121363502791, "grad_norm": 35.006961822509766, "learning_rate": 8.088571072667467e-07, "logits/chosen": -1.1861752271652222, "logits/rejected": -0.9679210782051086, "logps/chosen": -358.0697937011719, "logps/rejected": -311.8282470703125, "loss": 0.3093, "rewards/accuracies": 0.8979166746139526, "rewards/chosen": -0.12442357838153839, "rewards/margins": 1.7039713859558105, "rewards/rejected": -1.828395128250122, "step": 660 }, { "epoch": 1.0407483592908218, "eval_logits/chosen": -1.1818084716796875, "eval_logits/rejected": -0.9633015394210815, "eval_logps/chosen": -356.53192138671875, "eval_logps/rejected": -301.01409912109375, "eval_loss": 0.4742245674133301, "eval_rewards/accuracies": 0.7726894021034241, "eval_rewards/chosen": -0.4217943549156189, "eval_rewards/margins": 1.1362972259521484, "eval_rewards/rejected": -1.558091640472412, "eval_runtime": 182.4015, "eval_samples_per_second": 13.169, "eval_steps_per_second": 6.584, "step": 665 }, { "epoch": 1.056420805171907, "grad_norm": 38.69508361816406, "learning_rate": 7.982622883338412e-07, "logits/chosen": -1.3052853345870972, "logits/rejected": -0.9795024991035461, "logps/chosen": -354.4381103515625, "logps/rejected": -300.4755859375, "loss": 0.2806, "rewards/accuracies": 0.9145833253860474, "rewards/chosen": -0.05176251009106636, "rewards/margins": 1.8534504175186157, "rewards/rejected": -1.9052128791809082, "step": 675 }, { "epoch": 1.079929473993535, "grad_norm": 38.487979888916016, "learning_rate": 7.87455349327083e-07, "logits/chosen": -1.1824771165847778, "logits/rejected": -1.0179836750030518, "logps/chosen": -359.6732482910156, "logps/rejected": -310.7752685546875, "loss": 0.3073, "rewards/accuracies": 0.9145833253860474, "rewards/chosen": -0.17797540128231049, "rewards/margins": 1.6661934852600098, "rewards/rejected": -1.8441689014434814, "step": 690 }, { "epoch": 1.0956019198746205, "eval_logits/chosen": -1.2035633325576782, "eval_logits/rejected": -0.988406777381897, "eval_logps/chosen": -357.3915100097656, "eval_logps/rejected": -302.4732666015625, "eval_loss": 0.47595611214637756, "eval_rewards/accuracies": 0.765612006187439, "eval_rewards/chosen": -0.5077541470527649, "eval_rewards/margins": 1.1962535381317139, "eval_rewards/rejected": -1.7040073871612549, "eval_runtime": 182.096, "eval_samples_per_second": 13.191, "eval_steps_per_second": 6.595, "step": 700 }, { "epoch": 1.1034381428151632, "grad_norm": 36.37774658203125, "learning_rate": 7.76443975994184e-07, "logits/chosen": -1.2869796752929688, "logits/rejected": -0.9491466879844666, "logps/chosen": -367.9986877441406, "logps/rejected": -300.4508972167969, "loss": 0.2897, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.16376915574073792, "rewards/margins": 1.7783530950546265, "rewards/rejected": -1.942122220993042, "step": 705 }, { "epoch": 1.1269468116367911, "grad_norm": 46.2029914855957, "learning_rate": 7.652359994737627e-07, "logits/chosen": -1.236419916152954, "logits/rejected": -1.0354564189910889, "logps/chosen": -379.1826171875, "logps/rejected": -316.03631591796875, "loss": 0.2747, "rewards/accuracies": 0.9083333611488342, "rewards/chosen": -0.18529005348682404, "rewards/margins": 1.8945099115371704, "rewards/rejected": -2.0797998905181885, "step": 720 }, { "epoch": 1.150455480458419, "grad_norm": 30.26227569580078, "learning_rate": 7.538393907259448e-07, "logits/chosen": -1.1049808263778687, "logits/rejected": -0.9194839596748352, "logps/chosen": -378.98443603515625, "logps/rejected": -327.0231628417969, "loss": 0.2608, "rewards/accuracies": 0.9270833134651184, "rewards/chosen": -0.19015556573867798, "rewards/margins": 1.9543516635894775, "rewards/rejected": -2.14450740814209, "step": 735 }, { "epoch": 1.150455480458419, "eval_logits/chosen": -1.1943178176879883, "eval_logits/rejected": -0.9850893020629883, "eval_logps/chosen": -358.00390625, "eval_logps/rejected": -303.1407775878906, "eval_loss": 0.4748242497444153, "eval_rewards/accuracies": 0.7668609619140625, "eval_rewards/chosen": -0.568992018699646, "eval_rewards/margins": 1.2017687559127808, "eval_rewards/rejected": -1.7707608938217163, "eval_runtime": 182.4368, "eval_samples_per_second": 13.166, "eval_steps_per_second": 6.583, "step": 735 }, { "epoch": 1.173964149280047, "grad_norm": 37.521915435791016, "learning_rate": 7.422622548635244e-07, "logits/chosen": -1.2676866054534912, "logits/rejected": -1.0028654336929321, "logps/chosen": -362.83648681640625, "logps/rejected": -321.3612060546875, "loss": 0.2729, "rewards/accuracies": 0.90625, "rewards/chosen": -0.2053557187318802, "rewards/margins": 1.8843697309494019, "rewards/rejected": -2.0897254943847656, "step": 750 }, { "epoch": 1.197472818101675, "grad_norm": 33.50147247314453, "learning_rate": 7.305128253877195e-07, "logits/chosen": -1.2824881076812744, "logits/rejected": -0.964570939540863, "logps/chosen": -340.29644775390625, "logps/rejected": -311.6012268066406, "loss": 0.3037, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.22407926619052887, "rewards/margins": 1.7219882011413574, "rewards/rejected": -1.9460675716400146, "step": 765 }, { "epoch": 1.2053090410422176, "eval_logits/chosen": -1.2377439737319946, "eval_logits/rejected": -1.0323067903518677, "eval_logps/chosen": -357.77703857421875, "eval_logps/rejected": -302.7647399902344, "eval_loss": 0.4716549813747406, "eval_rewards/accuracies": 0.7697752118110657, "eval_rewards/chosen": -0.5463067293167114, "eval_rewards/margins": 1.1868516206741333, "eval_rewards/rejected": -1.7331583499908447, "eval_runtime": 181.9712, "eval_samples_per_second": 13.2, "eval_steps_per_second": 6.6, "step": 770 }, { "epoch": 1.2209814869233029, "grad_norm": 37.49959182739258, "learning_rate": 7.185994583326165e-07, "logits/chosen": -1.1782593727111816, "logits/rejected": -1.0032784938812256, "logps/chosen": -380.5348205566406, "logps/rejected": -333.3002624511719, "loss": 0.2849, "rewards/accuracies": 0.9104166626930237, "rewards/chosen": -0.21467427909374237, "rewards/margins": 1.8065526485443115, "rewards/rejected": -2.0212271213531494, "step": 780 }, { "epoch": 1.244490155744931, "grad_norm": 35.8784294128418, "learning_rate": 7.065306263224742e-07, "logits/chosen": -1.148066759109497, "logits/rejected": -0.9383103847503662, "logps/chosen": -367.6258239746094, "logps/rejected": -306.7835388183594, "loss": 0.2828, "rewards/accuracies": 0.9020833373069763, "rewards/chosen": -0.21600770950317383, "rewards/margins": 1.806153416633606, "rewards/rejected": -2.0221610069274902, "step": 795 }, { "epoch": 1.2601626016260163, "eval_logits/chosen": -1.2068735361099243, "eval_logits/rejected": -1.0000766515731812, "eval_logps/chosen": -358.01068115234375, "eval_logps/rejected": -303.16278076171875, "eval_loss": 0.47309279441833496, "eval_rewards/accuracies": 0.7714404463768005, "eval_rewards/chosen": -0.569669246673584, "eval_rewards/margins": 1.203292965888977, "eval_rewards/rejected": -1.772962212562561, "eval_runtime": 182.4089, "eval_samples_per_second": 13.168, "eval_steps_per_second": 6.584, "step": 805 }, { "epoch": 1.267998824566559, "grad_norm": 24.38459014892578, "learning_rate": 6.94314912546108e-07, "logits/chosen": -1.2680124044418335, "logits/rejected": -0.9894136786460876, "logps/chosen": -350.9635009765625, "logps/rejected": -310.0843811035156, "loss": 0.2591, "rewards/accuracies": 0.9041666388511658, "rewards/chosen": -0.1324070394039154, "rewards/margins": 1.9815576076507568, "rewards/rejected": -2.113964796066284, "step": 810 }, { "epoch": 1.291507493388187, "grad_norm": 39.18712615966797, "learning_rate": 6.819610046526436e-07, "logits/chosen": -1.2195533514022827, "logits/rejected": -1.0207655429840088, "logps/chosen": -365.35595703125, "logps/rejected": -308.2247009277344, "loss": 0.29, "rewards/accuracies": 0.8916666507720947, "rewards/chosen": -0.2736612856388092, "rewards/margins": 1.791273593902588, "rewards/rejected": -2.0649349689483643, "step": 825 }, { "epoch": 1.3150161622098149, "grad_norm": 32.96370315551758, "learning_rate": 6.694776885729787e-07, "logits/chosen": -1.1510590314865112, "logits/rejected": -0.9895613193511963, "logps/chosen": -346.2775573730469, "logps/rejected": -303.6460876464844, "loss": 0.3013, "rewards/accuracies": 0.8895833492279053, "rewards/chosen": -0.276450514793396, "rewards/margins": 1.8475831747055054, "rewards/rejected": -2.1240336894989014, "step": 840 }, { "epoch": 1.3150161622098149, "eval_logits/chosen": -1.2594456672668457, "eval_logits/rejected": -1.056649923324585, "eval_logps/chosen": -358.96270751953125, "eval_logps/rejected": -304.4568176269531, "eval_loss": 0.47552865743637085, "eval_rewards/accuracies": 0.770191490650177, "eval_rewards/chosen": -0.6648746728897095, "eval_rewards/margins": 1.2374926805496216, "eval_rewards/rejected": -1.9023675918579102, "eval_runtime": 182.3073, "eval_samples_per_second": 13.176, "eval_steps_per_second": 6.588, "step": 840 }, { "epoch": 1.3385248310314428, "grad_norm": 43.61619567871094, "learning_rate": 6.568738422713492e-07, "logits/chosen": -1.3479042053222656, "logits/rejected": -1.1390700340270996, "logps/chosen": -356.8882751464844, "logps/rejected": -316.7053527832031, "loss": 0.253, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.34459179639816284, "rewards/margins": 2.0109331607818604, "rewards/rejected": -2.355525016784668, "step": 855 }, { "epoch": 1.3620334998530708, "grad_norm": 37.48102569580078, "learning_rate": 6.441584294314419e-07, "logits/chosen": -1.2495485544204712, "logits/rejected": -0.9884099364280701, "logps/chosen": -361.2688293457031, "logps/rejected": -326.4720458984375, "loss": 0.2693, "rewards/accuracies": 0.9020833373069763, "rewards/chosen": -0.33273938298225403, "rewards/margins": 2.039118528366089, "rewards/rejected": -2.3718576431274414, "step": 870 }, { "epoch": 1.3698697227936134, "eval_logits/chosen": -1.2140861749649048, "eval_logits/rejected": -1.0111212730407715, "eval_logps/chosen": -359.7378845214844, "eval_logps/rejected": -305.63800048828125, "eval_loss": 0.47619202733039856, "eval_rewards/accuracies": 0.765612006187439, "eval_rewards/chosen": -0.7423911094665527, "eval_rewards/margins": 1.2780914306640625, "eval_rewards/rejected": -2.0204825401306152, "eval_runtime": 182.596, "eval_samples_per_second": 13.155, "eval_steps_per_second": 6.577, "step": 875 }, { "epoch": 1.3855421686746987, "grad_norm": 49.734066009521484, "learning_rate": 6.313404930815452e-07, "logits/chosen": -1.163791537284851, "logits/rejected": -0.9558045864105225, "logps/chosen": -359.283203125, "logps/rejected": -306.8938293457031, "loss": 0.2767, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -0.4080369770526886, "rewards/margins": 1.97806978225708, "rewards/rejected": -2.3861069679260254, "step": 885 }, { "epoch": 1.4090508374963266, "grad_norm": 30.948827743530273, "learning_rate": 6.184291491632694e-07, "logits/chosen": -1.1903069019317627, "logits/rejected": -0.9102679491043091, "logps/chosen": -341.7632141113281, "logps/rejected": -303.9745178222656, "loss": 0.2875, "rewards/accuracies": 0.8979166746139526, "rewards/chosen": -0.32621437311172485, "rewards/margins": 1.8810745477676392, "rewards/rejected": -2.207289218902588, "step": 900 }, { "epoch": 1.4247232833774122, "eval_logits/chosen": -1.2810581922531128, "eval_logits/rejected": -1.083061933517456, "eval_logps/chosen": -359.7813720703125, "eval_logps/rejected": -305.91448974609375, "eval_loss": 0.4744855761528015, "eval_rewards/accuracies": 0.768109917640686, "eval_rewards/chosen": -0.7467413544654846, "eval_rewards/margins": 1.3013904094696045, "eval_rewards/rejected": -2.0481317043304443, "eval_runtime": 181.9732, "eval_samples_per_second": 13.2, "eval_steps_per_second": 6.6, "step": 910 }, { "epoch": 1.4325595063179548, "grad_norm": 39.57698440551758, "learning_rate": 6.054335800484152e-07, "logits/chosen": -1.3292855024337769, "logits/rejected": -0.9953449368476868, "logps/chosen": -373.2196044921875, "logps/rejected": -335.6629943847656, "loss": 0.2296, "rewards/accuracies": 0.9291666746139526, "rewards/chosen": -0.20129725337028503, "rewards/margins": 2.1831979751586914, "rewards/rejected": -2.384495258331299, "step": 915 }, { "epoch": 1.4560681751395828, "grad_norm": 43.79376983642578, "learning_rate": 5.923630280085947e-07, "logits/chosen": -1.2759658098220825, "logits/rejected": -1.0600124597549438, "logps/chosen": -356.8258056640625, "logps/rejected": -317.5994567871094, "loss": 0.271, "rewards/accuracies": 0.90625, "rewards/chosen": -0.3783254027366638, "rewards/margins": 2.0156712532043457, "rewards/rejected": -2.3939969539642334, "step": 930 }, { "epoch": 1.4795768439612107, "grad_norm": 36.227500915527344, "learning_rate": 5.792267886422537e-07, "logits/chosen": -1.1942765712738037, "logits/rejected": -0.9782482385635376, "logps/chosen": -368.8127136230469, "logps/rejected": -322.2507629394531, "loss": 0.2462, "rewards/accuracies": 0.9145833253860474, "rewards/chosen": -0.2551910877227783, "rewards/margins": 2.0672876834869385, "rewards/rejected": -2.322478771209717, "step": 945 }, { "epoch": 1.4795768439612107, "eval_logits/chosen": -1.2686865329742432, "eval_logits/rejected": -1.0674816370010376, "eval_logps/chosen": -359.4958801269531, "eval_logps/rejected": -305.484130859375, "eval_loss": 0.473550945520401, "eval_rewards/accuracies": 0.7722731232643127, "eval_rewards/chosen": -0.7181934118270874, "eval_rewards/margins": 1.286901593208313, "eval_rewards/rejected": -2.0050950050354004, "eval_runtime": 182.2235, "eval_samples_per_second": 13.182, "eval_steps_per_second": 6.591, "step": 945 }, { "epoch": 1.5030855127828386, "grad_norm": 35.31517028808594, "learning_rate": 5.660342042637701e-07, "logits/chosen": -1.278725504875183, "logits/rejected": -0.9818956255912781, "logps/chosen": -354.6424560546875, "logps/rejected": -320.46343994140625, "loss": 0.266, "rewards/accuracies": 0.9041666388511658, "rewards/chosen": -0.37783387303352356, "rewards/margins": 2.0141122341156006, "rewards/rejected": -2.3919460773468018, "step": 960 }, { "epoch": 1.5265941816044668, "grad_norm": 43.222660064697266, "learning_rate": 5.527946572593254e-07, "logits/chosen": -1.3914339542388916, "logits/rejected": -1.0797264575958252, "logps/chosen": -360.6291198730469, "logps/rejected": -320.04779052734375, "loss": 0.2524, "rewards/accuracies": 0.9145833253860474, "rewards/chosen": -0.3379184603691101, "rewards/margins": 2.10139536857605, "rewards/rejected": -2.4393134117126465, "step": 975 }, { "epoch": 1.5344304045450095, "eval_logits/chosen": -1.2813137769699097, "eval_logits/rejected": -1.0804452896118164, "eval_logps/chosen": -359.6990661621094, "eval_logps/rejected": -306.07623291015625, "eval_loss": 0.47156351804733276, "eval_rewards/accuracies": 0.770191490650177, "eval_rewards/chosen": -0.7385069131851196, "eval_rewards/margins": 1.3257992267608643, "eval_rewards/rejected": -2.0643060207366943, "eval_runtime": 182.7063, "eval_samples_per_second": 13.147, "eval_steps_per_second": 6.573, "step": 980 }, { "epoch": 1.5501028504260947, "grad_norm": 26.045185089111328, "learning_rate": 5.395175634142814e-07, "logits/chosen": -1.3037978410720825, "logits/rejected": -1.1191579103469849, "logps/chosen": -376.4510498046875, "logps/rejected": -316.0899658203125, "loss": 0.2476, "rewards/accuracies": 0.9208333492279053, "rewards/chosen": -0.27149498462677, "rewards/margins": 2.1316168308258057, "rewards/rejected": -2.4031119346618652, "step": 990 }, { "epoch": 1.5736115192477227, "grad_norm": 32.610233306884766, "learning_rate": 5.262123652168005e-07, "logits/chosen": -1.2782458066940308, "logits/rejected": -0.9592511653900146, "logps/chosen": -351.68524169921875, "logps/rejected": -310.4751281738281, "loss": 0.2421, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.37502792477607727, "rewards/margins": 2.187502145767212, "rewards/rejected": -2.562530040740967, "step": 1005 }, { "epoch": 1.589283965128808, "eval_logits/chosen": -1.2604247331619263, "eval_logits/rejected": -1.0631768703460693, "eval_logps/chosen": -360.0327453613281, "eval_logps/rejected": -306.8144226074219, "eval_loss": 0.4726716876029968, "eval_rewards/accuracies": 0.7743546962738037, "eval_rewards/chosen": -0.7718815803527832, "eval_rewards/margins": 1.366243839263916, "eval_rewards/rejected": -2.138125419616699, "eval_runtime": 182.6755, "eval_samples_per_second": 13.149, "eval_steps_per_second": 6.574, "step": 1015 }, { "epoch": 1.5971201880693506, "grad_norm": 39.39152908325195, "learning_rate": 5.128885251424781e-07, "logits/chosen": -1.367023229598999, "logits/rejected": -1.0569322109222412, "logps/chosen": -360.5495910644531, "logps/rejected": -323.3975524902344, "loss": 0.2304, "rewards/accuracies": 0.9208333492279053, "rewards/chosen": -0.40084362030029297, "rewards/margins": 2.2848780155181885, "rewards/rejected": -2.6857216358184814, "step": 1020 }, { "epoch": 1.6206288568909786, "grad_norm": 27.369558334350586, "learning_rate": 4.995555189247576e-07, "logits/chosen": -1.2406485080718994, "logits/rejected": -0.989262044429779, "logps/chosen": -357.1865234375, "logps/rejected": -297.4217529296875, "loss": 0.2357, "rewards/accuracies": 0.9354166388511658, "rewards/chosen": -0.38933005928993225, "rewards/margins": 2.214608907699585, "rewards/rejected": -2.6039390563964844, "step": 1035 }, { "epoch": 1.6441375257126065, "grad_norm": 35.30168533325195, "learning_rate": 4.86222828815919e-07, "logits/chosen": -1.3355810642242432, "logits/rejected": -1.0357699394226074, "logps/chosen": -360.48468017578125, "logps/rejected": -318.0264892578125, "loss": 0.2545, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.3565267324447632, "rewards/margins": 2.160374641418457, "rewards/rejected": -2.5169014930725098, "step": 1050 }, { "epoch": 1.6441375257126065, "eval_logits/chosen": -1.3108830451965332, "eval_logits/rejected": -1.1151154041290283, "eval_logps/chosen": -360.46136474609375, "eval_logps/rejected": -307.2978820800781, "eval_loss": 0.47149062156677246, "eval_rewards/accuracies": 0.7706078290939331, "eval_rewards/chosen": -0.8147412538528442, "eval_rewards/margins": 1.3717304468154907, "eval_rewards/rejected": -2.186471700668335, "eval_runtime": 182.8736, "eval_samples_per_second": 13.135, "eval_steps_per_second": 6.567, "step": 1050 }, { "epoch": 1.6676461945342345, "grad_norm": 23.27956199645996, "learning_rate": 4.728999368434301e-07, "logits/chosen": -1.3302825689315796, "logits/rejected": -1.044356107711792, "logps/chosen": -348.8330078125, "logps/rejected": -303.2927551269531, "loss": 0.2709, "rewards/accuracies": 0.8958333134651184, "rewards/chosen": -0.4363751709461212, "rewards/margins": 2.0858278274536133, "rewards/rejected": -2.522202968597412, "step": 1065 }, { "epoch": 1.6911548633558624, "grad_norm": 30.386754989624023, "learning_rate": 4.595963180664576e-07, "logits/chosen": -1.2675700187683105, "logits/rejected": -1.0812220573425293, "logps/chosen": -342.9148864746094, "logps/rejected": -290.79949951171875, "loss": 0.253, "rewards/accuracies": 0.9145833253860474, "rewards/chosen": -0.4274654984474182, "rewards/margins": 2.0667319297790527, "rewards/rejected": -2.494197368621826, "step": 1080 }, { "epoch": 1.698991086296405, "eval_logits/chosen": -1.3290081024169922, "eval_logits/rejected": -1.1328283548355103, "eval_logps/chosen": -359.980224609375, "eval_logps/rejected": -306.26348876953125, "eval_loss": 0.4710884392261505, "eval_rewards/accuracies": 0.7726894021034241, "eval_rewards/chosen": -0.7666258215904236, "eval_rewards/margins": 1.3164042234420776, "eval_rewards/rejected": -2.0830299854278564, "eval_runtime": 182.8851, "eval_samples_per_second": 13.134, "eval_steps_per_second": 6.567, "step": 1085 }, { "epoch": 1.7146635321774903, "grad_norm": 41.24742126464844, "learning_rate": 4.4632143383733394e-07, "logits/chosen": -1.3585143089294434, "logits/rejected": -1.1536871194839478, "logps/chosen": -355.0205993652344, "logps/rejected": -318.7183532714844, "loss": 0.2484, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.43984732031822205, "rewards/margins": 2.0352494716644287, "rewards/rejected": -2.4750969409942627, "step": 1095 }, { "epoch": 1.7381722009991183, "grad_norm": 26.021894454956055, "learning_rate": 4.330847250727732e-07, "logits/chosen": -1.3933275938034058, "logits/rejected": -1.1843206882476807, "logps/chosen": -370.0199279785156, "logps/rejected": -322.68731689453125, "loss": 0.2263, "rewards/accuracies": 0.9395833611488342, "rewards/chosen": -0.4698244035243988, "rewards/margins": 2.1898274421691895, "rewards/rejected": -2.6596519947052, "step": 1110 }, { "epoch": 1.7538446468802036, "eval_logits/chosen": -1.3134483098983765, "eval_logits/rejected": -1.1194194555282593, "eval_logps/chosen": -360.1936340332031, "eval_logps/rejected": -306.6624755859375, "eval_loss": 0.46780040860176086, "eval_rewards/accuracies": 0.7756036520004272, "eval_rewards/chosen": -0.7879676818847656, "eval_rewards/margins": 1.3349637985229492, "eval_rewards/rejected": -2.122931480407715, "eval_runtime": 182.6424, "eval_samples_per_second": 13.151, "eval_steps_per_second": 6.576, "step": 1120 }, { "epoch": 1.7616808698207465, "grad_norm": 32.77103042602539, "learning_rate": 4.198956055396194e-07, "logits/chosen": -1.3157938718795776, "logits/rejected": -1.1473113298416138, "logps/chosen": -370.41925048828125, "logps/rejected": -322.4455261230469, "loss": 0.2544, "rewards/accuracies": 0.90625, "rewards/chosen": -0.35879072546958923, "rewards/margins": 2.2119410037994385, "rewards/rejected": -2.5707318782806396, "step": 1125 }, { "epoch": 1.7851895386423744, "grad_norm": 33.03691101074219, "learning_rate": 4.0676345515990384e-07, "logits/chosen": -1.2871196269989014, "logits/rejected": -1.11886465549469, "logps/chosen": -371.6841735839844, "logps/rejected": -308.32806396484375, "loss": 0.2314, "rewards/accuracies": 0.9166666865348816, "rewards/chosen": -0.3601396977901459, "rewards/margins": 2.2169458866119385, "rewards/rejected": -2.577085256576538, "step": 1140 }, { "epoch": 1.8086982074640023, "grad_norm": 35.31661605834961, "learning_rate": 3.9369761333997193e-07, "logits/chosen": -1.3224281072616577, "logits/rejected": -1.102236270904541, "logps/chosen": -355.3537292480469, "logps/rejected": -324.50274658203125, "loss": 0.2507, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.41861438751220703, "rewards/margins": 2.1263999938964844, "rewards/rejected": -2.5450143814086914, "step": 1155 }, { "epoch": 1.8086982074640023, "eval_logits/chosen": -1.2986286878585815, "eval_logits/rejected": -1.1063010692596436, "eval_logps/chosen": -360.1191101074219, "eval_logps/rejected": -306.6952209472656, "eval_loss": 0.4674054980278015, "eval_rewards/accuracies": 0.7781015634536743, "eval_rewards/chosen": -0.7805167436599731, "eval_rewards/margins": 1.3456897735595703, "eval_rewards/rejected": -2.126206636428833, "eval_runtime": 182.9858, "eval_samples_per_second": 13.127, "eval_steps_per_second": 6.563, "step": 1155 }, { "epoch": 1.8322068762856303, "grad_norm": 29.87853240966797, "learning_rate": 3.8070737232842614e-07, "logits/chosen": -1.3065639734268188, "logits/rejected": -1.0679255723953247, "logps/chosen": -374.07391357421875, "logps/rejected": -335.5907287597656, "loss": 0.2381, "rewards/accuracies": 0.90625, "rewards/chosen": -0.30956602096557617, "rewards/margins": 2.238311290740967, "rewards/rejected": -2.547877311706543, "step": 1170 }, { "epoch": 1.8557155451072584, "grad_norm": 39.42518997192383, "learning_rate": 3.678019706076039e-07, "logits/chosen": -1.3437250852584839, "logits/rejected": -1.1545518636703491, "logps/chosen": -377.6987609863281, "logps/rejected": -333.1975402832031, "loss": 0.2465, "rewards/accuracies": 0.9208333492279053, "rewards/chosen": -0.3938901126384735, "rewards/margins": 2.213219165802002, "rewards/rejected": -2.607109308242798, "step": 1185 }, { "epoch": 1.863551768047801, "eval_logits/chosen": -1.3109138011932373, "eval_logits/rejected": -1.1203300952911377, "eval_logps/chosen": -360.5334167480469, "eval_logps/rejected": -307.3110656738281, "eval_loss": 0.46806973218917847, "eval_rewards/accuracies": 0.7764363288879395, "eval_rewards/chosen": -0.8219457864761353, "eval_rewards/margins": 1.3658442497253418, "eval_rewards/rejected": -2.1877899169921875, "eval_runtime": 183.179, "eval_samples_per_second": 13.113, "eval_steps_per_second": 6.556, "step": 1190 }, { "epoch": 1.8792242139288864, "grad_norm": 35.16496276855469, "learning_rate": 3.5499058632329536e-07, "logits/chosen": -1.3551256656646729, "logits/rejected": -1.003768801689148, "logps/chosen": -343.90545654296875, "logps/rejected": -318.3465576171875, "loss": 0.2508, "rewards/accuracies": 0.9104166626930237, "rewards/chosen": -0.38343942165374756, "rewards/margins": 2.2458739280700684, "rewards/rejected": -2.6293132305145264, "step": 1200 }, { "epoch": 1.9027328827505143, "grad_norm": 40.798072814941406, "learning_rate": 3.4228233075737223e-07, "logits/chosen": -1.3036648035049438, "logits/rejected": -1.1327162981033325, "logps/chosen": -375.63385009765625, "logps/rejected": -311.95782470703125, "loss": 0.2338, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.48579010367393494, "rewards/margins": 2.2717065811157227, "rewards/rejected": -2.7574965953826904, "step": 1215 }, { "epoch": 1.9184053286315996, "eval_logits/chosen": -1.2938474416732788, "eval_logits/rejected": -1.1041306257247925, "eval_logps/chosen": -360.88861083984375, "eval_logps/rejected": -307.92333984375, "eval_loss": 0.4688262343406677, "eval_rewards/accuracies": 0.777685284614563, "eval_rewards/chosen": -0.8574661016464233, "eval_rewards/margins": 1.391554355621338, "eval_rewards/rejected": -2.2490200996398926, "eval_runtime": 183.0197, "eval_samples_per_second": 13.124, "eval_steps_per_second": 6.562, "step": 1225 }, { "epoch": 1.9262415515721423, "grad_norm": 25.399438858032227, "learning_rate": 3.2968624184796654e-07, "logits/chosen": -1.2950618267059326, "logits/rejected": -1.0804049968719482, "logps/chosen": -351.4801025390625, "logps/rejected": -326.7780456542969, "loss": 0.2601, "rewards/accuracies": 0.8895833492279053, "rewards/chosen": -0.4700445830821991, "rewards/margins": 2.1944639682769775, "rewards/rejected": -2.664508581161499, "step": 1230 }, { "epoch": 1.9497502203937702, "grad_norm": 25.898283004760742, "learning_rate": 3.1721127776181364e-07, "logits/chosen": -1.2864807844161987, "logits/rejected": -0.9478799104690552, "logps/chosen": -376.6116027832031, "logps/rejected": -320.6463623046875, "loss": 0.2545, "rewards/accuracies": 0.9166666865348816, "rewards/chosen": -0.3715561330318451, "rewards/margins": 2.254369020462036, "rewards/rejected": -2.6259255409240723, "step": 1245 }, { "epoch": 1.9732588892153982, "grad_norm": 29.050125122070312, "learning_rate": 3.0486631052332534e-07, "logits/chosen": -1.2400730848312378, "logits/rejected": -1.0113612413406372, "logps/chosen": -350.2702331542969, "logps/rejected": -303.7456359863281, "loss": 0.2545, "rewards/accuracies": 0.8916666507720947, "rewards/chosen": -0.4329583942890167, "rewards/margins": 2.1629526615142822, "rewards/rejected": -2.5959107875823975, "step": 1260 }, { "epoch": 1.9732588892153982, "eval_logits/chosen": -1.279727816581726, "eval_logits/rejected": -1.0888946056365967, "eval_logps/chosen": -360.5728454589844, "eval_logps/rejected": -307.5393981933594, "eval_loss": 0.4675270617008209, "eval_rewards/accuracies": 0.7735220789909363, "eval_rewards/chosen": -0.8258867859840393, "eval_rewards/margins": 1.3847379684448242, "eval_rewards/rejected": -2.210624933242798, "eval_runtime": 182.6913, "eval_samples_per_second": 13.148, "eval_steps_per_second": 6.574, "step": 1260 }, { "epoch": 1.996767558037026, "grad_norm": 34.406803131103516, "learning_rate": 2.9266011970492775e-07, "logits/chosen": -1.347962737083435, "logits/rejected": -1.1148425340652466, "logps/chosen": -360.68511962890625, "logps/rejected": -309.3987731933594, "loss": 0.2562, "rewards/accuracies": 0.9104166626930237, "rewards/chosen": -0.32166415452957153, "rewards/margins": 2.071373224258423, "rewards/rejected": -2.3930373191833496, "step": 1275 }, { "epoch": 2.0188069350573024, "grad_norm": 16.827268600463867, "learning_rate": 2.806013861831484e-07, "logits/chosen": -1.38746178150177, "logits/rejected": -1.0861637592315674, "logps/chosen": -369.17529296875, "logps/rejected": -314.560302734375, "loss": 0.1532, "rewards/accuracies": 0.9577777981758118, "rewards/chosen": -0.18039265275001526, "rewards/margins": 2.5118191242218018, "rewards/rejected": -2.692211627960205, "step": 1290 }, { "epoch": 2.026643157997845, "eval_logits/chosen": -1.3171147108078003, "eval_logits/rejected": -1.1266576051712036, "eval_logps/chosen": -361.1548767089844, "eval_logps/rejected": -308.2939758300781, "eval_loss": 0.4683936536312103, "eval_rewards/accuracies": 0.7735220789909363, "eval_rewards/chosen": -0.8840944170951843, "eval_rewards/margins": 1.4019862413406372, "eval_rewards/rejected": -2.286080837249756, "eval_runtime": 183.0594, "eval_samples_per_second": 13.121, "eval_steps_per_second": 6.561, "step": 1295 }, { "epoch": 2.0423156038789303, "grad_norm": 20.37907600402832, "learning_rate": 2.686986859648953e-07, "logits/chosen": -1.3132938146591187, "logits/rejected": -1.0580551624298096, "logps/chosen": -353.71502685546875, "logps/rejected": -316.21600341796875, "loss": 0.1542, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -0.22374172508716583, "rewards/margins": 2.5709550380706787, "rewards/rejected": -2.794696807861328, "step": 1305 }, { "epoch": 2.0658242727005582, "grad_norm": 31.620746612548828, "learning_rate": 2.5696048408831833e-07, "logits/chosen": -1.3634592294692993, "logits/rejected": -1.1027321815490723, "logps/chosen": -355.3858947753906, "logps/rejected": -324.46295166015625, "loss": 0.159, "rewards/accuracies": 0.9729166626930237, "rewards/chosen": -0.32911619544029236, "rewards/margins": 2.6501662731170654, "rewards/rejected": -2.979282855987549, "step": 1320 }, { "epoch": 2.0814967185816435, "eval_logits/chosen": -1.3730605840682983, "eval_logits/rejected": -1.1844755411148071, "eval_logps/chosen": -362.084716796875, "eval_logps/rejected": -309.5851745605469, "eval_loss": 0.4718104600906372, "eval_rewards/accuracies": 0.7722731232643127, "eval_rewards/chosen": -0.9770740866661072, "eval_rewards/margins": 1.438130497932434, "eval_rewards/rejected": -2.4152042865753174, "eval_runtime": 184.2931, "eval_samples_per_second": 13.034, "eval_steps_per_second": 6.517, "step": 1330 }, { "epoch": 2.089332941522186, "grad_norm": 24.57793426513672, "learning_rate": 2.4539512860258874e-07, "logits/chosen": -1.40749990940094, "logits/rejected": -1.219661831855774, "logps/chosen": -375.73541259765625, "logps/rejected": -332.43707275390625, "loss": 0.1548, "rewards/accuracies": 0.96875, "rewards/chosen": -0.33681097626686096, "rewards/margins": 2.658933401107788, "rewards/rejected": -2.9957449436187744, "step": 1335 }, { "epoch": 2.112841610343814, "grad_norm": 24.045860290527344, "learning_rate": 2.3401084463088095e-07, "logits/chosen": -1.3906124830245972, "logits/rejected": -1.10317862033844, "logps/chosen": -353.56524658203125, "logps/rejected": -316.1902160644531, "loss": 0.1594, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -0.5038239359855652, "rewards/margins": 2.6673552989959717, "rewards/rejected": -3.1711785793304443, "step": 1350 }, { "epoch": 2.136350279165442, "grad_norm": 15.747769355773926, "learning_rate": 2.2281572852077602e-07, "logits/chosen": -1.3541206121444702, "logits/rejected": -1.1997876167297363, "logps/chosen": -352.85357666015625, "logps/rejected": -318.2190246582031, "loss": 0.1644, "rewards/accuracies": 0.9520833492279053, "rewards/chosen": -0.3332318961620331, "rewards/margins": 2.7003839015960693, "rewards/rejected": -3.033615827560425, "step": 1365 }, { "epoch": 2.136350279165442, "eval_logits/chosen": -1.3989009857177734, "eval_logits/rejected": -1.2132837772369385, "eval_logps/chosen": -362.2392272949219, "eval_logps/rejected": -309.7401428222656, "eval_loss": 0.47149989008903503, "eval_rewards/accuracies": 0.7739383578300476, "eval_rewards/chosen": -0.9925247430801392, "eval_rewards/margins": 1.438172459602356, "eval_rewards/rejected": -2.430696964263916, "eval_runtime": 183.7972, "eval_samples_per_second": 13.069, "eval_steps_per_second": 6.534, "step": 1365 }, { "epoch": 2.15985894798707, "grad_norm": 19.07766342163086, "learning_rate": 2.1181774208625169e-07, "logits/chosen": -1.4445375204086304, "logits/rejected": -1.2832697629928589, "logps/chosen": -351.9157409667969, "logps/rejected": -310.8402404785156, "loss": 0.1382, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -0.36597153544425964, "rewards/margins": 2.7161738872528076, "rewards/rejected": -3.0821452140808105, "step": 1380 }, { "epoch": 2.1833676168086984, "grad_norm": 26.579313278198242, "learning_rate": 2.010247069453462e-07, "logits/chosen": -1.3750087022781372, "logits/rejected": -1.1288143396377563, "logps/chosen": -363.9015197753906, "logps/rejected": -326.6436767578125, "loss": 0.1696, "rewards/accuracies": 0.9479166865348816, "rewards/chosen": -0.4505316913127899, "rewards/margins": 2.662269115447998, "rewards/rejected": -3.1128008365631104, "step": 1395 }, { "epoch": 2.191203839749241, "eval_logits/chosen": -1.4125560522079468, "eval_logits/rejected": -1.2285879850387573, "eval_logps/chosen": -362.7055358886719, "eval_logps/rejected": -310.5049743652344, "eval_loss": 0.4731534719467163, "eval_rewards/accuracies": 0.7718567848205566, "eval_rewards/chosen": -1.0391572713851929, "eval_rewards/margins": 1.4680242538452148, "eval_rewards/rejected": -2.5071816444396973, "eval_runtime": 183.2882, "eval_samples_per_second": 13.105, "eval_steps_per_second": 6.553, "step": 1400 }, { "epoch": 2.2068762856303263, "grad_norm": 31.265209197998047, "learning_rate": 1.904442989575324e-07, "logits/chosen": -1.4215842485427856, "logits/rejected": -1.2600691318511963, "logps/chosen": -371.39471435546875, "logps/rejected": -328.9493408203125, "loss": 0.1491, "rewards/accuracies": 0.9666666388511658, "rewards/chosen": -0.3961075246334076, "rewards/margins": 2.718886375427246, "rewards/rejected": -3.1149940490722656, "step": 1410 }, { "epoch": 2.2303849544519543, "grad_norm": 19.422006607055664, "learning_rate": 1.800840427647492e-07, "logits/chosen": -1.4670078754425049, "logits/rejected": -1.1978638172149658, "logps/chosen": -366.31884765625, "logps/rejected": -310.2874450683594, "loss": 0.1714, "rewards/accuracies": 0.9604166746139526, "rewards/chosen": -0.40646833181381226, "rewards/margins": 2.6268420219421387, "rewards/rejected": -3.0333104133605957, "step": 1425 }, { "epoch": 2.2460574003330396, "eval_logits/chosen": -1.4236711263656616, "eval_logits/rejected": -1.2401769161224365, "eval_logps/chosen": -362.6986389160156, "eval_logps/rejected": -310.4892272949219, "eval_loss": 0.47280353307724, "eval_rewards/accuracies": 0.7726894021034241, "eval_rewards/chosen": -1.0384666919708252, "eval_rewards/margins": 1.4671398401260376, "eval_rewards/rejected": -2.5056064128875732, "eval_runtime": 183.4182, "eval_samples_per_second": 13.096, "eval_steps_per_second": 6.548, "step": 1435 }, { "epoch": 2.2538936232735822, "grad_norm": 22.030029296875, "learning_rate": 1.699513064399812e-07, "logits/chosen": -1.4837629795074463, "logits/rejected": -1.2376011610031128, "logps/chosen": -347.2449035644531, "logps/rejected": -323.3914489746094, "loss": 0.1497, "rewards/accuracies": 0.9708333611488342, "rewards/chosen": -0.4454355239868164, "rewards/margins": 2.712066650390625, "rewards/rejected": -3.1575021743774414, "step": 1440 }, { "epoch": 2.27740229209521, "grad_norm": 22.95709991455078, "learning_rate": 1.6005329624718272e-07, "logits/chosen": -1.3890153169631958, "logits/rejected": -1.2241790294647217, "logps/chosen": -359.70880126953125, "logps/rejected": -325.8527526855469, "loss": 0.1543, "rewards/accuracies": 0.9645833373069763, "rewards/chosen": -0.3865669369697571, "rewards/margins": 2.7711870670318604, "rewards/rejected": -3.1577537059783936, "step": 1455 }, { "epoch": 2.300910960916838, "grad_norm": 16.600971221923828, "learning_rate": 1.503970515162834e-07, "logits/chosen": -1.4796699285507202, "logits/rejected": -1.283089518547058, "logps/chosen": -346.07806396484375, "logps/rejected": -313.4685974121094, "loss": 0.161, "rewards/accuracies": 0.9666666388511658, "rewards/chosen": -0.3793748915195465, "rewards/margins": 2.6482481956481934, "rewards/rejected": -3.027622938156128, "step": 1470 }, { "epoch": 2.300910960916838, "eval_logits/chosen": -1.4251654148101807, "eval_logits/rejected": -1.2433199882507324, "eval_logps/chosen": -362.67913818359375, "eval_logps/rejected": -310.4259338378906, "eval_loss": 0.4735627770423889, "eval_rewards/accuracies": 0.7739383578300476, "eval_rewards/chosen": -1.0365179777145386, "eval_rewards/margins": 1.4627617597579956, "eval_rewards/rejected": -2.499279499053955, "eval_runtime": 183.8286, "eval_samples_per_second": 13.067, "eval_steps_per_second": 6.533, "step": 1470 }, { "epoch": 2.324419629738466, "grad_norm": 21.0710506439209, "learning_rate": 1.4098943963691178e-07, "logits/chosen": -1.469247817993164, "logits/rejected": -1.2050501108169556, "logps/chosen": -346.5850830078125, "logps/rejected": -338.1049499511719, "loss": 0.1465, "rewards/accuracies": 0.9729166626930237, "rewards/chosen": -0.35343825817108154, "rewards/margins": 2.7935383319854736, "rewards/rejected": -3.1469767093658447, "step": 1485 }, { "epoch": 2.347928298560094, "grad_norm": 22.436262130737305, "learning_rate": 1.3183715117440142e-07, "logits/chosen": -1.5117619037628174, "logits/rejected": -1.2061889171600342, "logps/chosen": -347.35565185546875, "logps/rejected": -311.8982238769531, "loss": 0.1468, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -0.4175235331058502, "rewards/margins": 2.734891653060913, "rewards/rejected": -3.1524152755737305, "step": 1500 }, { "epoch": 2.3557645215006366, "eval_logits/chosen": -1.4360556602478027, "eval_logits/rejected": -1.2548272609710693, "eval_logps/chosen": -363.044921875, "eval_logps/rejected": -311.045166015625, "eval_loss": 0.4749804139137268, "eval_rewards/accuracies": 0.7764363288879395, "eval_rewards/chosen": -1.0730968713760376, "eval_rewards/margins": 1.4881056547164917, "eval_rewards/rejected": -2.5612027645111084, "eval_runtime": 183.375, "eval_samples_per_second": 13.099, "eval_steps_per_second": 6.549, "step": 1505 }, { "epoch": 2.371436967381722, "grad_norm": 12.820830345153809, "learning_rate": 1.229466951115519e-07, "logits/chosen": -1.4124829769134521, "logits/rejected": -1.3270103931427002, "logps/chosen": -363.55633544921875, "logps/rejected": -315.1530456542969, "loss": 0.138, "rewards/accuracies": 0.9708333611488342, "rewards/chosen": -0.4280143082141876, "rewards/margins": 2.7927613258361816, "rewards/rejected": -3.220775604248047, "step": 1515 } ], "logging_steps": 15, "max_steps": 1917, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 40, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }