| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9984, |
| "eval_steps": 500, |
| "global_step": 468, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.010666666666666666, |
| "grad_norm": 11.732805335723311, |
| "learning_rate": 1.0638297872340425e-07, |
| "logits/chosen": -0.2567896246910095, |
| "logits/rejected": -0.2542869448661804, |
| "logps/chosen": -0.4896683692932129, |
| "logps/rejected": -0.47771158814430237, |
| "loss": 3.0994, |
| "rewards/accuracies": 0.4312500059604645, |
| "rewards/chosen": -4.896683692932129, |
| "rewards/margins": -0.11956818401813507, |
| "rewards/rejected": -4.777115821838379, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.021333333333333333, |
| "grad_norm": 10.469899900506974, |
| "learning_rate": 2.127659574468085e-07, |
| "logits/chosen": -0.3307115435600281, |
| "logits/rejected": -0.3123416602611542, |
| "logps/chosen": -0.5048332810401917, |
| "logps/rejected": -0.5071265697479248, |
| "loss": 3.0526, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -5.048332691192627, |
| "rewards/margins": 0.022932911291718483, |
| "rewards/rejected": -5.07126522064209, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.032, |
| "grad_norm": 11.130249026763659, |
| "learning_rate": 3.1914893617021275e-07, |
| "logits/chosen": -0.3020132780075073, |
| "logits/rejected": -0.29736775159835815, |
| "logps/chosen": -0.5014998316764832, |
| "logps/rejected": -0.49107274413108826, |
| "loss": 3.1181, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -5.014998435974121, |
| "rewards/margins": -0.10427095741033554, |
| "rewards/rejected": -4.910727500915527, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.042666666666666665, |
| "grad_norm": 11.30143344344881, |
| "learning_rate": 4.25531914893617e-07, |
| "logits/chosen": -0.3109033703804016, |
| "logits/rejected": -0.2914695143699646, |
| "logps/chosen": -0.5114676356315613, |
| "logps/rejected": -0.511113703250885, |
| "loss": 3.082, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -5.114676475524902, |
| "rewards/margins": -0.003539660479873419, |
| "rewards/rejected": -5.1111369132995605, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.05333333333333334, |
| "grad_norm": 10.854210486308324, |
| "learning_rate": 5.319148936170212e-07, |
| "logits/chosen": -0.2695028781890869, |
| "logits/rejected": -0.26218557357788086, |
| "logps/chosen": -0.4970417618751526, |
| "logps/rejected": -0.5018347501754761, |
| "loss": 3.0406, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -4.970417499542236, |
| "rewards/margins": 0.04793013259768486, |
| "rewards/rejected": -5.018347263336182, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.064, |
| "grad_norm": 10.469426816097055, |
| "learning_rate": 6.382978723404255e-07, |
| "logits/chosen": -0.26622432470321655, |
| "logits/rejected": -0.25182226300239563, |
| "logps/chosen": -0.5109531879425049, |
| "logps/rejected": -0.5139020681381226, |
| "loss": 3.0747, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -5.109531879425049, |
| "rewards/margins": 0.029488753527402878, |
| "rewards/rejected": -5.139019966125488, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07466666666666667, |
| "grad_norm": 10.631824817820393, |
| "learning_rate": 7.446808510638297e-07, |
| "logits/chosen": -0.24236233532428741, |
| "logits/rejected": -0.21462127566337585, |
| "logps/chosen": -0.4976476728916168, |
| "logps/rejected": -0.5021973848342896, |
| "loss": 3.0565, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -4.97647762298584, |
| "rewards/margins": 0.04549681395292282, |
| "rewards/rejected": -5.021974086761475, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.08533333333333333, |
| "grad_norm": 11.061177336739798, |
| "learning_rate": 8.51063829787234e-07, |
| "logits/chosen": -0.25426605343818665, |
| "logits/rejected": -0.21733298897743225, |
| "logps/chosen": -0.5162211656570435, |
| "logps/rejected": -0.5088873505592346, |
| "loss": 3.1144, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -5.1622114181518555, |
| "rewards/margins": -0.07333739101886749, |
| "rewards/rejected": -5.088873863220215, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.096, |
| "grad_norm": 10.752155973868787, |
| "learning_rate": 9.574468085106384e-07, |
| "logits/chosen": -0.2861650586128235, |
| "logits/rejected": -0.2492944896221161, |
| "logps/chosen": -0.5205284357070923, |
| "logps/rejected": -0.528952956199646, |
| "loss": 3.0191, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -5.2052836418151855, |
| "rewards/margins": 0.08424559980630875, |
| "rewards/rejected": -5.289529800415039, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.10666666666666667, |
| "grad_norm": 11.860710525100078, |
| "learning_rate": 9.998747147528373e-07, |
| "logits/chosen": -0.23958039283752441, |
| "logits/rejected": -0.2043449431657791, |
| "logps/chosen": -0.5231865644454956, |
| "logps/rejected": -0.5274239778518677, |
| "loss": 3.0309, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -5.231865882873535, |
| "rewards/margins": 0.04237395524978638, |
| "rewards/rejected": -5.274239540100098, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11733333333333333, |
| "grad_norm": 12.04146587440933, |
| "learning_rate": 9.991093100466482e-07, |
| "logits/chosen": -0.14567208290100098, |
| "logits/rejected": -0.11820191144943237, |
| "logps/chosen": -0.511086642742157, |
| "logps/rejected": -0.5330460071563721, |
| "loss": 3.0182, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -5.110866069793701, |
| "rewards/margins": 0.21959321200847626, |
| "rewards/rejected": -5.330460548400879, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.128, |
| "grad_norm": 18.468755064638835, |
| "learning_rate": 9.976491676662678e-07, |
| "logits/chosen": -0.18508504331111908, |
| "logits/rejected": -0.14062678813934326, |
| "logps/chosen": -0.5508230328559875, |
| "logps/rejected": -0.5570359826087952, |
| "loss": 3.0079, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -5.508230686187744, |
| "rewards/margins": 0.06212924048304558, |
| "rewards/rejected": -5.5703606605529785, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.13866666666666666, |
| "grad_norm": 11.705381703902521, |
| "learning_rate": 9.95496320064109e-07, |
| "logits/chosen": -0.17819233238697052, |
| "logits/rejected": -0.14004084467887878, |
| "logps/chosen": -0.5656185746192932, |
| "logps/rejected": -0.5872910618782043, |
| "loss": 2.9632, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -5.656185150146484, |
| "rewards/margins": 0.21672514081001282, |
| "rewards/rejected": -5.872910499572754, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.14933333333333335, |
| "grad_norm": 12.396143570324279, |
| "learning_rate": 9.926537639070456e-07, |
| "logits/chosen": -0.17236974835395813, |
| "logits/rejected": -0.13080790638923645, |
| "logps/chosen": -0.5695115327835083, |
| "logps/rejected": -0.6027958989143372, |
| "loss": 2.9352, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -5.695115089416504, |
| "rewards/margins": 0.33284443616867065, |
| "rewards/rejected": -6.027959823608398, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 11.876686176930853, |
| "learning_rate": 9.891254559051884e-07, |
| "logits/chosen": -0.19168080389499664, |
| "logits/rejected": -0.21411211788654327, |
| "logps/chosen": -0.5882201194763184, |
| "logps/rejected": -0.590097963809967, |
| "loss": 3.0043, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -5.882201671600342, |
| "rewards/margins": 0.01877792738378048, |
| "rewards/rejected": -5.900979042053223, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.17066666666666666, |
| "grad_norm": 12.284088688322067, |
| "learning_rate": 9.849163073043223e-07, |
| "logits/chosen": -0.21736201643943787, |
| "logits/rejected": -0.16785207390785217, |
| "logps/chosen": -0.5772870779037476, |
| "logps/rejected": -0.6213977336883545, |
| "loss": 2.9784, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -5.772871971130371, |
| "rewards/margins": 0.44110551476478577, |
| "rewards/rejected": -6.213976860046387, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18133333333333335, |
| "grad_norm": 11.997500432885564, |
| "learning_rate": 9.800321770496724e-07, |
| "logits/chosen": -0.20522455871105194, |
| "logits/rejected": -0.19095680117607117, |
| "logps/chosen": -0.5711790323257446, |
| "logps/rejected": -0.6087047457695007, |
| "loss": 2.8496, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -5.711790561676025, |
| "rewards/margins": 0.37525734305381775, |
| "rewards/rejected": -6.087048530578613, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.192, |
| "grad_norm": 12.297007234031984, |
| "learning_rate": 9.744798636305187e-07, |
| "logits/chosen": -0.21251866221427917, |
| "logits/rejected": -0.17701497673988342, |
| "logps/chosen": -0.5993844866752625, |
| "logps/rejected": -0.6125088930130005, |
| "loss": 3.0024, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -5.993845462799072, |
| "rewards/margins": 0.13124337792396545, |
| "rewards/rejected": -6.125088691711426, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.20266666666666666, |
| "grad_norm": 13.408809367946777, |
| "learning_rate": 9.68267095617003e-07, |
| "logits/chosen": -0.23027431964874268, |
| "logits/rejected": -0.21198849380016327, |
| "logps/chosen": -0.6354126334190369, |
| "logps/rejected": -0.6721860766410828, |
| "loss": 2.8925, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -6.3541259765625, |
| "rewards/margins": 0.36773455142974854, |
| "rewards/rejected": -6.721861362457275, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.21333333333333335, |
| "grad_norm": 12.920388414966602, |
| "learning_rate": 9.614025209023083e-07, |
| "logits/chosen": -0.2552734613418579, |
| "logits/rejected": -0.22966066002845764, |
| "logps/chosen": -0.6462411284446716, |
| "logps/rejected": -0.6792125701904297, |
| "loss": 2.9268, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -6.462411403656006, |
| "rewards/margins": 0.3297148644924164, |
| "rewards/rejected": -6.792125701904297, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.224, |
| "grad_norm": 13.637742964214516, |
| "learning_rate": 9.538956946651815e-07, |
| "logits/chosen": -0.2632606029510498, |
| "logits/rejected": -0.25038760900497437, |
| "logps/chosen": -0.6604565382003784, |
| "logps/rejected": -0.6915026903152466, |
| "loss": 2.9493, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -6.604565620422363, |
| "rewards/margins": 0.31046155095100403, |
| "rewards/rejected": -6.915026664733887, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.23466666666666666, |
| "grad_norm": 13.979440698873718, |
| "learning_rate": 9.457570660695539e-07, |
| "logits/chosen": -0.21594388782978058, |
| "logits/rejected": -0.18064400553703308, |
| "logps/chosen": -0.7305887937545776, |
| "logps/rejected": -0.7668353319168091, |
| "loss": 2.8873, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -7.305887699127197, |
| "rewards/margins": 0.36246541142463684, |
| "rewards/rejected": -7.668353080749512, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.24533333333333332, |
| "grad_norm": 13.926217101691021, |
| "learning_rate": 9.369979637197774e-07, |
| "logits/chosen": -0.24407616257667542, |
| "logits/rejected": -0.25407880544662476, |
| "logps/chosen": -0.67913419008255, |
| "logps/rejected": -0.7247743606567383, |
| "loss": 2.8965, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -6.791342258453369, |
| "rewards/margins": 0.4564014971256256, |
| "rewards/rejected": -7.247744083404541, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.256, |
| "grad_norm": 13.77277812367389, |
| "learning_rate": 9.276305798917158e-07, |
| "logits/chosen": -0.26810163259506226, |
| "logits/rejected": -0.24792496860027313, |
| "logps/chosen": -0.6853538751602173, |
| "logps/rejected": -0.7199205160140991, |
| "loss": 2.8996, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -6.853537559509277, |
| "rewards/margins": 0.3456660211086273, |
| "rewards/rejected": -7.199204444885254, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.26666666666666666, |
| "grad_norm": 14.10182994991175, |
| "learning_rate": 9.176679535616476e-07, |
| "logits/chosen": -0.27036887407302856, |
| "logits/rejected": -0.24968424439430237, |
| "logps/chosen": -0.6387192010879517, |
| "logps/rejected": -0.6508135795593262, |
| "loss": 2.9717, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -6.387192249298096, |
| "rewards/margins": 0.12094371020793915, |
| "rewards/rejected": -6.5081353187561035, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.2773333333333333, |
| "grad_norm": 13.356925694901157, |
| "learning_rate": 9.071239522565976e-07, |
| "logits/chosen": -0.26543933153152466, |
| "logits/rejected": -0.26540714502334595, |
| "logps/chosen": -0.6872167587280273, |
| "logps/rejected": -0.7145885825157166, |
| "loss": 2.9038, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -6.872167110443115, |
| "rewards/margins": 0.27371835708618164, |
| "rewards/rejected": -7.145886421203613, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.288, |
| "grad_norm": 15.7980283059669, |
| "learning_rate": 8.960132527513642e-07, |
| "logits/chosen": -0.2294592559337616, |
| "logits/rejected": -0.18822762370109558, |
| "logps/chosen": -0.7463935017585754, |
| "logps/rejected": -0.7754577398300171, |
| "loss": 2.9888, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -7.463934898376465, |
| "rewards/margins": 0.2906419336795807, |
| "rewards/rejected": -7.754576206207275, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.2986666666666667, |
| "grad_norm": 13.867652100686131, |
| "learning_rate": 8.8435132063911e-07, |
| "logits/chosen": -0.3160232901573181, |
| "logits/rejected": -0.26623860001564026, |
| "logps/chosen": -0.6544682383537292, |
| "logps/rejected": -0.688135027885437, |
| "loss": 2.9079, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -6.544682502746582, |
| "rewards/margins": 0.33666834235191345, |
| "rewards/rejected": -6.881350517272949, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.30933333333333335, |
| "grad_norm": 14.512621902026536, |
| "learning_rate": 8.721543888039532e-07, |
| "logits/chosen": -0.3039087653160095, |
| "logits/rejected": -0.30713099241256714, |
| "logps/chosen": -0.7150325775146484, |
| "logps/rejected": -0.7520148158073425, |
| "loss": 2.9382, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -7.150325775146484, |
| "rewards/margins": 0.36982235312461853, |
| "rewards/rejected": -7.520148277282715, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 14.436611201212568, |
| "learning_rate": 8.594394348255237e-07, |
| "logits/chosen": -0.34436482191085815, |
| "logits/rejected": -0.3056212067604065, |
| "logps/chosen": -0.7449463605880737, |
| "logps/rejected": -0.8144839406013489, |
| "loss": 2.8342, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -7.449461936950684, |
| "rewards/margins": 0.6953767538070679, |
| "rewards/rejected": -8.1448392868042, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33066666666666666, |
| "grad_norm": 13.682677476734403, |
| "learning_rate": 8.462241573469377e-07, |
| "logits/chosen": -0.3564246594905853, |
| "logits/rejected": -0.3462735414505005, |
| "logps/chosen": -0.7523462176322937, |
| "logps/rejected": -0.8084806203842163, |
| "loss": 2.9526, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -7.523462772369385, |
| "rewards/margins": 0.5613434910774231, |
| "rewards/rejected": -8.084805488586426, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.3413333333333333, |
| "grad_norm": 14.618544685980286, |
| "learning_rate": 8.325269514390834e-07, |
| "logits/chosen": -0.39723479747772217, |
| "logits/rejected": -0.3815813958644867, |
| "logps/chosen": -0.7263676524162292, |
| "logps/rejected": -0.7540715932846069, |
| "loss": 2.8924, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -7.26367712020874, |
| "rewards/margins": 0.2770387828350067, |
| "rewards/rejected": -7.54071569442749, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.352, |
| "grad_norm": 12.869926653197572, |
| "learning_rate": 8.183668829955111e-07, |
| "logits/chosen": -0.4017302095890045, |
| "logits/rejected": -0.386869877576828, |
| "logps/chosen": -0.6907881498336792, |
| "logps/rejected": -0.7190796732902527, |
| "loss": 2.9077, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -6.907881259918213, |
| "rewards/margins": 0.282915860414505, |
| "rewards/rejected": -7.190796852111816, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.3626666666666667, |
| "grad_norm": 14.999256839126888, |
| "learning_rate": 8.037636621935684e-07, |
| "logits/chosen": -0.38499245047569275, |
| "logits/rejected": -0.3716858923435211, |
| "logps/chosen": -0.6659557819366455, |
| "logps/rejected": -0.6983980536460876, |
| "loss": 2.8837, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -6.659557342529297, |
| "rewards/margins": 0.3244229853153229, |
| "rewards/rejected": -6.983981132507324, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.37333333333333335, |
| "grad_norm": 15.12004004484812, |
| "learning_rate": 7.887376160587213e-07, |
| "logits/chosen": -0.38568025827407837, |
| "logits/rejected": -0.361832857131958, |
| "logps/chosen": -0.7137908339500427, |
| "logps/rejected": -0.7906131744384766, |
| "loss": 2.9156, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -7.137907981872559, |
| "rewards/margins": 0.7682233452796936, |
| "rewards/rejected": -7.906131744384766, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.384, |
| "grad_norm": 13.114026390928577, |
| "learning_rate": 7.733096601702507e-07, |
| "logits/chosen": -0.38464826345443726, |
| "logits/rejected": -0.3654468357563019, |
| "logps/chosen": -0.6692460775375366, |
| "logps/rejected": -0.7017225027084351, |
| "loss": 2.9035, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -6.6924614906311035, |
| "rewards/margins": 0.32476380467414856, |
| "rewards/rejected": -7.017224311828613, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39466666666666667, |
| "grad_norm": 14.125998430016006, |
| "learning_rate": 7.575012695477076e-07, |
| "logits/chosen": -0.4029085040092468, |
| "logits/rejected": -0.38777080178260803, |
| "logps/chosen": -0.6527448892593384, |
| "logps/rejected": -0.7220766544342041, |
| "loss": 2.8666, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -6.527449607849121, |
| "rewards/margins": 0.6933171153068542, |
| "rewards/rejected": -7.220765590667725, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.4053333333333333, |
| "grad_norm": 13.049863611060802, |
| "learning_rate": 7.413344487586542e-07, |
| "logits/chosen": -0.3920533061027527, |
| "logits/rejected": -0.3556768596172333, |
| "logps/chosen": -0.6590090990066528, |
| "logps/rejected": -0.7017866373062134, |
| "loss": 2.8632, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -6.590091705322266, |
| "rewards/margins": 0.42777472734451294, |
| "rewards/rejected": -7.0178656578063965, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.416, |
| "grad_norm": 14.676013098665036, |
| "learning_rate": 7.248317012892968e-07, |
| "logits/chosen": -0.43694472312927246, |
| "logits/rejected": -0.4099256098270416, |
| "logps/chosen": -0.6819888353347778, |
| "logps/rejected": -0.7181512117385864, |
| "loss": 2.9495, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -6.819889068603516, |
| "rewards/margins": 0.36162319779396057, |
| "rewards/rejected": -7.181512355804443, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.4266666666666667, |
| "grad_norm": 15.713156748679351, |
| "learning_rate": 7.08015998220647e-07, |
| "logits/chosen": -0.4263511300086975, |
| "logits/rejected": -0.4301479458808899, |
| "logps/chosen": -0.7229773998260498, |
| "logps/rejected": -0.7311898469924927, |
| "loss": 2.9526, |
| "rewards/accuracies": 0.45625001192092896, |
| "rewards/chosen": -7.22977352142334, |
| "rewards/margins": 0.08212558925151825, |
| "rewards/rejected": -7.311899662017822, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.43733333333333335, |
| "grad_norm": 13.220260563253577, |
| "learning_rate": 6.909107462538111e-07, |
| "logits/chosen": -0.3825225234031677, |
| "logits/rejected": -0.35514765977859497, |
| "logps/chosen": -0.689651370048523, |
| "logps/rejected": -0.7226651906967163, |
| "loss": 2.8976, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -6.896513938903809, |
| "rewards/margins": 0.3301384747028351, |
| "rewards/rejected": -7.2266526222229, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.448, |
| "grad_norm": 14.184681407574283, |
| "learning_rate": 6.735397551289178e-07, |
| "logits/chosen": -0.3693349361419678, |
| "logits/rejected": -0.34236425161361694, |
| "logps/chosen": -0.7217646837234497, |
| "logps/rejected": -0.7970829010009766, |
| "loss": 2.8931, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -7.217646598815918, |
| "rewards/margins": 0.7531824111938477, |
| "rewards/rejected": -7.970829010009766, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.45866666666666667, |
| "grad_norm": 13.969746933429045, |
| "learning_rate": 6.559272044830316e-07, |
| "logits/chosen": -0.4436856210231781, |
| "logits/rejected": -0.43038907647132874, |
| "logps/chosen": -0.7077959775924683, |
| "logps/rejected": -0.7319620847702026, |
| "loss": 2.9769, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -7.0779595375061035, |
| "rewards/margins": 0.24166087806224823, |
| "rewards/rejected": -7.319620609283447, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.4693333333333333, |
| "grad_norm": 15.643175869810904, |
| "learning_rate": 6.380976101931879e-07, |
| "logits/chosen": -0.4383578300476074, |
| "logits/rejected": -0.4107333719730377, |
| "logps/chosen": -0.6930493116378784, |
| "logps/rejected": -0.7319644689559937, |
| "loss": 2.798, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -6.930493354797363, |
| "rewards/margins": 0.38915205001831055, |
| "rewards/rejected": -7.319644927978516, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 14.071461506962004, |
| "learning_rate": 6.200757902513962e-07, |
| "logits/chosen": -0.4078153073787689, |
| "logits/rejected": -0.37852078676223755, |
| "logps/chosen": -0.7787854671478271, |
| "logps/rejected": -0.8084853291511536, |
| "loss": 2.9259, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -7.787854194641113, |
| "rewards/margins": 0.2969990670681, |
| "rewards/rejected": -8.084854125976562, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.49066666666666664, |
| "grad_norm": 14.70735554158027, |
| "learning_rate": 6.018868302191139e-07, |
| "logits/chosen": -0.39765116572380066, |
| "logits/rejected": -0.37791210412979126, |
| "logps/chosen": -0.7530070543289185, |
| "logps/rejected": -0.7994042634963989, |
| "loss": 2.9924, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -7.530069828033447, |
| "rewards/margins": 0.46397218108177185, |
| "rewards/rejected": -7.99404239654541, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5013333333333333, |
| "grad_norm": 15.209803885325115, |
| "learning_rate": 5.835560483092742e-07, |
| "logits/chosen": -0.43543845415115356, |
| "logits/rejected": -0.400688111782074, |
| "logps/chosen": -0.7642709612846375, |
| "logps/rejected": -0.8203216791152954, |
| "loss": 2.879, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -7.642709255218506, |
| "rewards/margins": 0.56050705909729, |
| "rewards/rejected": -8.203216552734375, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.512, |
| "grad_norm": 17.403184515776818, |
| "learning_rate": 5.651089601444752e-07, |
| "logits/chosen": -0.4311800003051758, |
| "logits/rejected": -0.42130523920059204, |
| "logps/chosen": -0.8272747993469238, |
| "logps/rejected": -0.8601797223091125, |
| "loss": 2.9416, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -8.272747993469238, |
| "rewards/margins": 0.3290492594242096, |
| "rewards/rejected": -8.601797103881836, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5226666666666666, |
| "grad_norm": 14.85502463503016, |
| "learning_rate": 5.465712432403811e-07, |
| "logits/chosen": -0.477522611618042, |
| "logits/rejected": -0.43428006768226624, |
| "logps/chosen": -0.7840465307235718, |
| "logps/rejected": -0.8169954419136047, |
| "loss": 2.8989, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -7.8404645919799805, |
| "rewards/margins": 0.32948940992355347, |
| "rewards/rejected": -8.169954299926758, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5333333333333333, |
| "grad_norm": 14.518585077709004, |
| "learning_rate": 5.279687012637798e-07, |
| "logits/chosen": -0.5134187936782837, |
| "logits/rejected": -0.4909071922302246, |
| "logps/chosen": -0.7666435241699219, |
| "logps/rejected": -0.8026981353759766, |
| "loss": 2.9357, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -7.666435241699219, |
| "rewards/margins": 0.36054641008377075, |
| "rewards/rejected": -8.02698040008545, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.544, |
| "grad_norm": 18.35565839056467, |
| "learning_rate": 5.093272281150382e-07, |
| "logits/chosen": -0.4790865480899811, |
| "logits/rejected": -0.45897984504699707, |
| "logps/chosen": -0.8199352025985718, |
| "logps/rejected": -0.8931600451469421, |
| "loss": 2.9983, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -8.199353218078613, |
| "rewards/margins": 0.7322477698326111, |
| "rewards/rejected": -8.931600570678711, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5546666666666666, |
| "grad_norm": 14.14082105154021, |
| "learning_rate": 4.906727718849618e-07, |
| "logits/chosen": -0.5037890672683716, |
| "logits/rejected": -0.4815933108329773, |
| "logps/chosen": -0.6989194750785828, |
| "logps/rejected": -0.7129279375076294, |
| "loss": 2.8415, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -6.989194393157959, |
| "rewards/margins": 0.14008501172065735, |
| "rewards/rejected": -7.129279136657715, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5653333333333334, |
| "grad_norm": 13.279665907018064, |
| "learning_rate": 4.7203129873622036e-07, |
| "logits/chosen": -0.4404175877571106, |
| "logits/rejected": -0.4400077760219574, |
| "logps/chosen": -0.7215054631233215, |
| "logps/rejected": -0.7625397443771362, |
| "loss": 2.8401, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -7.215054512023926, |
| "rewards/margins": 0.4103423058986664, |
| "rewards/rejected": -7.625396728515625, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.576, |
| "grad_norm": 13.746628428841586, |
| "learning_rate": 4.534287567596188e-07, |
| "logits/chosen": -0.5114065408706665, |
| "logits/rejected": -0.49541646242141724, |
| "logps/chosen": -0.7559850811958313, |
| "logps/rejected": -0.7878262400627136, |
| "loss": 2.8697, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -7.559851169586182, |
| "rewards/margins": 0.31841152906417847, |
| "rewards/rejected": -7.878262519836426, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5866666666666667, |
| "grad_norm": 15.406038129926618, |
| "learning_rate": 4.348910398555249e-07, |
| "logits/chosen": -0.5457953214645386, |
| "logits/rejected": -0.4762287139892578, |
| "logps/chosen": -0.7813851237297058, |
| "logps/rejected": -0.8478168249130249, |
| "loss": 2.8572, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -7.813851833343506, |
| "rewards/margins": 0.6643169522285461, |
| "rewards/rejected": -8.478169441223145, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5973333333333334, |
| "grad_norm": 14.070560615997694, |
| "learning_rate": 4.1644395169072575e-07, |
| "logits/chosen": -0.538383424282074, |
| "logits/rejected": -0.5068949460983276, |
| "logps/chosen": -0.8055634498596191, |
| "logps/rejected": -0.8467632532119751, |
| "loss": 2.8542, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -8.055633544921875, |
| "rewards/margins": 0.4119985103607178, |
| "rewards/rejected": -8.467633247375488, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.608, |
| "grad_norm": 14.39885856794737, |
| "learning_rate": 3.9811316978088615e-07, |
| "logits/chosen": -0.4664926528930664, |
| "logits/rejected": -0.4394296705722809, |
| "logps/chosen": -0.8325020670890808, |
| "logps/rejected": -0.9406170845031738, |
| "loss": 2.903, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -8.325021743774414, |
| "rewards/margins": 1.0811493396759033, |
| "rewards/rejected": -9.406169891357422, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.6186666666666667, |
| "grad_norm": 14.11854363733785, |
| "learning_rate": 3.799242097486038e-07, |
| "logits/chosen": -0.46769508719444275, |
| "logits/rejected": -0.4558941721916199, |
| "logps/chosen": -0.8696973919868469, |
| "logps/rejected": -0.9103899002075195, |
| "loss": 2.9108, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -8.696972846984863, |
| "rewards/margins": 0.40692609548568726, |
| "rewards/rejected": -9.103899955749512, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6293333333333333, |
| "grad_norm": 13.09705398003357, |
| "learning_rate": 3.619023898068123e-07, |
| "logits/chosen": -0.4639492630958557, |
| "logits/rejected": -0.40117162466049194, |
| "logps/chosen": -0.8408247828483582, |
| "logps/rejected": -0.9566766619682312, |
| "loss": 2.8343, |
| "rewards/accuracies": 0.581250011920929, |
| "rewards/chosen": -8.408247947692871, |
| "rewards/margins": 1.1585181951522827, |
| "rewards/rejected": -9.566766738891602, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 14.74843010829423, |
| "learning_rate": 3.4407279551696846e-07, |
| "logits/chosen": -0.4986533522605896, |
| "logits/rejected": -0.47083503007888794, |
| "logps/chosen": -0.8254755139350891, |
| "logps/rejected": -0.8391461372375488, |
| "loss": 3.0153, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -8.254755020141602, |
| "rewards/margins": 0.13670669496059418, |
| "rewards/rejected": -8.391461372375488, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6506666666666666, |
| "grad_norm": 13.906424975643212, |
| "learning_rate": 3.2646024487108213e-07, |
| "logits/chosen": -0.4865950644016266, |
| "logits/rejected": -0.4660443365573883, |
| "logps/chosen": -0.7600260972976685, |
| "logps/rejected": -0.8434259295463562, |
| "loss": 2.8537, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -7.6002607345581055, |
| "rewards/margins": 0.8339985609054565, |
| "rewards/rejected": -8.434259414672852, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.6613333333333333, |
| "grad_norm": 14.40884629312328, |
| "learning_rate": 3.0908925374618887e-07, |
| "logits/chosen": -0.5216479301452637, |
| "logits/rejected": -0.485219806432724, |
| "logps/chosen": -0.7851314544677734, |
| "logps/rejected": -0.8616622686386108, |
| "loss": 2.8632, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -7.851315498352051, |
| "rewards/margins": 0.7653086185455322, |
| "rewards/rejected": -8.616623878479004, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.672, |
| "grad_norm": 15.731408776836984, |
| "learning_rate": 2.91984001779353e-07, |
| "logits/chosen": -0.5304073095321655, |
| "logits/rejected": -0.5378426313400269, |
| "logps/chosen": -0.812860369682312, |
| "logps/rejected": -0.8587455749511719, |
| "loss": 2.9407, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -8.1286039352417, |
| "rewards/margins": 0.45885151624679565, |
| "rewards/rejected": -8.587456703186035, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6826666666666666, |
| "grad_norm": 14.656113742338617, |
| "learning_rate": 2.751682987107029e-07, |
| "logits/chosen": -0.5356782078742981, |
| "logits/rejected": -0.5007396936416626, |
| "logps/chosen": -0.798676073551178, |
| "logps/rejected": -0.8625814318656921, |
| "loss": 2.8971, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -7.98676061630249, |
| "rewards/margins": 0.6390543580055237, |
| "rewards/rejected": -8.625814437866211, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6933333333333334, |
| "grad_norm": 15.918152402610213, |
| "learning_rate": 2.5866555124134577e-07, |
| "logits/chosen": -0.5085067749023438, |
| "logits/rejected": -0.5119614601135254, |
| "logps/chosen": -0.764369785785675, |
| "logps/rejected": -0.7916201949119568, |
| "loss": 2.8012, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -7.643698215484619, |
| "rewards/margins": 0.27250438928604126, |
| "rewards/rejected": -7.916202545166016, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.704, |
| "grad_norm": 14.925354413062523, |
| "learning_rate": 2.424987304522924e-07, |
| "logits/chosen": -0.5266901254653931, |
| "logits/rejected": -0.4882999360561371, |
| "logps/chosen": -0.7844299077987671, |
| "logps/rejected": -0.8563938140869141, |
| "loss": 2.8274, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -7.84429931640625, |
| "rewards/margins": 0.719638466835022, |
| "rewards/rejected": -8.563937187194824, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7146666666666667, |
| "grad_norm": 15.521210965830875, |
| "learning_rate": 2.2669033982974944e-07, |
| "logits/chosen": -0.5255261063575745, |
| "logits/rejected": -0.5050898790359497, |
| "logps/chosen": -0.7706557512283325, |
| "logps/rejected": -0.8051782846450806, |
| "loss": 2.8849, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -7.706557273864746, |
| "rewards/margins": 0.3452245891094208, |
| "rewards/rejected": -8.05178165435791, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7253333333333334, |
| "grad_norm": 16.440289438668614, |
| "learning_rate": 2.1126238394127867e-07, |
| "logits/chosen": -0.49614062905311584, |
| "logits/rejected": -0.4573017954826355, |
| "logps/chosen": -0.8138383626937866, |
| "logps/rejected": -0.8721502423286438, |
| "loss": 2.8235, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -8.138383865356445, |
| "rewards/margins": 0.5831184387207031, |
| "rewards/rejected": -8.721502304077148, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.736, |
| "grad_norm": 26.868935766217437, |
| "learning_rate": 1.9623633780643155e-07, |
| "logits/chosen": -0.485830694437027, |
| "logits/rejected": -0.4887874722480774, |
| "logps/chosen": -0.824845016002655, |
| "logps/rejected": -0.8695980310440063, |
| "loss": 2.9745, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -8.248449325561523, |
| "rewards/margins": 0.4475303292274475, |
| "rewards/rejected": -8.6959810256958, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.7466666666666667, |
| "grad_norm": 14.695673192972276, |
| "learning_rate": 1.8163311700448898e-07, |
| "logits/chosen": -0.5577523112297058, |
| "logits/rejected": -0.5077025294303894, |
| "logps/chosen": -0.7684929370880127, |
| "logps/rejected": -0.824779212474823, |
| "loss": 2.7818, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -7.684928894042969, |
| "rewards/margins": 0.5628635287284851, |
| "rewards/rejected": -8.247793197631836, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7573333333333333, |
| "grad_norm": 15.834449897172663, |
| "learning_rate": 1.674730485609166e-07, |
| "logits/chosen": -0.5468079447746277, |
| "logits/rejected": -0.5012609958648682, |
| "logps/chosen": -0.7990966439247131, |
| "logps/rejected": -0.8461319804191589, |
| "loss": 2.943, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -7.990965843200684, |
| "rewards/margins": 0.4703536927700043, |
| "rewards/rejected": -8.461319923400879, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.768, |
| "grad_norm": 15.386612940043147, |
| "learning_rate": 1.537758426530622e-07, |
| "logits/chosen": -0.5115416646003723, |
| "logits/rejected": -0.4699644446372986, |
| "logps/chosen": -0.8920881152153015, |
| "logps/rejected": -0.9769732356071472, |
| "loss": 2.7639, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -8.920880317687988, |
| "rewards/margins": 0.8488510847091675, |
| "rewards/rejected": -9.769731521606445, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7786666666666666, |
| "grad_norm": 15.656467749234869, |
| "learning_rate": 1.4056056517447634e-07, |
| "logits/chosen": -0.5276600122451782, |
| "logits/rejected": -0.49349960684776306, |
| "logps/chosen": -0.8332331776618958, |
| "logps/rejected": -0.9140599966049194, |
| "loss": 2.9014, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -8.332330703735352, |
| "rewards/margins": 0.808269202709198, |
| "rewards/rejected": -9.140600204467773, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7893333333333333, |
| "grad_norm": 17.277302293211417, |
| "learning_rate": 1.2784561119604682e-07, |
| "logits/chosen": -0.5142337679862976, |
| "logits/rejected": -0.48429346084594727, |
| "logps/chosen": -0.8171199560165405, |
| "logps/rejected": -0.9075886607170105, |
| "loss": 2.8581, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -8.1712007522583, |
| "rewards/margins": 0.9046867489814758, |
| "rewards/rejected": -9.075886726379395, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 15.192850018209791, |
| "learning_rate": 1.156486793608899e-07, |
| "logits/chosen": -0.5062755346298218, |
| "logits/rejected": -0.4999562203884125, |
| "logps/chosen": -0.8260412216186523, |
| "logps/rejected": -0.8741272687911987, |
| "loss": 2.795, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -8.260412216186523, |
| "rewards/margins": 0.48086076974868774, |
| "rewards/rejected": -8.74127197265625, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.8106666666666666, |
| "grad_norm": 16.990615643980465, |
| "learning_rate": 1.0398674724863581e-07, |
| "logits/chosen": -0.49277371168136597, |
| "logits/rejected": -0.5056179761886597, |
| "logps/chosen": -0.8936569094657898, |
| "logps/rejected": -0.9541202783584595, |
| "loss": 2.9408, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -8.936570167541504, |
| "rewards/margins": 0.6046336889266968, |
| "rewards/rejected": -9.541203498840332, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8213333333333334, |
| "grad_norm": 15.62223343762673, |
| "learning_rate": 9.287604774340235e-08, |
| "logits/chosen": -0.5418406128883362, |
| "logits/rejected": -0.5228685140609741, |
| "logps/chosen": -0.8050872087478638, |
| "logps/rejected": -0.880950927734375, |
| "loss": 2.8933, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -8.050871849060059, |
| "rewards/margins": 0.7586367726325989, |
| "rewards/rejected": -8.80950927734375, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.832, |
| "grad_norm": 14.628553467618065, |
| "learning_rate": 8.233204643835234e-08, |
| "logits/chosen": -0.5186488628387451, |
| "logits/rejected": -0.531872570514679, |
| "logps/chosen": -0.824377715587616, |
| "logps/rejected": -0.841328501701355, |
| "loss": 2.9315, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -8.24377727508545, |
| "rewards/margins": 0.1695086658000946, |
| "rewards/rejected": -8.413286209106445, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.8426666666666667, |
| "grad_norm": 16.098240638683464, |
| "learning_rate": 7.236942010828429e-08, |
| "logits/chosen": -0.5238023400306702, |
| "logits/rejected": -0.5310443043708801, |
| "logps/chosen": -0.8085333704948425, |
| "logps/rejected": -0.8205422163009644, |
| "loss": 2.818, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -8.085333824157715, |
| "rewards/margins": 0.12008871883153915, |
| "rewards/rejected": -8.205423355102539, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8533333333333334, |
| "grad_norm": 18.5937182107884, |
| "learning_rate": 6.300203628022271e-08, |
| "logits/chosen": -0.5449612736701965, |
| "logits/rejected": -0.5202792882919312, |
| "logps/chosen": -0.7899402379989624, |
| "logps/rejected": -0.8360351324081421, |
| "loss": 2.892, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -7.899402618408203, |
| "rewards/margins": 0.46094831824302673, |
| "rewards/rejected": -8.3603515625, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.864, |
| "grad_norm": 27.476647128759996, |
| "learning_rate": 5.42429339304461e-08, |
| "logits/chosen": -0.44413289427757263, |
| "logits/rejected": -0.42316532135009766, |
| "logps/chosen": -0.8095054626464844, |
| "logps/rejected": -0.8948721885681152, |
| "loss": 2.9365, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -8.095054626464844, |
| "rewards/margins": 0.8536664843559265, |
| "rewards/rejected": -8.948721885681152, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.8746666666666667, |
| "grad_norm": 16.376688883362988, |
| "learning_rate": 4.610430533481857e-08, |
| "logits/chosen": -0.5368673801422119, |
| "logits/rejected": -0.5123342275619507, |
| "logps/chosen": -0.8019696474075317, |
| "logps/rejected": -0.8381240963935852, |
| "loss": 2.8302, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -8.019696235656738, |
| "rewards/margins": 0.36154404282569885, |
| "rewards/rejected": -8.38123893737793, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8853333333333333, |
| "grad_norm": 14.616440154757584, |
| "learning_rate": 3.859747909769162e-08, |
| "logits/chosen": -0.5262904167175293, |
| "logits/rejected": -0.493990421295166, |
| "logps/chosen": -0.7962188720703125, |
| "logps/rejected": -0.8315852880477905, |
| "loss": 2.8724, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -7.962189674377441, |
| "rewards/margins": 0.3536640703678131, |
| "rewards/rejected": -8.315853118896484, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.896, |
| "grad_norm": 15.49360265689598, |
| "learning_rate": 3.173290438299697e-08, |
| "logits/chosen": -0.5184012651443481, |
| "logits/rejected": -0.49308672547340393, |
| "logps/chosen": -0.7899567484855652, |
| "logps/rejected": -0.8208295702934265, |
| "loss": 2.8914, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -7.8995680809021, |
| "rewards/margins": 0.3087272644042969, |
| "rewards/rejected": -8.208294868469238, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.9066666666666666, |
| "grad_norm": 15.840634841357453, |
| "learning_rate": 2.5520136369481194e-08, |
| "logits/chosen": -0.5332978367805481, |
| "logits/rejected": -0.5073381662368774, |
| "logps/chosen": -0.7771866321563721, |
| "logps/rejected": -0.8295711278915405, |
| "loss": 2.8116, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -7.771866798400879, |
| "rewards/margins": 0.5238449573516846, |
| "rewards/rejected": -8.295711517333984, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.9173333333333333, |
| "grad_norm": 15.841915213065072, |
| "learning_rate": 1.996782295032745e-08, |
| "logits/chosen": -0.4586135745048523, |
| "logits/rejected": -0.452159583568573, |
| "logps/chosen": -0.8384187817573547, |
| "logps/rejected": -0.8715507388114929, |
| "loss": 2.8993, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -8.384187698364258, |
| "rewards/margins": 0.3313189148902893, |
| "rewards/rejected": -8.715507507324219, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.928, |
| "grad_norm": 15.55344706729787, |
| "learning_rate": 1.508369269567783e-08, |
| "logits/chosen": -0.496934711933136, |
| "logits/rejected": -0.4732961654663086, |
| "logps/chosen": -0.8125640153884888, |
| "logps/rejected": -0.8842247724533081, |
| "loss": 2.8844, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -8.125639915466309, |
| "rewards/margins": 0.7166072130203247, |
| "rewards/rejected": -8.84224796295166, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9386666666666666, |
| "grad_norm": 13.681145246098804, |
| "learning_rate": 1.0874544094811422e-08, |
| "logits/chosen": -0.5602749586105347, |
| "logits/rejected": -0.521491527557373, |
| "logps/chosen": -0.7614573240280151, |
| "logps/rejected": -0.817762553691864, |
| "loss": 2.8496, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -7.614572048187256, |
| "rewards/margins": 0.5630531907081604, |
| "rewards/rejected": -8.17762565612793, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9493333333333334, |
| "grad_norm": 19.437700888018206, |
| "learning_rate": 7.346236092954316e-09, |
| "logits/chosen": -0.5439106225967407, |
| "logits/rejected": -0.5047253966331482, |
| "logps/chosen": -0.8181453943252563, |
| "logps/rejected": -0.8678085207939148, |
| "loss": 2.8608, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -8.181453704833984, |
| "rewards/margins": 0.4966312348842621, |
| "rewards/rejected": -8.678085327148438, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 15.340465523330005, |
| "learning_rate": 4.50367993589107e-09, |
| "logits/chosen": -0.5867766737937927, |
| "logits/rejected": -0.576478123664856, |
| "logps/chosen": -0.7500745058059692, |
| "logps/rejected": -0.7640315890312195, |
| "loss": 2.9123, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -7.5007452964782715, |
| "rewards/margins": 0.13957078754901886, |
| "rewards/rejected": -7.640316009521484, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9706666666666667, |
| "grad_norm": 16.396666538032456, |
| "learning_rate": 2.3508323337321224e-09, |
| "logits/chosen": -0.5304497480392456, |
| "logits/rejected": -0.4988110661506653, |
| "logps/chosen": -0.7995911836624146, |
| "logps/rejected": -0.8580360412597656, |
| "loss": 2.8268, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -7.995911598205566, |
| "rewards/margins": 0.584447979927063, |
| "rewards/rejected": -8.580358505249023, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9813333333333333, |
| "grad_norm": 14.774707254643907, |
| "learning_rate": 8.906899533517864e-10, |
| "logits/chosen": -0.4948996901512146, |
| "logits/rejected": -0.48175182938575745, |
| "logps/chosen": -0.7842663526535034, |
| "logps/rejected": -0.7961934804916382, |
| "loss": 2.9312, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -7.8426642417907715, |
| "rewards/margins": 0.1192709431052208, |
| "rewards/rejected": -7.961934566497803, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.992, |
| "grad_norm": 20.715790321715897, |
| "learning_rate": 1.252852471625987e-10, |
| "logits/chosen": -0.5066460371017456, |
| "logits/rejected": -0.4637778699398041, |
| "logps/chosen": -0.8228558301925659, |
| "logps/rejected": -0.9186854362487793, |
| "loss": 2.9005, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -8.228558540344238, |
| "rewards/margins": 0.9582956433296204, |
| "rewards/rejected": -9.186854362487793, |
| "step": 465 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 468, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 300, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 219889542889472.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
